From 3c7499facf5f7e8b7db26b2b31fc7220df7c6beb Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Wed, 11 Oct 2023 16:42:29 -0600 Subject: [PATCH 01/24] test: mark unit test secrets as not secret There are 2 cases of randomly generated secrets copied into Rook's unit test code that have been flagged by Gitleaks. Add a comment to both cases to help the tool understand that these aren't real production secrets -- just unit test stand-ins. Signed-off-by: Blaine Gardner --- pkg/operator/ceph/object/controller_test.go | 2 +- pkg/operator/ceph/object/user/controller_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/operator/ceph/object/controller_test.go b/pkg/operator/ceph/object/controller_test.go index efdd303538cb..76f3fcfe930d 100644 --- a/pkg/operator/ceph/object/controller_test.go +++ b/pkg/operator/ceph/object/controller_test.go @@ -348,7 +348,7 @@ var mockMultisiteAdminOpsCtxFunc = func(objContext *Context, spec *cephv1.Object return &AdminOpsContext{ Context: *context, AdminOpsUserAccessKey: "EOE7FYCNOBZJ5VFV909G", - AdminOpsUserSecretKey: "qmIqpWm8HxCzmynCrD6U6vKWi4hnDBndOnmxXNsV", + AdminOpsUserSecretKey: "qmIqpWm8HxCzmynCrD6U6vKWi4hnDBndOnmxXNsV", // notsecret AdminOpsClient: adminClient, }, nil } diff --git a/pkg/operator/ceph/object/user/controller_test.go b/pkg/operator/ceph/object/user/controller_test.go index 4094b1a719ac..498e6a503e5a 100644 --- a/pkg/operator/ceph/object/user/controller_test.go +++ b/pkg/operator/ceph/object/user/controller_test.go @@ -316,7 +316,7 @@ func TestCephObjectStoreUserController(t *testing.T) { return &cephobject.AdminOpsContext{ Context: *context, AdminOpsUserAccessKey: "53S6B9S809NUP19IJ2K3", - AdminOpsUserSecretKey: "1bXPegzsGClvoGAiJdHQD1uOW2sQBLAZM9j9VtXR", + AdminOpsUserSecretKey: "1bXPegzsGClvoGAiJdHQD1uOW2sQBLAZM9j9VtXR", // notsecret AdminOpsClient: adminClient, }, nil } From 25d89d2b681fd1d5a36d3a2a1e9e5681c4b8d584 Mon Sep 17 00:00:00 2001 From: Bin Wang Date: Sun, 15 Oct 2023 22:54:11 -0400 Subject: [PATCH 02/24] doc: fix document about viewing dashboard outside of cluster https://github.com/rook/rook/pull/11845 added dynamic label `mgr_role`. The example yaml files are updated but this doc is missed. I think it would be better to refer to `deploy/examples` but I find all other places are not doing that. So keep it inlined to be consistent. Signed-off-by: Bin Wang --- Documentation/Storage-Configuration/Monitoring/ceph-dashboard.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/Storage-Configuration/Monitoring/ceph-dashboard.md b/Documentation/Storage-Configuration/Monitoring/ceph-dashboard.md index a2053a19f53b..684a0bf009ca 100755 --- a/Documentation/Storage-Configuration/Monitoring/ceph-dashboard.md +++ b/Documentation/Storage-Configuration/Monitoring/ceph-dashboard.md @@ -133,6 +133,7 @@ spec: selector: app: rook-ceph-mgr rook_cluster: rook-ceph + mgr_role: active sessionAffinity: None type: NodePort ``` From dd7a7b3f61f454c9c4f6567b0e34671dc3b34146 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Oct 2023 12:22:42 +0000 Subject: [PATCH 03/24] build(deps): bump the github-dependencies group with 3 updates Bumps the github-dependencies group with 3 updates: [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go), [github.com/ceph/go-ceph](https://github.com/ceph/go-ceph) and [github.com/google/go-cmp](https://github.com/google/go-cmp). Updates `github.com/aws/aws-sdk-go` from 1.45.24 to 1.45.25 - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.45.24...v1.45.25) Updates `github.com/ceph/go-ceph` from 0.23.0 to 0.24.0 - [Release notes](https://github.com/ceph/go-ceph/releases) - [Changelog](https://github.com/ceph/go-ceph/blob/master/docs/release-process.md) - [Commits](https://github.com/ceph/go-ceph/compare/v0.23.0...v0.24.0) Updates `github.com/google/go-cmp` from 0.5.9 to 0.6.0 - [Release notes](https://github.com/google/go-cmp/releases) - [Commits](https://github.com/google/go-cmp/compare/v0.5.9...v0.6.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-dependencies - dependency-name: github.com/ceph/go-ceph dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-dependencies - dependency-name: github.com/google/go-cmp dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 960aedbe434c..23d2d783d33a 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,14 @@ go 1.20 require ( github.com/IBM/keyprotect-go-client v0.12.2 - github.com/aws/aws-sdk-go v1.45.24 + github.com/aws/aws-sdk-go v1.45.25 github.com/banzaicloud/k8s-objectmatcher v1.8.0 - github.com/ceph/go-ceph v0.23.0 + github.com/ceph/go-ceph v0.24.0 github.com/coreos/pkg v0.0.0-20230601102743-20bbbf26f4d8 github.com/csi-addons/kubernetes-csi-addons v0.7.0 github.com/gemalto/kmip-go v0.0.10 github.com/go-ini/ini v1.67.0 - github.com/google/go-cmp v0.5.9 + github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.3.1 github.com/hashicorp/vault/api v1.10.0 github.com/jetstack/cert-manager v1.7.3 diff --git a/go.sum b/go.sum index 06246e3ad7b5..30b6b0494e2a 100644 --- a/go.sum +++ b/go.sum @@ -450,8 +450,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.44.164/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.45.24 h1:TZx/CizkmCQn8Rtsb11iLYutEQVGK5PK9wAhwouELBo= -github.com/aws/aws-sdk-go v1.45.24/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.45.25 h1:c4fLlh5sLdK2DCRTY1z0hyuJZU4ygxX8m1FswL6/nF4= +github.com/aws/aws-sdk-go v1.45.25/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/banzaicloud/k8s-objectmatcher v1.8.0 h1:Nugn25elKtPMTA2br+JgHNeSQ04sc05MDPmpJnd1N2A= github.com/banzaicloud/k8s-objectmatcher v1.8.0/go.mod h1:p2LSNAjlECf07fbhDyebTkPUIYnU05G+WfGgkTmgeMg= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -468,8 +468,8 @@ github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4r github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= -github.com/ceph/go-ceph v0.23.0 h1:b/glx9y7vXQJd5m6VqfNv9krYHny7W/+Zc3fNyTbIIo= -github.com/ceph/go-ceph v0.23.0/go.mod h1:QDYBF+MBdj9B5nSAvevZOrh5WSOflN15t/tZ67b17Qw= +github.com/ceph/go-ceph v0.24.0 h1:ab1pQCTiNrwjJJJ3bebwQM9tjDQ4tXGKfXAZBNdFiYI= +github.com/ceph/go-ceph v0.24.0/go.mod h1:gdL5+ewDeHcbV4ZsfD3EH3na35trT07YaTVD1hhJWEg= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -728,8 +728,9 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= From 4a705b46b04cb2d6f56d94f1a7707f8adc5f1b9c Mon Sep 17 00:00:00 2001 From: Bin Wang Date: Mon, 16 Oct 2023 00:18:23 -0400 Subject: [PATCH 04/24] osd: print warning message if no matching node found for osd As seen in https://github.com/rook/rook/issues/1988, it's a common mistake to configure rook nodes with names that don't match Kubernetes's node label. This PR prints more detailed message to help debugging problem. Signed-off-by: Bin Wang Add break according to review comment Co-authored-by: Travis Nielsen Update log message according to review comment Co-authored-by: Travis Nielsen --- pkg/operator/k8sutil/node.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/operator/k8sutil/node.go b/pkg/operator/k8sutil/node.go index d9cb27a255c3..0962c26929ad 100644 --- a/pkg/operator/k8sutil/node.go +++ b/pkg/operator/k8sutil/node.go @@ -281,12 +281,18 @@ func GetKubernetesNodesMatchingRookNodes(ctx context.Context, rookNodes []cephv1 if err != nil { return nodes, fmt.Errorf("failed to list kubernetes nodes. %+v", err) } - for _, kn := range k8sNodes.Items { - for _, rn := range rookNodes { + for _, rn := range rookNodes { + nodeFound := false + for _, kn := range k8sNodes.Items { if rookNodeMatchesKubernetesNode(rn, kn) { nodes = append(nodes, kn) + nodeFound = true + break } } + if !nodeFound { + logger.Warningf("failed to find matching kubernetes node for %q. Check the CephCluster's config and confirm each 'name' field in spec.storage.nodes matches their 'kubernetes.io/hostname' label", rn.Name) + } } return nodes, nil } From b186150e1c6a5d25d91e693623a1a18ed2607529 Mon Sep 17 00:00:00 2001 From: subhamkrai Date: Wed, 18 Oct 2023 12:19:52 +0530 Subject: [PATCH 05/24] ci: fix mkdocs deprecated imports migrate to using mkdocs-material's material.extensions.emoji.twemoji and material.extensions.emoji.to_svg in place of the respective materialx.emoji.twemoji and materialx.emoji.to_svg. Also, this requires latest version of mkdocs-material. Signed-off-by: subhamkrai --- build/release/requirements_docs.txt | 2 +- mkdocs.yml | 29 +++++++++++++++-------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/build/release/requirements_docs.txt b/build/release/requirements_docs.txt index 6833874ebc1c..56068eba11c9 100644 --- a/build/release/requirements_docs.txt +++ b/build/release/requirements_docs.txt @@ -3,7 +3,7 @@ mkdocs mkdocs-awesome-pages-plugin mkdocs-exclude mkdocs-macros-plugin -mkdocs-material==8.* +mkdocs-material mkdocs-material-extensions mkdocs-minify-plugin mkdocs-redirects diff --git a/mkdocs.yml b/mkdocs.yml index fe2cbdf7149d..6b7b20db610f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,6 +1,6 @@ site_name: Rook Ceph Documentation docs_dir: Documentation/ -site_url: 'https://rook.io' +site_url: "https://rook.io" repo_url: https://github.com/rook/rook edit_uri: edit/master/Documentation/ site_author: Rook Authors @@ -26,15 +26,15 @@ theme: favicon: https://rook.io/images/favicon_192x192.png logo: https://rook.io/images/rook-logo.svg palette: - - scheme: 'default' - primary: 'rook-blue' - accent: 'deep orange' + - scheme: "default" + primary: "rook-blue" + accent: "deep orange" toggle: icon: material/toggle-switch-off-outline name: Switch to dark mode - - scheme: 'slate' - primary: 'rook-blue' - accent: 'red' + - scheme: "slate" + primary: "rook-blue" + accent: "red" toggle: icon: material/toggle-switch name: Switch to light mode @@ -75,10 +75,11 @@ plugins: README.md: Getting-Started/intro.md - mike: # these fields are all optional; the defaults are as below... - version_selector: true # set to false to leave out the version selector - css_dir: css # the directory to put the version selector's CSS - javascript_dir: js # the directory to put the version selector's JS - canonical_version: null # the version for ; `null` + version_selector: true # set to false to leave out the version selector + css_dir: css # the directory to put the version selector's CSS + javascript_dir: js # the directory to put the version selector's JS + canonical_version: + null # the version for ; `null` # uses the version specified via `mike deploy` markdown_extensions: - admonition @@ -91,8 +92,8 @@ markdown_extensions: - tables - pymdownx.details - pymdownx.emoji: - emoji_index: !!python/name:materialx.emoji.twemoji - emoji_generator: !!python/name:materialx.emoji.to_svg + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg - pymdownx.highlight: anchor_linenums: true use_pygments: true @@ -116,6 +117,6 @@ extra: - icon: fontawesome/brands/twitter link: https://twitter.com/rook_io - icon: fontawesome/solid/envelopes-bulk - link: 'https://groups.google.com/forum/#!forum/rook-dev' + link: "https://groups.google.com/forum/#!forum/rook-dev" - icon: fontawesome/brands/medium link: https://blog.rook.io/ From 0e132f45a6948859114af8d65e4c0ef50a46cbcc Mon Sep 17 00:00:00 2001 From: Redouane Kachach Date: Wed, 18 Oct 2023 14:33:46 +0200 Subject: [PATCH 06/24] docs: adding specific instructions to upgrade Rook toolbox Closes: https://github.com/rook/rook/issues/13010 Signed-off-by: Redouane Kachach --- Documentation/Upgrade/ceph-upgrade.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Documentation/Upgrade/ceph-upgrade.md b/Documentation/Upgrade/ceph-upgrade.md index f60f793f59d1..a302940b3081 100644 --- a/Documentation/Upgrade/ceph-upgrade.md +++ b/Documentation/Upgrade/ceph-upgrade.md @@ -85,7 +85,18 @@ NEW_CEPH_IMAGE='quay.io/ceph/ceph:v17.2.6-20230410' kubectl -n $ROOK_CLUSTER_NAMESPACE patch CephCluster $ROOK_CLUSTER_NAMESPACE --type=merge -p "{\"spec\": {\"cephVersion\": {\"image\": \"$NEW_CEPH_IMAGE\"}}}" ``` -#### **2. Wait for the pod updates** +#### **2. Update the toolbox image** + +Since the [Rook toolbox](https://rook.io/docs/rook/latest/Troubleshooting/ceph-toolbox/) is not controlled by +the Rook operator, users must perform a manual upgrade by modifying the `image` to match the ceph version +employed by the new Rook operator release. Employing an outdated Ceph version within the toolbox may result +in unexpected behaviour. + +```console +kubectl -n rook-ceph set image deploy/rook-ceph-tools rook-ceph-tools=quay.io/ceph/ceph:v17.2.6-20230410 +``` + +#### **3. Wait for the pod updates** As with upgrading Rook, now wait for the upgrade to complete. Status can be determined in a similar way to the Rook upgrade as well. @@ -105,6 +116,6 @@ This cluster is finished: ceph-version=v17.2.6-0 ``` -#### **3. Verify cluster health** +#### **4. Verify cluster health** Verify the Ceph cluster's health using the [health verification](health-verification.md). From 67759f0fea63c11aaa17d16d51d91ac8bf8ff9c4 Mon Sep 17 00:00:00 2001 From: jxlwqq Date: Thu, 19 Oct 2023 13:44:34 +0800 Subject: [PATCH 07/24] docs: fix install command Signed-off-by: jxlwqq --- Documentation/Troubleshooting/kubectl-plugin.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/Troubleshooting/kubectl-plugin.md b/Documentation/Troubleshooting/kubectl-plugin.md index c4f80338f334..0cb210fad66d 100644 --- a/Documentation/Troubleshooting/kubectl-plugin.md +++ b/Documentation/Troubleshooting/kubectl-plugin.md @@ -19,7 +19,7 @@ See the [kubectl-rook-ceph documentation](https://github.com/rook/kubectl-rook-c - Install Rook plugin ```console - kubectl kubectl install rook-ceph + kubectl krew install rook-ceph ``` ## Ceph Commands From a421143dddd31f8786d232a21ddda3de81f5b186 Mon Sep 17 00:00:00 2001 From: travisn Date: Fri, 20 Oct 2023 14:57:26 -0600 Subject: [PATCH 08/24] Revert "core: use crash profile in crash daemon keyring" This reverts commit fab23d34072dac479a719ce362a448d02e856542. The mgr requires rw access for the cron job that collects the crashes. Signed-off-by: Travis Nielsen --- pkg/operator/ceph/cluster/nodedaemon/keyring.go | 4 ++-- pkg/operator/ceph/cluster/nodedaemon/keyring_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/operator/ceph/cluster/nodedaemon/keyring.go b/pkg/operator/ceph/cluster/nodedaemon/keyring.go index 0472ba3e799d..af59694bf086 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/keyring.go +++ b/pkg/operator/ceph/cluster/nodedaemon/keyring.go @@ -34,7 +34,7 @@ const ( [client.crash] key = %s caps mon = "allow profile crash" - caps mgr = "allow profile crash" + caps mgr = "allow rw" ` ) @@ -59,7 +59,7 @@ func CreateCrashCollectorSecret(context *clusterd.Context, clusterInfo *client.C func cephCrashCollectorKeyringCaps() []string { return []string{ "mon", "allow profile crash", - "mgr", "allow profile crash", + "mgr", "allow rw", } } diff --git a/pkg/operator/ceph/cluster/nodedaemon/keyring_test.go b/pkg/operator/ceph/cluster/nodedaemon/keyring_test.go index 376c23b60ba1..4dc9f244c92b 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/keyring_test.go +++ b/pkg/operator/ceph/cluster/nodedaemon/keyring_test.go @@ -24,5 +24,5 @@ import ( func TestCephCrashCollectorKeyringCaps(t *testing.T) { caps := cephCrashCollectorKeyringCaps() - assert.Equal(t, caps, []string{"mon", "allow profile crash", "mgr", "allow profile crash"}) + assert.Equal(t, caps, []string{"mon", "allow profile crash", "mgr", "allow rw"}) } From 559720f4cee83532275abc868571478788299043 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Oct 2023 12:27:51 +0000 Subject: [PATCH 09/24] build(deps): bump the k8s-dependencies group with 5 updates Bumps the k8s-dependencies group with 5 updates: | Package | From | To | | --- | --- | --- | | [k8s.io/api](https://github.com/kubernetes/api) | `0.28.2` | `0.28.3` | | [k8s.io/apiextensions-apiserver](https://github.com/kubernetes/apiextensions-apiserver) | `0.28.2` | `0.28.3` | | [k8s.io/cli-runtime](https://github.com/kubernetes/cli-runtime) | `0.28.2` | `0.28.3` | | [k8s.io/cloud-provider](https://github.com/kubernetes/cloud-provider) | `0.28.2` | `0.28.3` | | [sigs.k8s.io/controller-runtime](https://github.com/kubernetes-sigs/controller-runtime) | `0.16.2` | `0.16.3` | Updates `k8s.io/api` from 0.28.2 to 0.28.3 - [Commits](https://github.com/kubernetes/api/compare/v0.28.2...v0.28.3) Updates `k8s.io/apiextensions-apiserver` from 0.28.2 to 0.28.3 - [Release notes](https://github.com/kubernetes/apiextensions-apiserver/releases) - [Commits](https://github.com/kubernetes/apiextensions-apiserver/compare/v0.28.2...v0.28.3) Updates `k8s.io/cli-runtime` from 0.28.2 to 0.28.3 - [Commits](https://github.com/kubernetes/cli-runtime/compare/v0.28.2...v0.28.3) Updates `k8s.io/cloud-provider` from 0.28.2 to 0.28.3 - [Commits](https://github.com/kubernetes/cloud-provider/compare/v0.28.2...v0.28.3) Updates `sigs.k8s.io/controller-runtime` from 0.16.2 to 0.16.3 - [Release notes](https://github.com/kubernetes-sigs/controller-runtime/releases) - [Changelog](https://github.com/kubernetes-sigs/controller-runtime/blob/main/RELEASE.md) - [Commits](https://github.com/kubernetes-sigs/controller-runtime/compare/v0.16.2...v0.16.3) --- updated-dependencies: - dependency-name: k8s.io/api dependency-type: direct:production update-type: version-update:semver-patch dependency-group: k8s-dependencies - dependency-name: k8s.io/apiextensions-apiserver dependency-type: direct:production update-type: version-update:semver-patch dependency-group: k8s-dependencies - dependency-name: k8s.io/cli-runtime dependency-type: direct:production update-type: version-update:semver-patch dependency-group: k8s-dependencies - dependency-name: k8s.io/cloud-provider dependency-type: direct:production update-type: version-update:semver-patch dependency-group: k8s-dependencies - dependency-name: sigs.k8s.io/controller-runtime dependency-type: direct:production update-type: version-update:semver-patch dependency-group: k8s-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 16 ++++++++-------- go.sum | 32 ++++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 23d2d783d33a..f250195229b9 100644 --- a/go.mod +++ b/go.mod @@ -30,14 +30,14 @@ require ( golang.org/x/sync v0.4.0 gopkg.in/ini.v1 v1.67.0 gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.28.2 - k8s.io/apiextensions-apiserver v0.28.2 - k8s.io/apimachinery v0.28.2 - k8s.io/cli-runtime v0.28.2 - k8s.io/client-go v0.28.2 - k8s.io/cloud-provider v0.28.2 + k8s.io/api v0.28.3 + k8s.io/apiextensions-apiserver v0.28.3 + k8s.io/apimachinery v0.28.3 + k8s.io/cli-runtime v0.28.3 + k8s.io/client-go v0.28.3 + k8s.io/cloud-provider v0.28.3 k8s.io/utils v0.0.0-20230726121419-3b25d923346b - sigs.k8s.io/controller-runtime v0.16.2 + sigs.k8s.io/controller-runtime v0.16.3 sigs.k8s.io/mcs-api v0.1.0 sigs.k8s.io/yaml v1.3.0 ) @@ -129,7 +129,7 @@ require ( google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/component-base v0.28.2 // indirect + k8s.io/component-base v0.28.3 // indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index 30b6b0494e2a..a07eb91929cd 100644 --- a/go.sum +++ b/go.sum @@ -1876,37 +1876,37 @@ k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78= k8s.io/api v0.18.4/go.mod h1:lOIQAKYgai1+vz9J7YcDZwC26Z0zQewYOGWdyIPUUQ4= k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8= k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg= -k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw= -k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg= +k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM= +k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc= k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY= k8s.io/apiextensions-apiserver v0.18.4/go.mod h1:NYeyeYq4SIpFlPxSAB6jHPIdvu3hL0pc36wuRChybio= -k8s.io/apiextensions-apiserver v0.28.2 h1:J6/QRWIKV2/HwBhHRVITMLYoypCoPY1ftigDM0Kn+QU= -k8s.io/apiextensions-apiserver v0.28.2/go.mod h1:5tnkxLGa9nefefYzWuAlWZ7RZYuN/765Au8cWLA6SRg= +k8s.io/apiextensions-apiserver v0.28.3 h1:Od7DEnhXHnHPZG+W9I97/fSQkVpVPQx2diy+2EtmY08= +k8s.io/apiextensions-apiserver v0.28.3/go.mod h1:NE1XJZ4On0hS11aWWJUTNkmVB03j9LM7gJSisbRt8Lc= k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= k8s.io/apimachinery v0.18.4/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= -k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ= -k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU= +k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A= +k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8= k8s.io/apiserver v0.18.2/go.mod h1:Xbh066NqrZO8cbsoenCwyDJ1OSi8Ag8I2lezeHxzwzw= k8s.io/apiserver v0.18.4/go.mod h1:q+zoFct5ABNnYkGIaGQ3bcbUNdmPyOCoEBcg51LChY8= -k8s.io/cli-runtime v0.28.2 h1:64meB2fDj10/ThIMEJLO29a1oujSm0GQmKzh1RtA/uk= -k8s.io/cli-runtime v0.28.2/go.mod h1:bTpGOvpdsPtDKoyfG4EG041WIyFZLV9qq4rPlkyYfDA= +k8s.io/cli-runtime v0.28.3 h1:lvuJYVkwCqHEvpS6KuTZsUVwPePFjBfSGvuaLl2SxzA= +k8s.io/cli-runtime v0.28.3/go.mod h1:jeX37ZPjIcENVuXDDTskG3+FnVuZms5D9omDXS/2Jjc= k8s.io/client-go v0.18.2/go.mod h1:Xcm5wVGXX9HAA2JJ2sSBUn3tCJ+4SVlCbl2MNNv+CIU= k8s.io/client-go v0.18.4/go.mod h1:f5sXwL4yAZRkAtzOxRWUhA/N8XzGCb+nPZI8PfobZ9g= k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4= -k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY= -k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY= -k8s.io/cloud-provider v0.28.2 h1:9qsYm86hm4bnPgZbl9LE29Zfgjuq3NZR2dgtPioJ40s= -k8s.io/cloud-provider v0.28.2/go.mod h1:40fqf6MtgYho5Eu4gkyLgh5abxU/QKTMTIwBxt4ILyU= +k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4= +k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo= +k8s.io/cloud-provider v0.28.3 h1:9u+JjA3zIn0nqLOOa8tWnprFkffguSAhfBvo8p7LhBQ= +k8s.io/cloud-provider v0.28.3/go.mod h1:shAJxdrKu+SwwGUhkodxByPjaH8KBFZqXo6jU1F0ehI= k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= k8s.io/code-generator v0.18.4/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM= k8s.io/component-base v0.18.4/go.mod h1:7jr/Ef5PGmKwQhyAz/pjByxJbC58mhKAhiaDu0vXfPk= -k8s.io/component-base v0.28.2 h1:Yc1yU+6AQSlpJZyvehm/NkJBII72rzlEsd6MkBQ+G0E= -k8s.io/component-base v0.28.2/go.mod h1:4IuQPQviQCg3du4si8GpMrhAIegxpsgPngPRR/zWpzc= +k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= +k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= @@ -1944,8 +1944,8 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= sigs.k8s.io/controller-runtime v0.6.1/go.mod h1:XRYBPdbf5XJu9kpS84VJiZ7h/u1hF3gEORz0efEja7A= -sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU= -sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= +sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= +sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= sigs.k8s.io/controller-tools v0.3.0/go.mod h1:enhtKGfxZD1GFEoMgP8Fdbu+uKQ/cq1/WGJhdVChfvI= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= From b6aefea144f59cf79e0e297a1899915f88f0f95f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Oct 2023 12:29:06 +0000 Subject: [PATCH 10/24] build(deps): bump the github-dependencies group with 1 update Bumps the github-dependencies group with 1 update: [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go). - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.45.25...v1.46.1) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 23d2d783d33a..63239ffd4726 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/IBM/keyprotect-go-client v0.12.2 - github.com/aws/aws-sdk-go v1.45.25 + github.com/aws/aws-sdk-go v1.46.1 github.com/banzaicloud/k8s-objectmatcher v1.8.0 github.com/ceph/go-ceph v0.24.0 github.com/coreos/pkg v0.0.0-20230601102743-20bbbf26f4d8 diff --git a/go.sum b/go.sum index 30b6b0494e2a..a7b864bb1f34 100644 --- a/go.sum +++ b/go.sum @@ -450,8 +450,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.44.164/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.45.25 h1:c4fLlh5sLdK2DCRTY1z0hyuJZU4ygxX8m1FswL6/nF4= -github.com/aws/aws-sdk-go v1.45.25/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.46.1 h1:U26quvBWFZMQuultLw5tloW4GnmWaChEwMZNq8uYatw= +github.com/aws/aws-sdk-go v1.46.1/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/banzaicloud/k8s-objectmatcher v1.8.0 h1:Nugn25elKtPMTA2br+JgHNeSQ04sc05MDPmpJnd1N2A= github.com/banzaicloud/k8s-objectmatcher v1.8.0/go.mod h1:p2LSNAjlECf07fbhDyebTkPUIYnU05G+WfGgkTmgeMg= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= From 40008bf18ddf0bfe7426e3cb130ce466257deed5 Mon Sep 17 00:00:00 2001 From: parth-gr Date: Wed, 18 Oct 2023 20:17:31 +0530 Subject: [PATCH 11/24] external: automatically init the rbd pool Sometimes the pool is not enabled and the ceph cluster shows health warning, so automatically initalize the pool from the script Signed-off-by: parth-gr --- deploy/examples/create-external-cluster-resources.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/deploy/examples/create-external-cluster-resources.py b/deploy/examples/create-external-cluster-resources.py index 148f7d68b7da..83a24ed4857c 100644 --- a/deploy/examples/create-external-cluster-resources.py +++ b/deploy/examples/create-external-cluster-resources.py @@ -1299,6 +1299,14 @@ def validate_rbd_pool(self): f"The provided pool, '{self._arg_parser.rbd_data_pool_name}', does not exist" ) + def init_rbd_pool(self): + if isinstance(self.cluster, DummyRados): + return + rbd_pool_name = self._arg_parser.rbd_data_pool_name + ioctx = self.cluster.open_ioctx(rbd_pool_name) + rbd_inst = rbd.RBD() + rbd_inst.pool_init(ioctx, True) + def validate_rados_namespace(self): rbd_pool_name = self._arg_parser.rbd_data_pool_name rados_namespace = self._arg_parser.rados_namespace @@ -1467,6 +1475,7 @@ def _gen_output_map(self): self._arg_parser.k8s_cluster_name.lower() ) # always convert cluster name to lowercase characters self.validate_rbd_pool() + self.init_rbd_pool() self.validate_rados_namespace() self._excluded_keys.add("K8S_CLUSTER_NAME") self.get_cephfs_data_pool_details() From 3363a195a80666674145d7f3d99465b80fe4c3d4 Mon Sep 17 00:00:00 2001 From: parth-gr Date: Wed, 18 Oct 2023 19:50:50 +0530 Subject: [PATCH 12/24] external: fix monitoring endpoint check if the monitoring endpoint is not present we were not returning any error, but the field is the mandatory output, so return error if not found Signed-off-by: parth-gr --- .../create-external-cluster-resources-tests.py | 15 +++++++++++++-- .../examples/create-external-cluster-resources.py | 5 ++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/deploy/examples/create-external-cluster-resources-tests.py b/deploy/examples/create-external-cluster-resources-tests.py index df37f0d0ca4b..b3a5196c7493 100644 --- a/deploy/examples/create-external-cluster-resources-tests.py +++ b/deploy/examples/create-external-cluster-resources-tests.py @@ -240,11 +240,22 @@ def test_skip_monitoring_endpoint_no_prometheus(self): del cmd_json_out["mgrmap"]["services"]["prometheus"] self.rjObj.cluster.cmd_output_map[cmd_key] = json.dumps(cmd_json_out) - endpoint, port = self.rjObj.get_active_and_standby_mgrs() + endpoint = "" + port = "" + try: + endpoint, port = self.rjObj.get_active_and_standby_mgrs() + self.fail("An Exception was expected to be thrown") + except ext.ExecutionFailureException as err: + print(f"Successfully thrown error: {err}") + if endpoint != "" or port != "": self.fail("Expected monitoring endpoint and port to be empty") - self.rjObj.main() + try: + self.rjObj.main() + self.fail("An Exception was expected to be thrown") + except ext.ExecutionFailureException as err: + print(f"Successfully thrown error: {err}") if self.rjObj.out_map["MONITORING_ENDPOINT"] != "": self.fail("MONITORING_ENDPOINT should be empty") diff --git a/deploy/examples/create-external-cluster-resources.py b/deploy/examples/create-external-cluster-resources.py index 148f7d68b7da..ff66ef93e844 100644 --- a/deploy/examples/create-external-cluster-resources.py +++ b/deploy/examples/create-external-cluster-resources.py @@ -775,7 +775,10 @@ def get_active_and_standby_mgrs(self): json_out.get("mgrmap", {}).get("services", {}).get("prometheus", "") ) if not monitoring_endpoint: - return "", "" + raise ExecutionFailureException( + "can't find monitoring_endpoint, prometheus module might not be enabled, " + "enable the module by running 'ceph mgr module enable prometheus'" + ) # now check the stand-by mgr-s standby_arr = json_out.get("mgrmap", {}).get("standbys", []) for each_standby in standby_arr: From ff081cc4aed7a562a56d9be1cfb230ee964d568f Mon Sep 17 00:00:00 2001 From: Redouane Kachach Date: Mon, 23 Oct 2023 15:02:50 +0200 Subject: [PATCH 13/24] namespace: adding namespace to all rook-ceph namespaces references To make it easier for the user to switch to a namespace other than the default 'rook-ceph', the tag '# namespace:cluster' must be inserted in all the places where this namespace is used. This way, a simple 'sed' command can update all the Yaml files to the new namespace. Signed-off-by: Redouane Kachach --- deploy/examples/cluster-on-local-pvc.yaml | 2 +- deploy/examples/dashboard-external-http.yaml | 2 +- deploy/examples/dashboard-external-https.yaml | 2 +- deploy/examples/dashboard-loadbalancer.yaml | 2 +- deploy/examples/monitoring/exporter-service-monitor.yaml | 2 +- deploy/examples/monitoring/service-monitor.yaml | 2 +- deploy/examples/nfs-load-balancer.yaml | 2 +- deploy/examples/osd-env-override.yaml | 2 +- deploy/examples/pool-mirrored.yaml | 2 +- deploy/examples/sqlitevfs-client.yaml | 2 +- deploy/examples/volume-replication-class.yaml | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/deploy/examples/cluster-on-local-pvc.yaml b/deploy/examples/cluster-on-local-pvc.yaml index 2963c1720528..4e9eadd99a59 100644 --- a/deploy/examples/cluster-on-local-pvc.yaml +++ b/deploy/examples/cluster-on-local-pvc.yaml @@ -160,7 +160,7 @@ apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: name: rook-ceph - namespace: rook-ceph + namespace: rook-ceph # namespace:cluster spec: dataDirHostPath: /var/lib/rook mon: diff --git a/deploy/examples/dashboard-external-http.yaml b/deploy/examples/dashboard-external-http.yaml index 82338367f554..8d9f72d419c7 100644 --- a/deploy/examples/dashboard-external-http.yaml +++ b/deploy/examples/dashboard-external-http.yaml @@ -15,6 +15,6 @@ spec: selector: app: rook-ceph-mgr mgr_role: active - rook_cluster: rook-ceph + rook_cluster: rook-ceph # namespace:cluster sessionAffinity: None type: NodePort diff --git a/deploy/examples/dashboard-external-https.yaml b/deploy/examples/dashboard-external-https.yaml index 6a18dd30ceaa..86d46b847942 100644 --- a/deploy/examples/dashboard-external-https.yaml +++ b/deploy/examples/dashboard-external-https.yaml @@ -15,6 +15,6 @@ spec: selector: app: rook-ceph-mgr mgr_role: active - rook_cluster: rook-ceph + rook_cluster: rook-ceph # namespace:cluster sessionAffinity: None type: NodePort diff --git a/deploy/examples/dashboard-loadbalancer.yaml b/deploy/examples/dashboard-loadbalancer.yaml index 19fe8eb5c302..1785621bd5b3 100644 --- a/deploy/examples/dashboard-loadbalancer.yaml +++ b/deploy/examples/dashboard-loadbalancer.yaml @@ -15,6 +15,6 @@ spec: selector: app: rook-ceph-mgr mgr_role: active - rook_cluster: rook-ceph + rook_cluster: rook-ceph # namespace:cluster sessionAffinity: None type: LoadBalancer diff --git a/deploy/examples/monitoring/exporter-service-monitor.yaml b/deploy/examples/monitoring/exporter-service-monitor.yaml index 10a0b7934f1c..f195f490e13e 100644 --- a/deploy/examples/monitoring/exporter-service-monitor.yaml +++ b/deploy/examples/monitoring/exporter-service-monitor.yaml @@ -12,7 +12,7 @@ spec: selector: matchLabels: app: rook-ceph-exporter - rook_cluster: rook-ceph + rook_cluster: rook-ceph # namespace:cluster ceph_daemon_id: exporter endpoints: - port: ceph-exporter-http-metrics diff --git a/deploy/examples/monitoring/service-monitor.yaml b/deploy/examples/monitoring/service-monitor.yaml index 9bf062687828..7ef491f428ff 100644 --- a/deploy/examples/monitoring/service-monitor.yaml +++ b/deploy/examples/monitoring/service-monitor.yaml @@ -12,7 +12,7 @@ spec: selector: matchLabels: app: rook-ceph-mgr - rook_cluster: rook-ceph + rook_cluster: rook-ceph # namespace:cluster endpoints: - port: http-metrics path: /metrics diff --git a/deploy/examples/nfs-load-balancer.yaml b/deploy/examples/nfs-load-balancer.yaml index 4b7f82afeefd..5fcf73d96030 100644 --- a/deploy/examples/nfs-load-balancer.yaml +++ b/deploy/examples/nfs-load-balancer.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: name: rook-ceph-nfs-my-nfs-load-balancer - namespace: rook-ceph + namespace: rook-ceph # namespace:cluster spec: ports: - name: nfs diff --git a/deploy/examples/osd-env-override.yaml b/deploy/examples/osd-env-override.yaml index 454ccc50a5ed..2eb80a8d0460 100644 --- a/deploy/examples/osd-env-override.yaml +++ b/deploy/examples/osd-env-override.yaml @@ -8,7 +8,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: rook-ceph-osd-env-override - namespace: rook-ceph + namespace: rook-ceph # namespace:cluster data: # Bypass the ASan's assertion that it is the very first loaded DSO. # This is necessary for crimson-osd as it's currently built with diff --git a/deploy/examples/pool-mirrored.yaml b/deploy/examples/pool-mirrored.yaml index 7fe22d1980e5..4b7a16a2a893 100644 --- a/deploy/examples/pool-mirrored.yaml +++ b/deploy/examples/pool-mirrored.yaml @@ -7,7 +7,7 @@ apiVersion: ceph.rook.io/v1 kind: CephBlockPool metadata: name: mirrored-pool - namespace: rook-ceph + namespace: rook-ceph # namespace:cluster spec: replicated: size: 3 diff --git a/deploy/examples/sqlitevfs-client.yaml b/deploy/examples/sqlitevfs-client.yaml index f6fc9674ff48..a821bd2923f1 100644 --- a/deploy/examples/sqlitevfs-client.yaml +++ b/deploy/examples/sqlitevfs-client.yaml @@ -6,7 +6,7 @@ apiVersion: ceph.rook.io/v1 kind: CephClient metadata: name: sqlitevfs - namespace: rook-ceph + namespace: rook-ceph # namespace:cluster spec: caps: mon: 'allow r' diff --git a/deploy/examples/volume-replication-class.yaml b/deploy/examples/volume-replication-class.yaml index 5700285cf2ea..312c0a2eac4d 100644 --- a/deploy/examples/volume-replication-class.yaml +++ b/deploy/examples/volume-replication-class.yaml @@ -9,4 +9,4 @@ spec: schedulingInterval: "12m" schedulingStartTime: "16:18:43" replication.storage.openshift.io/replication-secret-name: rook-csi-rbd-provisioner - replication.storage.openshift.io/replication-secret-namespace: rook-ceph + replication.storage.openshift.io/replication-secret-namespace: rook-ceph # namespace:operator From 745d6dc86fb637cfb49aeb55fa948d61184568b0 Mon Sep 17 00:00:00 2001 From: Redouane Kachach Date: Mon, 23 Oct 2023 15:35:34 +0200 Subject: [PATCH 14/24] docs: adding specific instructions to create operator namespace When a user attempts to adhere to the current documentation to use alternative namespaces for both the operator and the cluster, they fail because our common YAML file only has a single namespace for the cluster. This change adds specific instructions to create operator namespace. Closes: https://github.com/rook/rook/issues/13079 Signed-off-by: Redouane Kachach --- .../Storage-Configuration/Advanced/ceph-configuration.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Documentation/Storage-Configuration/Advanced/ceph-configuration.md b/Documentation/Storage-Configuration/Advanced/ceph-configuration.md index 5b9ff44a459f..334ce92365a1 100644 --- a/Documentation/Storage-Configuration/Advanced/ceph-configuration.md +++ b/Documentation/Storage-Configuration/Advanced/ceph-configuration.md @@ -23,6 +23,14 @@ to also change `ROOK_OPERATOR_NAMESPACE` to create a new Rook Operator for each forget to set `ROOK_CURRENT_NAMESPACE_ONLY`), or you can leave it at the same value for every Ceph cluster if you only wish to have one Operator manage all Ceph clusters. +If the operator namespace is different from the cluster namespace, the operator namespace must be +created before running the steps below. The cluster namespace does not need to be created first, +as it will be created by `common.yaml` in the script below. + +```console +kubectl create namespace $ROOK_OPERATOR_NAMESPACE +``` + This will help you manage namespaces more easily, but you should still make sure the resources are configured to your liking. From 1d830207a2bf57383d202154429d50be3af7d366 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Tue, 24 Oct 2023 09:09:38 -0600 Subject: [PATCH 15/24] docs: update "synk" -> "snyk" spelling Update some old typos that say "synk" instead of "snyk". out of sync. This should fix the top-level README.md to use the latest snyk security scan badge -- as opposed to only referring to the last one that used the misspelled file from about a month ago. Signed-off-by: Blaine Gardner --- Documentation/Contributing/ci-configuration.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/Contributing/ci-configuration.md b/Documentation/Contributing/ci-configuration.md index 5365d77c6e25..bc218c735088 100644 --- a/Documentation/Contributing/ci-configuration.md +++ b/Documentation/Contributing/ci-configuration.md @@ -7,7 +7,7 @@ This page contains information regarding the CI configuration used for the Rook ## Secrets * Snyk (Security Scan): - * `SNYK_TOKEN` - API Token for the [snyk security scanner](https://snyk.io/) (workflow file: `synk.yaml`). + * `SNYK_TOKEN` - API Token for the [snyk security scanner](https://snyk.io/) (workflow file: `snyk.yaml`). * Testing: * `IBM_INSTANCE_ID`: Used for KMS (Key Management System) IBM Key Protect access (see [`.github/workflows/encryption-pvc-kms-ibm-kp/action.yml`](https://github.com/rook/rook/blob/master/.github/workflows/encryption-pvc-kms-ibm-kp/action.yml)). * `IBM_SERVICE_API_KEY`: Used for KMS (Key Management System) IBM Key Protect access (see [`.github/workflows/encryption-pvc-kms-ibm-kp/action.yml`](https://github.com/rook/rook/blob/master/.github/workflows/encryption-pvc-kms-ibm-kp/action.yml)). diff --git a/README.md b/README.md index 9124ee8a7bd5..4c2edd78e4da 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![Docker Pulls](https://img.shields.io/docker/pulls/rook/ceph)](https://hub.docker.com/u/rook) [![Go Report Card](https://goreportcard.com/badge/github.com/rook/rook)](https://goreportcard.com/report/github.com/rook/rook) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1599/badge)](https://bestpractices.coreinfrastructure.org/projects/1599) -[![Security scanning](https://github.com/rook/rook/actions/workflows/synk.yaml/badge.svg)](https://github.com/rook/rook/actions/workflows/synk.yaml) +[![Security scanning](https://github.com/rook/rook/actions/workflows/snyk.yaml/badge.svg)](https://github.com/rook/rook/actions/workflows/snyk.yaml) [![Slack](https://img.shields.io/badge/rook-slack-blue)](https://slack.rook.io) [![Twitter Follow](https://img.shields.io/twitter/follow/rook_io.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=rook_io&user_id=788180534543339520) From bb492e1329caa1dd4403f029306a785a085f4ed3 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Tue, 24 Oct 2023 09:56:11 -0600 Subject: [PATCH 16/24] security: update snyk ignores for MPL-2.0 Update the snyk ignores related to Vault MPL-2.0 licenses. Signed-off-by: Blaine Gardner --- .snyk | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.snyk b/.snyk index c9c5fee1271f..8d22a93f238d 100644 --- a/.snyk +++ b/.snyk @@ -3,13 +3,13 @@ ignore: "snyk:lic:golang:github.com:hashicorp:vault:sdk:MPL-2.0": - "*": reason: Mozilla Public License 2.0 is compatible with Rook's Apache 2.0 license - "snyk:lic:golang:github.com:hashicorp:vault:MPL-2.0": + "snyk:lic:golang:github.com:hashicorp:vault:api:MPL-2.0": - "*": reason: Mozilla Public License 2.0 is compatible with Rook's Apache 2.0 license - "snyk:lic:golang:github.com:hashicorp:vault:api:auth:approle:MPL-2.0": + "snyk:lic:golang:github.com:hashicorp:vault:api:auth:kubernetes:MPL-2.0": - "*": reason: Mozilla Public License 2.0 is compatible with Rook's Apache 2.0 license - "snyk:lic:golang:github.com:hashicorp:vault:api:MPL-2.0": + "snyk:lic:golang:github.com:hashicorp:vault:api:auth:approle:MPL-2.0": - "*": reason: Mozilla Public License 2.0 is compatible with Rook's Apache 2.0 license "snyk:lic:golang:github.com:hashicorp:hcl:MPL-2.0": @@ -45,5 +45,3 @@ ignore: "snyk:lic:golang:github.com:hashicorp:errwrap:MPL-2.0": - "*": reason: Mozilla Public License 2.0 is compatible with Rook's Apache 2.0 license -version: v1.25.0 -patch: {} From f90e069ba93f25946091bd81e3db203ed2bba6b2 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Tue, 24 Oct 2023 11:47:10 -0600 Subject: [PATCH 17/24] test: add govulncheck to look for go vulnerabilities govulncheck is a static code analyzer that scans more deeply than Snyk. Additionally, because govulncheck checks the code itself, it can rule out vulnerabilities that are present in dependencies but that are not used in Rook's code path. Signed-off-by: Blaine Gardner --- .github/workflows/golangci-lint.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml index 45943f80a62d..1e9777e38228 100644 --- a/.github/workflows/golangci-lint.yaml +++ b/.github/workflows/golangci-lint.yaml @@ -44,3 +44,10 @@ jobs: # Optional: show only new issues if it's a pull request. The default value is `false`. # only-new-issues: true + + govulncheck: + name: govulncheck + runs-on: ubuntu-latest + steps: + - name: govulncheck + uses: golang/govulncheck-action@v1 From beef4e2cde40cb9d4eb584e8fc23c2cc7a2b4181 Mon Sep 17 00:00:00 2001 From: StepSecurity Bot Date: Tue, 24 Oct 2023 18:05:30 +0000 Subject: [PATCH 18/24] test: limit permissions in github actions Automatic PR from app.stepsecurity.io (initiated by BlaineEXE) to limit permissions in github actions to only those needed. Signed-off-by: StepSecurity Bot --- .github/workflows/commitlint.yml | 6 ++++++ .github/workflows/docs-check.yml | 3 +++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml index dc1f937348c8..216d12a25fd7 100644 --- a/.github/workflows/commitlint.yml +++ b/.github/workflows/commitlint.yml @@ -16,8 +16,14 @@ concurrency: group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} cancel-in-progress: true +permissions: + contents: read + jobs: lint: + permissions: + contents: read # for actions/checkout to fetch code + pull-requests: read # for wagoid/commitlint-github-action to get commits in PR runs-on: ubuntu-20.04 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml index e0bff0982c00..a99175d28cef 100644 --- a/.github/workflows/docs-check.yml +++ b/.github/workflows/docs-check.yml @@ -16,6 +16,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} cancel-in-progress: true +permissions: + contents: read + jobs: docs-check: name: docs-check From 8854497d00ad31f4e98022295d466282b935515c Mon Sep 17 00:00:00 2001 From: sp98 Date: Wed, 25 Oct 2023 12:02:45 +0530 Subject: [PATCH 19/24] doc: update lvm prerequisites lvm packages should be required if `osdPerDevice` is set to greater than 1. Signed-off-by: sp98 --- Documentation/Getting-Started/Prerequisites/prerequisites.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/Getting-Started/Prerequisites/prerequisites.md b/Documentation/Getting-Started/Prerequisites/prerequisites.md index 6cb7977b455f..d05e31a2eac2 100644 --- a/Documentation/Getting-Started/Prerequisites/prerequisites.md +++ b/Documentation/Getting-Started/Prerequisites/prerequisites.md @@ -52,6 +52,7 @@ Ceph OSDs have a dependency on LVM in the following scenarios: * If encryption is enabled (`encryptedDevice: "true"` in the cluster CR) * A `metadata` device is specified +* `osdsPerDevice` is greater than 1 LVM is not required for OSDs in these scenarios: From 13654d3c3dce5c9153a887a6406725e5e72f1fcc Mon Sep 17 00:00:00 2001 From: Redouane Kachach Date: Tue, 24 Oct 2023 14:43:14 +0200 Subject: [PATCH 20/24] docs: unifying namespaces handling for the second cluster creation Let's use the same technique based on 'sed' while employing two different namespaces for the cluster and the operator when creating the second cluster. Signed-off-by: Redouane Kachach --- .../Advanced/ceph-configuration.md | 15 ++++- deploy/examples/common-second-cluster.yaml | 64 +++++++++---------- 2 files changed, 45 insertions(+), 34 deletions(-) diff --git a/Documentation/Storage-Configuration/Advanced/ceph-configuration.md b/Documentation/Storage-Configuration/Advanced/ceph-configuration.md index 5b9ff44a459f..db1bbad96107 100644 --- a/Documentation/Storage-Configuration/Advanced/ceph-configuration.md +++ b/Documentation/Storage-Configuration/Advanced/ceph-configuration.md @@ -47,12 +47,23 @@ kubectl apply -f common.yaml -f operator.yaml -f cluster.yaml # add other files ## Deploying a second cluster -If you wish to create a new CephCluster in a different namespace than `rook-ceph` while using a single operator to manage both clusters execute the following: +If you wish to create a new CephCluster in a separate namespace, you can easily do so +by modifying the `ROOK_OPERATOR_NAMESPACE` and `SECOND_ROOK_CLUSTER_NAMESPACE` values in the +below instructions. The default configuration in `common-second-cluster.yaml` is already +set up to utilize `rook-ceph` for the operator and `rook-ceph-secondary` for the cluster. +There's no need to run the `sed` command if you prefer to use these default values. ```console cd deploy/examples +export ROOK_OPERATOR_NAMESPACE="rook-ceph" +export SECOND_ROOK_CLUSTER_NAMESPACE="rook-ceph-secondary" + +sed -i.bak \ + -e "s/\(.*\):.*# namespace:operator/\1: $ROOK_OPERATOR_NAMESPACE # namespace:operator/g" \ + -e "s/\(.*\):.*# namespace:cluster/\1: $SECOND_ROOK_CLUSTER_NAMESPACE # namespace:cluster/g" \ + common-second-cluster.yaml -NAMESPACE=rook-ceph-secondary envsubst < common-second-cluster.yaml | kubectl create -f - +kubectl create -f common-second-cluster.yaml ``` This will create all the necessary RBACs as well as the new namespace. The script assumes that `common.yaml` was already created. diff --git a/deploy/examples/common-second-cluster.yaml b/deploy/examples/common-second-cluster.yaml index 684868e6c625..0000a1ffdb0f 100644 --- a/deploy/examples/common-second-cluster.yaml +++ b/deploy/examples/common-second-cluster.yaml @@ -7,13 +7,13 @@ apiVersion: v1 kind: Namespace metadata: - name: $NAMESPACE + name: rook-ceph-secondary # namespace:cluster --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-cluster-mgmt - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -21,13 +21,13 @@ roleRef: subjects: - kind: ServiceAccount name: rook-ceph-system - namespace: rook-ceph + namespace: rook-ceph # namespace:operator --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-cmd-reporter - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -35,13 +35,13 @@ roleRef: subjects: - kind: ServiceAccount name: rook-ceph-cmd-reporter - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-cmd-reporter-psp - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -49,13 +49,13 @@ roleRef: subjects: - kind: ServiceAccount name: rook-ceph-cmd-reporter - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-mgr-system - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -63,13 +63,13 @@ roleRef: subjects: - kind: ServiceAccount name: rook-ceph-mgr - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-mgr-psp - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -77,13 +77,13 @@ roleRef: subjects: - kind: ServiceAccount name: rook-ceph-mgr - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-osd-psp - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -91,13 +91,13 @@ roleRef: subjects: - kind: ServiceAccount name: rook-ceph-osd - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-purge-osd-psp - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -105,13 +105,13 @@ roleRef: subjects: - kind: ServiceAccount name: rook-ceph-purge-osd - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-rgw-psp - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -119,13 +119,13 @@ roleRef: subjects: - kind: ServiceAccount name: rook-ceph-rgw - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-default-psp - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -133,19 +133,19 @@ roleRef: subjects: - kind: ServiceAccount name: default - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster --- apiVersion: v1 kind: ServiceAccount metadata: name: rook-ceph-cmd-reporter - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-cmd-reporter - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster rules: - apiGroups: - "" @@ -164,7 +164,7 @@ kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-osd - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster rules: # this is needed for rook's "key-management" CLI to fetch the vault token from the secret when # validating the connection details and for key rotation operations. @@ -202,13 +202,13 @@ roleRef: subjects: - kind: ServiceAccount name: rook-ceph-osd - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-osd-external - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -216,26 +216,26 @@ roleRef: subjects: - kind: ServiceAccount name: rook-ceph-osd - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster --- apiVersion: v1 kind: ServiceAccount metadata: name: rook-ceph-mgr - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster --- apiVersion: v1 kind: ServiceAccount metadata: name: rook-ceph-osd - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster --- # Aspects of ceph osd purge job that require access to the operator/cluster namespace kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-purge-osd - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster rules: - apiGroups: [""] resources: ["configmaps"] @@ -255,7 +255,7 @@ kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-purge-osd - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -263,16 +263,16 @@ roleRef: subjects: - kind: ServiceAccount name: rook-ceph-purge-osd - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster --- apiVersion: v1 kind: ServiceAccount metadata: name: rook-ceph-purge-osd - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster --- apiVersion: v1 kind: ServiceAccount metadata: name: rook-ceph-rgw - namespace: $NAMESPACE + namespace: rook-ceph-secondary # namespace:cluster From c4cf300358081f1014d12e4f0b0737e856b7bbdb Mon Sep 17 00:00:00 2001 From: subhamkrai Date: Wed, 25 Oct 2023 20:30:24 +0530 Subject: [PATCH 21/24] ci: skip csv generation for mac mac build ci is failing during csv gen when trying to move files to some folder. Skipping csv gen for mac since it is not required. Signed-off-by: subhamkrai --- build/csv/csv-gen.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/build/csv/csv-gen.sh b/build/csv/csv-gen.sh index a75c67ea2a53..e55d3f448563 100755 --- a/build/csv/csv-gen.sh +++ b/build/csv/csv-gen.sh @@ -43,12 +43,14 @@ function generate_csv() { # This change are just to make the CSV file as it was earlier and as ocs-operator reads. # Skipping this change for darwin since `sed -i` doesn't work with darwin properly. # and the csv is not ever needed in the mac builds. - if [[ "$OSTYPE" != "darwin"* ]]; then - sed -i 's/image: rook\/ceph:.*/image: {{.RookOperatorImage}}/g' "$CSV_FILE_NAME" - sed -i 's/name: rook-ceph.v.*/name: rook-ceph.v{{.RookOperatorCsvVersion}}/g' "$CSV_FILE_NAME" - sed -i 's/version: 0.0.0/version: {{.RookOperatorCsvVersion}}/g' "$CSV_FILE_NAME" + if [[ "$OSTYPE" == "darwin"* ]]; then + return fi + sed -i 's/image: rook\/ceph:.*/image: {{.RookOperatorImage}}/g' "$CSV_FILE_NAME" + sed -i 's/name: rook-ceph.v.*/name: rook-ceph.v{{.RookOperatorCsvVersion}}/g' "$CSV_FILE_NAME" + sed -i 's/version: 0.0.0/version: {{.RookOperatorCsvVersion}}/g' "$CSV_FILE_NAME" + mv "$CSV_FILE_NAME" "../../build/csv/" mv "../../build/csv/ceph/$PLATFORM/manifests/"* "../../build/csv/ceph/" rm -rf "../../build/csv/ceph/$PLATFORM" From bab6e01d3729139cdb399100e3e783e882ec27d6 Mon Sep 17 00:00:00 2001 From: Satoru Takeuchi Date: Tue, 2 May 2023 13:08:36 +0000 Subject: [PATCH 22/24] doc: add the table to know supported osd configuration It's convenient to visualize the supported OSD configurations. In addition, remove the reference to the common issue section about OSD on LV-backed PVC because this issue doesn't happen anymore for new OSDs. Closes: https://github.com/rook/rook/issues/10859 Signed-off-by: Satoru Takeuchi --- .../CRDs/Cluster/ceph-cluster-crd.md | 20 ++++++++++++++++--- Documentation/Getting-Started/quickstart.md | 4 +++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/Documentation/CRDs/Cluster/ceph-cluster-crd.md b/Documentation/CRDs/Cluster/ceph-cluster-crd.md index 5b93adf00f38..db26fa9e02a4 100755 --- a/Documentation/CRDs/Cluster/ceph-cluster-crd.md +++ b/Documentation/CRDs/Cluster/ceph-cluster-crd.md @@ -101,7 +101,7 @@ Official releases of Ceph Container images are available from [Docker Hub](https These are general purpose Ceph container with all necessary daemons and dependencies installed. | TAG | MEANING | -| -------------------- | --------------------------------------------------------- | +|----------------------|-----------------------------------------------------------| | vRELNUM | Latest release in this series (e.g., *v17* = Quincy) | | vRELNUM.Y | Latest stable release in this stable series (e.g., v17.2) | | vRELNUM.Y.Z | A specific release (e.g., v17.2.6) | @@ -421,7 +421,7 @@ Below are the settings for host-based cluster. This type of cluster can specify * `name`: The name of the devices and partitions (e.g., `sda`). The full udev path can also be specified for devices, partitions, and logical volumes (e.g. `/dev/disk/by-id/ata-ST4000DM004-XXXX` - this will not change after reboots). * `config`: Device-specific config settings. See the [config settings](#osd-configuration-settings) below -Host-based cluster supports raw device, partition, and logical volume. Be sure to see the +Host-based cluster supports raw devices, partitions, logical volumes, encrypted devices, and multipath devices. Be sure to see the [quickstart doc prerequisites](../../Getting-Started/quickstart.md#prerequisites) for additional considerations. Below are the settings for a PVC-based cluster. @@ -456,13 +456,17 @@ The following are the settings for Storage Class Device Sets which can be config * `tuneDeviceClass`: For example, Ceph cannot detect AWS volumes as HDDs from the storage class "gp2", so you can improve Ceph performance by setting this to true. * `tuneFastDeviceClass`: For example, Ceph cannot detect Azure disks as SSDs from the storage class "managed-premium", so you can improve Ceph performance by setting this to true.. * `volumeClaimTemplates`: A list of PVC templates to use for provisioning the underlying storage devices. + * `metadata.name`: "data", "metadata", or "wal". If a single template is provided, the name must be "data". If the name is "metadata" or "wal", the devices are used to store the Ceph metadata or WAL respectively. In both cases, the devices must be raw devices or LVM logical volumes. + * `resources.requests.storage`: The desired capacity for the underlying storage devices. - * `storageClassName`: The StorageClass to provision PVCs from. Default would be to use the cluster-default StorageClass. This StorageClass should provide a raw block device, multipath device, or logical volume. Other types are not supported. If you want to use logical volume, please see [known issue of OSD on LV-backed PVC](../../Troubleshooting/ceph-common-issues.md#lvm-metadata-can-be-corrupted-with-osd-on-lv-backed-pvc) + * `storageClassName`: The StorageClass to provision PVCs from. Default would be to use the cluster-default StorageClass. * `volumeMode`: The volume mode to be set for the PVC. Which should be Block * `accessModes`: The access mode for the PVC to be bound by OSD. * `schedulerName`: Scheduler name for OSD pod placement. (Optional) * `encrypted`: whether to encrypt all the OSDs in a given storageClassDeviceSet +See the table in [OSD Configuration Settings](#osd-configuration-settings) to know the allowed configurations. + ### OSD Configuration Settings The following storage selection settings are specific to Ceph and do not apply to other backends. All variables are key-value pairs represented as strings. @@ -477,6 +481,16 @@ The following storage selection settings are specific to Ceph and do not apply t * `encryptedDevice`**: Encrypt OSD volumes using dmcrypt ("true" or "false"). By default this option is disabled. See [encryption](http://docs.ceph.com/docs/master/ceph-volume/lvm/encryption/) for more information on encryption in Ceph. * `crushRoot`: The value of the `root` CRUSH map label. The default is `default`. Generally, you should not need to change this. However, if any of your topology labels may have the value `default`, you need to change `crushRoot` to avoid conflicts, since CRUSH map values need to be unique. +Allowed configurations are: + +| block device type | host-based cluster | PVC-based cluster | +|:------------------|:------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------| +| disk | | | +| part | `encryptedDevice` should be "false" | `encrypted` must be `false` | +| lvm | `metadataDevice` should be "", `osdsPerDevice` should be "1", and `encryptedDevice` should be "false" | `metadata.name` must not be `metadata` or `wal` and `encrypted` must be `false` | +| crypt | | | +| mpath | | | + ### Annotations and Labels Annotations and Labels can be specified so that the Rook components will have those annotations / labels added to them. diff --git a/Documentation/Getting-Started/quickstart.md b/Documentation/Getting-Started/quickstart.md index 6a058fb53e7e..e598c41e6451 100644 --- a/Documentation/Getting-Started/quickstart.md +++ b/Documentation/Getting-Started/quickstart.md @@ -24,9 +24,11 @@ To check if a Kubernetes cluster is ready for `Rook`, see the [prerequisites](Pr To configure the Ceph storage cluster, at least one of these local storage options are required: -* Raw devices (no partitions or formatted filesystems) +* Raw devices (no partitions or formatted filesystem) * Raw partitions (no formatted filesystem) * LVM Logical Volumes (no formatted filesystem) +* Encrypted devices (no formatted filesystem) +* Multipath devices (no formatted filesystem) * Persistent Volumes available from a storage class in `block` mode ## TL;DR From d6aae628791043b1525d927d2b7c23556b877345 Mon Sep 17 00:00:00 2001 From: Redouane Kachach Date: Thu, 26 Oct 2023 15:47:14 +0200 Subject: [PATCH 23/24] manifest: fix rook-ceph-mgr-system role and namespace Signed-off-by: Redouane Kachach --- deploy/examples/common-second-cluster.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/examples/common-second-cluster.yaml b/deploy/examples/common-second-cluster.yaml index 0000a1ffdb0f..a7b7ff72d01a 100644 --- a/deploy/examples/common-second-cluster.yaml +++ b/deploy/examples/common-second-cluster.yaml @@ -54,8 +54,8 @@ subjects: kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: rook-ceph-mgr-system - namespace: rook-ceph-secondary # namespace:cluster + name: rook-ceph-mgr-system-secondary + namespace: rook-ceph # namespace:operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole From ef4dd76df1b8aa64415cef7c45d3d6aaaf8a38a1 Mon Sep 17 00:00:00 2001 From: subhamkrai Date: Fri, 27 Oct 2023 14:12:10 +0530 Subject: [PATCH 24/24] pool: rbd cmd shouldn't use admin in external mode when creating networkFence, rbd command was loading admin config and hence running rbd command use client.admin in case of external cluster also. With this commit instead of client.admin user it will use what is being passed to config. Signed-off-by: subhamkrai --- pkg/operator/ceph/cluster/watcher.go | 5 ++++- pkg/operator/ceph/cluster/watcher_test.go | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/pkg/operator/ceph/cluster/watcher.go b/pkg/operator/ceph/cluster/watcher.go index 599cfb840493..da55e20ce641 100644 --- a/pkg/operator/ceph/cluster/watcher.go +++ b/pkg/operator/ceph/cluster/watcher.go @@ -219,7 +219,10 @@ func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluste return nil } - clusterInfo := cephclient.AdminClusterInfo(ctx, cluster.Namespace, cluster.Name) + clusterInfo, _, _, err := opcontroller.LoadClusterInfo(c.context, ctx, cluster.Namespace, &cluster.Spec) + if err != nil { + return pkgerror.Wrapf(err, "Failed to load cluster info.") + } for i := range rbdPVList { err = c.fenceRbdImage(ctx, node, cluster, clusterInfo, rbdPVList[i]) diff --git a/pkg/operator/ceph/cluster/watcher_test.go b/pkg/operator/ceph/cluster/watcher_test.go index 2a28d755f1ea..19d659437ac1 100644 --- a/pkg/operator/ceph/cluster/watcher_test.go +++ b/pkg/operator/ceph/cluster/watcher_test.go @@ -271,7 +271,24 @@ func TestHandleNodeFailure(t *testing.T) { }, } - _, err := c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}) + // Mock clusterInfo + secrets := map[string][]byte{ + "fsid": []byte("c47cac40-9bee-4d52-823b-ccd803ba5bfe"), + "mon-secret": []byte("monsecret"), + "admin-secret": []byte("adminsecret"), + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-ceph-mon", + Namespace: ns, + }, + Data: secrets, + Type: k8sutil.RookType, + } + _, err := c.context.Clientset.CoreV1().Secrets(ns).Create(ctx, secret, metav1.CreateOptions{}) + assert.NoError(t, err) + + _, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}) assert.NoError(t, err) _, err = c.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(ctx, &v1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: "networkfences.csiaddons.openshift.io"}}, metav1.CreateOptions{})