Merge pull request #689 from red-hat-storage/sync_us--master

Syncing latest changes from upstream master for rook

build/csv/ceph/ceph.rook.io_cephclusters.yaml

@@ -449,6 +449,7 @@ spec:
                               - ""
                               - crush-compat
                               - upmap
+                              - read
                               - upmap-read
                               type: string
                           type: object

deploy/charts/rook-ceph/templates/resources.yaml

@@ -1601,6 +1601,7 @@ spec:
                                   - ""
                                   - crush-compat
                                   - upmap
+                                  - read
                                   - upmap-read
                                 type: string
                             type: object

deploy/examples/crds.yaml

@@ -1599,6 +1599,7 @@ spec:
                                   - ""
                                   - crush-compat
                                   - upmap
+                                  - read
                                   - upmap-read
                                 type: string
                             type: object

go.mod

@@ -16,7 +16,7 @@ replace (
 
 require (
 	github.com/IBM/keyprotect-go-client v0.14.3
-	github.com/aws/aws-sdk-go v1.54.20
+	github.com/aws/aws-sdk-go v1.55.3
 	github.com/banzaicloud/k8s-objectmatcher v1.8.0
 	github.com/ceph/go-ceph v0.28.0
 	github.com/coreos/pkg v0.0.0-20230601102743-20bbbf26f4d8
@@ -30,8 +30,8 @@ require (
 	github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1
 	github.com/libopenstorage/secrets v0.0.0-20240416031220-a17cf7f72c6c
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.75.1
-	github.com/prometheus-operator/prometheus-operator/pkg/client v0.75.1
+	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.75.2
+	github.com/prometheus-operator/prometheus-operator/pkg/client v0.75.2
 	github.com/rook/rook/pkg/apis v0.0.0-20231204200402-5287527732f7
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -144,8 +144,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.44.164/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/aws/aws-sdk-go v1.54.20 h1:FZ2UcXya7bUkvkpf7TaPmiL7EubK0go1nlXGLRwEsoo=
-github.com/aws/aws-sdk-go v1.54.20/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
+github.com/aws/aws-sdk-go v1.55.3 h1:0B5hOX+mIx7I5XPOrjrHlKSDQV/+ypFZpIHOx5LOk3E=
+github.com/aws/aws-sdk-go v1.55.3/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
 github.com/banzaicloud/k8s-objectmatcher v1.8.0 h1:Nugn25elKtPMTA2br+JgHNeSQ04sc05MDPmpJnd1N2A=
 github.com/banzaicloud/k8s-objectmatcher v1.8.0/go.mod h1:p2LSNAjlECf07fbhDyebTkPUIYnU05G+WfGgkTmgeMg=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -770,11 +770,11 @@ github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4
 github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.44.1/go.mod h1:3WYi4xqXxGGXWDdQIITnLNmuDzO5n6wYva9spVhR4fg=
 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.46.0/go.mod h1:3WYi4xqXxGGXWDdQIITnLNmuDzO5n6wYva9spVhR4fg=
-github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.75.1 h1:+iiljhJV6niK7MuifJs/n3NeLxikd85nrQfn53sLJkU=
-github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.75.1/go.mod h1:XYrdZw5dW12Cjkt4ndbeNZZTBp4UCHtW0ccR9+sTtPU=
+github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.75.2 h1:6UsAv+jAevuGO2yZFU/BukV4o9NKnFMOuoouSA4G0ns=
+github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.75.2/go.mod h1:XYrdZw5dW12Cjkt4ndbeNZZTBp4UCHtW0ccR9+sTtPU=
 github.com/prometheus-operator/prometheus-operator/pkg/client v0.46.0/go.mod h1:k4BrWlVQQsvBiTcDnKEMgyh/euRxyxgrHdur/ZX/sdA=
-github.com/prometheus-operator/prometheus-operator/pkg/client v0.75.1 h1:s7GlsRYGLWP+L1eQKy6RmLatX+k3v9NQwutUix4l5uM=
-github.com/prometheus-operator/prometheus-operator/pkg/client v0.75.1/go.mod h1:qca3qWGdknRpHvPyThepe5a6QYAh38IQ2ml93E6V3NY=
+github.com/prometheus-operator/prometheus-operator/pkg/client v0.75.2 h1:71GOmhZFA2/17maXqCcuJEzpJDyqPty8SpEOGZWyVec=
+github.com/prometheus-operator/prometheus-operator/pkg/client v0.75.2/go.mod h1:Sv6XsfGGkR9gKnhP92F5dNXEpsSePn0W+7JwYP0NVkc=
 github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=

pkg/apis/ceph.rook.io/v1/types.go

@@ -679,7 +679,7 @@ type Module struct {
 
 type ModuleSettings struct {
 	// BalancerMode sets the `balancer` module with different modes like `upmap`, `crush-compact` etc
-	// +kubebuilder:validation:Enum="";crush-compat;upmap;upmap-read
+	// +kubebuilder:validation:Enum="";crush-compat;upmap;read;upmap-read
 	BalancerMode string `json:"balancerMode,omitempty"`
 }
 

pkg/daemon/ceph/client/mgr.go

@@ -22,12 +22,18 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rook/rook/pkg/clusterd"
+	cephver "github.com/rook/rook/pkg/operator/ceph/version"
 )
 
 var (
 	moduleEnableWaitTime = 5 * time.Second
 )
 
+const (
+	readBalancerMode      = "read"
+	upmapReadBalancerMode = "upmap-read"
+)
+
 func CephMgrMap(context *clusterd.Context, clusterInfo *ClusterInfo) (*MgrMap, error) {
 	args := []string{"mgr", "dump"}
 	buf, err := NewCephCommand(context, clusterInfo, args).Run()
@@ -132,12 +138,12 @@ func setBalancerMode(context *clusterd.Context, clusterInfo *ClusterInfo, mode s
 	return nil
 }
 
-// setMinCompatClientLuminous set the minimum compatibility for clients to Luminous
-func setMinCompatClientLuminous(context *clusterd.Context, clusterInfo *ClusterInfo) error {
-	args := []string{"osd", "set-require-min-compat-client", "luminous", "--yes-i-really-mean-it"}
+// setMinCompatClient set the minimum compatibility for clients
+func setMinCompatClient(context *clusterd.Context, clusterInfo *ClusterInfo, version string) error {
+	args := []string{"osd", "set-require-min-compat-client", version, "--yes-i-really-mean-it"}
 	_, err := NewCephCommand(context, clusterInfo, args).Run()
 	if err != nil {
-		return errors.Wrap(err, "failed to set set-require-min-compat-client to luminous")
+		return errors.Wrapf(err, "failed to set set-require-min-compat-client to %q", version)
 	}
 
 	return nil
@@ -165,8 +171,12 @@ func mgrSetBalancerMode(context *clusterd.Context, clusterInfo *ClusterInfo, bal
 
 // ConfigureBalancerModule configures the balancer module
 func ConfigureBalancerModule(context *clusterd.Context, clusterInfo *ClusterInfo, balancerModuleMode string) error {
-	// Set min compat client to luminous before enabling the balancer mode "upmap"
-	err := setMinCompatClientLuminous(context, clusterInfo)
+	minCompatClientVersion, err := desiredMinCompatClientVersion(clusterInfo, balancerModuleMode)
+	if err != nil {
+		return errors.Wrap(err, "failed to get minimum compatibility client version")
+	}
+
+	err = setMinCompatClient(context, clusterInfo, minCompatClientVersion)
 	if err != nil {
 		return errors.Wrap(err, "failed to set minimum compatibility client")
 	}
@@ -179,3 +189,17 @@ func ConfigureBalancerModule(context *clusterd.Context, clusterInfo *ClusterInfo
 
 	return nil
 }
+
+func desiredMinCompatClientVersion(clusterInfo *ClusterInfo, balancerModuleMode string) (string, error) {
+	// Set min compat client to luminous before enabling the balancer mode "upmap"
+	minCompatClientVersion := "luminous"
+	if balancerModuleMode == readBalancerMode || balancerModuleMode == upmapReadBalancerMode {
+		if !clusterInfo.CephVersion.IsAtLeast(cephver.CephVersion{Major: 19}) {
+			return "", errors.New("minimum ceph v19 (Squid) is required for upmap-read or read balancer modes")
+		}
+		// Set min compat client to reef before enabling the balancer mode "upmap-read" or "read"
+		minCompatClientVersion = "reef"
+	}
+
+	return minCompatClientVersion, nil
+}

pkg/daemon/ceph/client/mgr_test.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rook/rook/pkg/clusterd"
+	cephver "github.com/rook/rook/pkg/operator/ceph/version"
 	exectest "github.com/rook/rook/pkg/util/exec/test"
 	"github.com/stretchr/testify/assert"
 )
@@ -135,3 +136,37 @@ func TestSetBalancerMode(t *testing.T) {
 	err := setBalancerMode(&clusterd.Context{Executor: executor}, AdminTestClusterInfo("mycluster"), "upmap")
 	assert.NoError(t, err)
 }
+
+func TestGetMinCompatClientVersion(t *testing.T) {
+	clusterInfo := AdminTestClusterInfo("mycluster")
+	t.Run("upmap-read balancer mode with ceph v19", func(t *testing.T) {
+		clusterInfo.CephVersion = cephver.CephVersion{Major: 19}
+		result, err := desiredMinCompatClientVersion(clusterInfo, upmapReadBalancerMode)
+		assert.NoError(t, err)
+		assert.Equal(t, "reef", result)
+	})
+
+	t.Run("read balancer mode with ceph v19", func(t *testing.T) {
+		clusterInfo.CephVersion = cephver.CephVersion{Major: 19}
+		result, err := desiredMinCompatClientVersion(clusterInfo, readBalancerMode)
+		assert.NoError(t, err)
+		assert.Equal(t, "reef", result)
+	})
+	t.Run("upmap-read balancer mode with ceph below v19 should fail", func(t *testing.T) {
+		clusterInfo.CephVersion = cephver.CephVersion{Major: 18}
+		_, err := desiredMinCompatClientVersion(clusterInfo, upmapReadBalancerMode)
+		assert.Error(t, err)
+	})
+	t.Run("read balancer mode with ceph below v19 should fail", func(t *testing.T) {
+		clusterInfo.CephVersion = cephver.CephVersion{Major: 18}
+		_, err := desiredMinCompatClientVersion(clusterInfo, readBalancerMode)
+		assert.Error(t, err)
+	})
+
+	t.Run("upmap balancer set min compat client to luminous", func(t *testing.T) {
+		clusterInfo.CephVersion = cephver.CephVersion{Major: 19}
+		result, err := desiredMinCompatClientVersion(clusterInfo, "upmap")
+		assert.NoError(t, err)
+		assert.Equal(t, "luminous", result)
+	})
+}