Syncing latest changes from upstream master for rook

.github/workflows/daily-nightly-jobs.yml

@@ -195,6 +195,46 @@ jobs:
           name: ceph-smoke-suite-reef-artifact
           path: /home/runner/work/rook/rook/tests/integration/_output/tests/
 
+  smoke-suite-squid-devel:
+    if: github.repository == 'rook/rook'
+    runs-on: ubuntu-22.04
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
+      - name: setup cluster resources
+        uses: ./.github/workflows/integration-test-config-latest-k8s
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          kubernetes-version: "1.28.4"
+
+      - name: TestCephSmokeSuite
+        run: |
+          export DEVICE_FILTER=$(tests/scripts/github-action-helper.sh find_extra_block_dev)
+          SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION="squid-devel" go test -v -timeout 1800s -run TestCephSmokeSuite github.com/rook/rook/tests/integration
+
+      - name: collect common logs
+        if: always()
+        run: |
+          export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/"
+          export CLUSTER_NAMESPACE="smoke-ns"
+          export OPERATOR_NAMESPACE="smoke-ns-system"
+          tests/scripts/collect-logs.sh
+
+      - name: Artifact
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: ceph-smoke-suite-squid-artifact
+          path: /home/runner/work/rook/rook/tests/integration/_output/tests/
+
   smoke-suite-ceph-main:
     if: github.repository == 'rook/rook'
     runs-on: ubuntu-22.04
@@ -315,6 +355,47 @@ jobs:
           name: ceph-object-suite-master-artifact
           path: /home/runner/work/rook/rook/tests/integration/_output/tests/
 
+  upgrade-from-squid-stable-to-squid-devel:
+    if: github.repository == 'rook/rook'
+    runs-on: ubuntu-22.04
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
+      - name: setup cluster resources
+        uses: ./.github/workflows/integration-test-config-latest-k8s
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          kubernetes-version: "1.28.4"
+
+      - name: TestCephUpgradeSuite
+        run: |
+          export DEVICE_FILTER=$(tests/scripts/github-action-helper.sh find_extra_block_dev)
+          go test -v -timeout 1800s -failfast -run TestCephUpgradeSuite/TestUpgradeCephToSquidDevel github.com/rook/rook/tests/integration
+
+      - name: collect common logs
+        if: always()
+        run: |
+          export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/"
+          export CLUSTER_NAMESPACE="upgrade"
+          export OPERATOR_NAMESPACE="upgrade-system"
+          tests/scripts/collect-logs.sh
+
+      - name: Artifact
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: ceph-upgrade-suite-squid-artifact
+          path: /home/runner/work/rook/rook/tests/integration/_output/tests/
+
+
   upgrade-from-reef-stable-to-reef-devel:
     if: github.repository == 'rook/rook'
     runs-on: ubuntu-22.04
@@ -399,5 +480,5 @@ jobs:
     if: github.repository == 'rook/rook'
     uses: ./.github/workflows/canary-integration-test.yml
     with:
-      ceph_images: '["quay.io/ceph/ceph:v18", "quay.io/ceph/daemon-base:latest-main-devel", "quay.io/ceph/daemon-base:latest-quincy-devel", "quay.io/ceph/daemon-base:latest-reef-devel"]'
+      ceph_images: '["quay.io/ceph/ceph:v18", "quay.io/ceph/daemon-base:latest-main-devel", "quay.io/ceph/daemon-base:latest-quincy-devel", "quay.io/ceph/daemon-base:latest-reef-devel", "quay.io/ceph/daemon-base:latest-squid-devel"]'
     secrets: inherit

PendingReleaseNotes.md

@@ -7,3 +7,5 @@ please update the `BucketClass` and `BucketAccessClass` for resolving refer [her
 - During CephBlockPool updates, return an error if an invalid device class is specified. Pools with invalid device classes may start failing reconcile until the correct device class is specified. See #14057.
 
 ## Features
+
+- Added support for Ceph Squid (v19)

deploy/examples/cluster-test.yaml

@@ -15,7 +15,7 @@ metadata:
 spec:
   dataDirHostPath: /var/lib/rook
   cephVersion:
-    image: quay.io/ceph/ceph:v18
+    image: quay.io/ceph/ceph:v19
     allowUnsupported: true
   mon:
     count: 1

go.mod

@@ -16,7 +16,7 @@ replace (
 
 require (
 	github.com/IBM/keyprotect-go-client v0.14.3
-	github.com/aws/aws-sdk-go v1.54.15
+	github.com/aws/aws-sdk-go v1.54.19
 	github.com/banzaicloud/k8s-objectmatcher v1.8.0
 	github.com/ceph/go-ceph v0.28.0
 	github.com/coreos/pkg v0.0.0-20230601102743-20bbbf26f4d8

go.sum

@@ -144,8 +144,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.44.164/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/aws/aws-sdk-go v1.54.15 h1:ErgCEVbzuSfuZl9nR+g8FFnzjgeJ/AqAGOEWn6tgAHo=
-github.com/aws/aws-sdk-go v1.54.15/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
+github.com/aws/aws-sdk-go v1.54.19 h1:tyWV+07jagrNiCcGRzRhdtVjQs7Vy41NwsuOcl0IbVI=
+github.com/aws/aws-sdk-go v1.54.19/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
 github.com/banzaicloud/k8s-objectmatcher v1.8.0 h1:Nugn25elKtPMTA2br+JgHNeSQ04sc05MDPmpJnd1N2A=
 github.com/banzaicloud/k8s-objectmatcher v1.8.0/go.mod h1:p2LSNAjlECf07fbhDyebTkPUIYnU05G+WfGgkTmgeMg=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=

pkg/operator/ceph/cluster/version_test.go

@@ -165,6 +165,8 @@ func TestMinVersion(t *testing.T) {
 	assert.NoError(t, c.validateCephVersion(v))
 	v = &cephver.CephVersion{Major: 18}
 	assert.NoError(t, c.validateCephVersion(v))
+	v = &cephver.CephVersion{Major: 19}
+	assert.NoError(t, c.validateCephVersion(v))
 }
 
 func TestSupportedVersion(t *testing.T) {
@@ -183,8 +185,12 @@ func TestSupportedVersion(t *testing.T) {
 	v = &cephver.CephVersion{Major: 18, Minor: 2, Extra: 0}
 	assert.NoError(t, c.validateCephVersion(v))
 
-	// Squid is not supported
-	v = &cephver.CephVersion{Major: 19, Minor: 1, Extra: 0}
+	// Squid is supported
+	v = &cephver.CephVersion{Major: 19, Minor: 2, Extra: 0}
+	assert.NoError(t, c.validateCephVersion(v))
+
+	// Tentacle release is not supported
+	v = &cephver.CephVersion{Major: 20, Minor: 1, Extra: 0}
 	assert.Error(t, c.validateCephVersion(v))
 
 	// Unsupported versions are now valid

pkg/operator/ceph/version/version.go

@@ -48,9 +48,11 @@ var (
 	Reef = CephVersion{18, 0, 0, 0, ""}
 	// Squid ceph version
 	Squid = CephVersion{19, 0, 0, 0, ""}
+	// Tentacle ceph version
+	Tentacle = CephVersion{20, 0, 0, 0, ""}
 
 	// supportedVersions are production-ready versions that rook supports
-	supportedVersions = []CephVersion{Quincy, Reef}
+	supportedVersions = []CephVersion{Quincy, Reef, Squid}
 
 	// unsupportedVersions are possibly Ceph pin-point release that introduced breaking changes and not recommended
 	unsupportedVersions []CephVersion
@@ -90,6 +92,8 @@ func (v *CephVersion) ReleaseName() string {
 		return "reef"
 	case Squid.Major:
 		return "squid"
+	case Tentacle.Major:
+		return "tentacle"
 	default:
 		return unknownVersionString
 	}
@@ -180,6 +184,11 @@ func (v *CephVersion) IsSquid() bool {
 	return v.isRelease(Squid)
 }
 
+// IsTentacle checks if the Ceph version is Tentacle
+func (v *CephVersion) IsTentacle() bool {
+	return v.isRelease(Tentacle)
+}
+
 // IsAtLeast checks a given Ceph version is at least a given one
 func (v *CephVersion) IsAtLeast(other CephVersion) bool {
 	if v.Major > other.Major {
@@ -203,12 +212,22 @@ func (v *CephVersion) IsAtLeast(other CephVersion) bool {
 	return true
 }
 
-// IsAtLeastReef check that the Ceph version is at least Reef
+// IsAtLeastTentacle checks that the Ceph version is at least Tentacle
+func (v *CephVersion) IsAtLeastTentacle() bool {
+	return v.IsAtLeast(Tentacle)
+}
+
+// IsAtLeastSquid checks that the Ceph version is at least Squid
+func (v *CephVersion) IsAtLeastSquid() bool {
+	return v.IsAtLeast(Squid)
+}
+
+// IsAtLeastReef checks that the Ceph version is at least Reef
 func (v *CephVersion) IsAtLeastReef() bool {
 	return v.IsAtLeast(Reef)
 }
 
-// IsAtLeastQuincy check that the Ceph version is at least Quincy
+// IsAtLeastQuincy checks that the Ceph version is at least Quincy
 func (v *CephVersion) IsAtLeastQuincy() bool {
 	return v.IsAtLeast(Quincy)
 }

pkg/operator/ceph/version/version_test.go

@@ -101,7 +101,7 @@ func TestSupported(t *testing.T) {
 	for _, v := range supportedVersions {
 		assert.True(t, v.Supported())
 	}
-	assert.False(t, Squid.Supported())
+	assert.False(t, Tentacle.Supported())
 }
 
 func TestIsRelease(t *testing.T) {

tests/framework/installer/ceph_installer.go

@@ -45,9 +45,11 @@ const (
 	// test with the latest releases
 	quincyTestImage = "quay.io/ceph/ceph:v17"
 	reefTestImage   = "quay.io/ceph/ceph:v18"
+	squidTestImage  = "quay.io/ceph/ceph:v19"
 	// test with the current development versions
 	quincyDevelTestImage = "quay.io/ceph/daemon-base:latest-quincy-devel"
 	reefDevelTestImage   = "quay.io/ceph/daemon-base:latest-reef-devel"
+	squidDevelTestImage  = "quay.io/ceph/daemon-base:latest-squid-devel"
 	// test with the latest Ceph main image
 	mainTestImage      = "quay.io/ceph/daemon-base:latest-main-devel"
 	cephOperatorLabel  = "app=rook-ceph-operator"
@@ -71,6 +73,8 @@ var (
 	QuincyDevelVersion           = cephv1.CephVersionSpec{Image: quincyDevelTestImage}
 	ReefVersion                  = cephv1.CephVersionSpec{Image: reefTestImage}
 	ReefDevelVersion             = cephv1.CephVersionSpec{Image: reefDevelTestImage}
+	SquidVersion                 = cephv1.CephVersionSpec{Image: squidTestImage}
+	SquidDevelVersion            = cephv1.CephVersionSpec{Image: squidDevelTestImage}
 	MainVersion                  = cephv1.CephVersionSpec{Image: mainTestImage, AllowUnsupported: true}
 	volumeReplicationBaseURL     = fmt.Sprintf("https://raw.githubusercontent.com/csi-addons/kubernetes-csi-addons/%s/config/crd/bases/", volumeReplicationVersion)
 	volumeReplicationCRDURL      = volumeReplicationBaseURL + "replication.storage.openshift.io_volumereplications.yaml"
@@ -97,8 +101,10 @@ func ReturnCephVersion() cephv1.CephVersionSpec {
 		return QuincyDevelVersion
 	case "reef-devel":
 		return ReefDevelVersion
+	case "squid-devel":
+		return SquidDevelVersion
 	default:
-		return ReefDevelVersion
+		return SquidDevelVersion
 	}
 }
 

tests/integration/ceph_upgrade_test.go

@@ -168,6 +168,17 @@ func (s *UpgradeSuite) testUpgrade(useHelm bool, initialCephVersion v1.CephVersi
 	s.verifyFilesAfterUpgrade(newFile, rbdFilesToRead, cephfsFilesToRead)
 	logger.Infof("Verified upgrade from quincy to reef")
 
+	//
+	// Upgrade from reef to squid
+	//
+	logger.Infof("*** UPGRADING CEPH FROM REEF TO SQUID ***")
+	s.gatherLogs(s.settings.OperatorNamespace, "_before_squid_upgrade")
+	s.upgradeCephVersion(installer.SquidVersion.Image, numOSDs)
+	// Verify reading and writing to the test clients
+	newFile = "post-squid-upgrade-file"
+	s.verifyFilesAfterUpgrade(newFile, rbdFilesToRead, cephfsFilesToRead)
+	logger.Infof("Verified upgrade from reef to squid")
+
 	checkCephObjectUser(&s.Suite, s.helper, s.k8sh, s.namespace, installer.ObjectStoreName, objectUserID, true, false)
 }
 
@@ -237,6 +248,39 @@ func (s *UpgradeSuite) TestUpgradeCephToReefDevel() {
 	checkCephObjectUser(&s.Suite, s.helper, s.k8sh, s.namespace, installer.ObjectStoreName, objectUserID, true, false)
 }
 
+func (s *UpgradeSuite) TestUpgradeCephToSquidDevel() {
+	s.baseSetup(false, installer.SquidVersion)
+
+	objectUserID := "upgraded-user"
+	preFilename := "pre-upgrade-file"
+	s.settings.CephVersion = installer.SquidVersion
+	numOSDs, rbdFilesToRead, cephfsFilesToRead := s.deployClusterforUpgrade(objectUserID, preFilename)
+	clusterInfo := client.AdminTestClusterInfo(s.namespace)
+	requireBlockImagesRemoved := false
+	defer func() {
+		blockTestDataCleanUp(s.helper, s.k8sh, &s.Suite, clusterInfo, installer.BlockPoolName, installer.BlockPoolSCName, blockName, rbdPodName, requireBlockImagesRemoved)
+		cleanupFilesystemConsumer(s.helper, s.k8sh, &s.Suite, s.namespace, filePodName)
+		cleanupFilesystem(s.helper, s.k8sh, &s.Suite, s.namespace, installer.FilesystemName)
+		_ = s.helper.ObjectUserClient.Delete(s.namespace, objectUserID)
+		_ = s.helper.BucketClient.DeleteObc(obcName, installer.ObjectStoreSCName, bucketPrefix, maxObject, false)
+		_ = s.helper.BucketClient.DeleteBucketStorageClass(s.namespace, installer.ObjectStoreName, installer.ObjectStoreSCName, "Delete")
+		objectStoreCleanUp(&s.Suite, s.helper, s.k8sh, s.settings.Namespace, installer.ObjectStoreName)
+	}()
+
+	//
+	// Upgrade from squid to squid devel
+	//
+	logger.Infof("*** UPGRADING CEPH FROM SQUID STABLE TO SQUID DEVEL ***")
+	s.gatherLogs(s.settings.OperatorNamespace, "_before_squid_upgrade")
+	s.upgradeCephVersion(installer.SquidDevelVersion.Image, numOSDs)
+	// Verify reading and writing to the test clients
+	newFile := "post-squid-upgrade-file"
+	s.verifyFilesAfterUpgrade(newFile, rbdFilesToRead, cephfsFilesToRead)
+	logger.Infof("verified upgrade from squid stable to squid devel")
+
+	checkCephObjectUser(&s.Suite, s.helper, s.k8sh, s.namespace, installer.ObjectStoreName, objectUserID, true, false)
+}
+
 func (s *UpgradeSuite) deployClusterforUpgrade(objectUserID, preFilename string) (int, []string, []string) {
 	//
 	// Create block, object, and file storage before the upgrade