diff --git a/.github/workflows/daily-nightly-jobs.yml b/.github/workflows/daily-nightly-jobs.yml index 2766a905e017..4fa73a4edf77 100644 --- a/.github/workflows/daily-nightly-jobs.yml +++ b/.github/workflows/daily-nightly-jobs.yml @@ -195,6 +195,46 @@ jobs: name: ceph-smoke-suite-reef-artifact path: /home/runner/work/rook/rook/tests/integration/_output/tests/ + smoke-suite-squid-devel: + if: github.repository == 'rook/rook' + runs-on: ubuntu-22.04 + steps: + - name: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: consider debugging + uses: ./.github/workflows/tmate_debug + with: + use-tmate: ${{ secrets.USE_TMATE }} + + - name: setup cluster resources + uses: ./.github/workflows/integration-test-config-latest-k8s + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + kubernetes-version: "1.28.4" + + - name: TestCephSmokeSuite + run: | + export DEVICE_FILTER=$(tests/scripts/github-action-helper.sh find_extra_block_dev) + SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION="squid-devel" go test -v -timeout 1800s -run TestCephSmokeSuite github.com/rook/rook/tests/integration + + - name: collect common logs + if: always() + run: | + export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/" + export CLUSTER_NAMESPACE="smoke-ns" + export OPERATOR_NAMESPACE="smoke-ns-system" + tests/scripts/collect-logs.sh + + - name: Artifact + uses: actions/upload-artifact@v4 + if: failure() + with: + name: ceph-smoke-suite-squid-artifact + path: /home/runner/work/rook/rook/tests/integration/_output/tests/ + smoke-suite-ceph-main: if: github.repository == 'rook/rook' runs-on: ubuntu-22.04 @@ -315,6 +355,47 @@ jobs: name: ceph-object-suite-master-artifact path: /home/runner/work/rook/rook/tests/integration/_output/tests/ + upgrade-from-squid-stable-to-squid-devel: + if: github.repository == 'rook/rook' + runs-on: ubuntu-22.04 + steps: + - name: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: consider debugging + uses: ./.github/workflows/tmate_debug + with: + use-tmate: ${{ secrets.USE_TMATE }} + + - name: setup cluster resources + uses: ./.github/workflows/integration-test-config-latest-k8s + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + kubernetes-version: "1.28.4" + + - name: TestCephUpgradeSuite + run: | + export DEVICE_FILTER=$(tests/scripts/github-action-helper.sh find_extra_block_dev) + go test -v -timeout 1800s -failfast -run TestCephUpgradeSuite/TestUpgradeCephToSquidDevel github.com/rook/rook/tests/integration + + - name: collect common logs + if: always() + run: | + export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/" + export CLUSTER_NAMESPACE="upgrade" + export OPERATOR_NAMESPACE="upgrade-system" + tests/scripts/collect-logs.sh + + - name: Artifact + uses: actions/upload-artifact@v4 + if: failure() + with: + name: ceph-upgrade-suite-squid-artifact + path: /home/runner/work/rook/rook/tests/integration/_output/tests/ + + upgrade-from-reef-stable-to-reef-devel: if: github.repository == 'rook/rook' runs-on: ubuntu-22.04 @@ -399,5 +480,5 @@ jobs: if: github.repository == 'rook/rook' uses: ./.github/workflows/canary-integration-test.yml with: - ceph_images: '["quay.io/ceph/ceph:v18", "quay.io/ceph/daemon-base:latest-main-devel", "quay.io/ceph/daemon-base:latest-quincy-devel", "quay.io/ceph/daemon-base:latest-reef-devel"]' + ceph_images: '["quay.io/ceph/ceph:v18", "quay.io/ceph/daemon-base:latest-main-devel", "quay.io/ceph/daemon-base:latest-quincy-devel", "quay.io/ceph/daemon-base:latest-reef-devel", "quay.io/ceph/daemon-base:latest-squid-devel"]' secrets: inherit diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 0cf183144e02..556160b3364c 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -7,3 +7,5 @@ please update the `BucketClass` and `BucketAccessClass` for resolving refer [her - During CephBlockPool updates, return an error if an invalid device class is specified. Pools with invalid device classes may start failing reconcile until the correct device class is specified. See #14057. ## Features + +- Added support for Ceph Squid (v19) diff --git a/deploy/examples/cluster-test.yaml b/deploy/examples/cluster-test.yaml index 2f088c5d5705..db4c6d75f0ad 100644 --- a/deploy/examples/cluster-test.yaml +++ b/deploy/examples/cluster-test.yaml @@ -15,7 +15,7 @@ metadata: spec: dataDirHostPath: /var/lib/rook cephVersion: - image: quay.io/ceph/ceph:v18 + image: quay.io/ceph/ceph:v19 allowUnsupported: true mon: count: 1 diff --git a/go.mod b/go.mod index 0f69a7a7fa1c..20767f86a896 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ replace ( require ( github.com/IBM/keyprotect-go-client v0.14.3 - github.com/aws/aws-sdk-go v1.54.15 + github.com/aws/aws-sdk-go v1.54.19 github.com/banzaicloud/k8s-objectmatcher v1.8.0 github.com/ceph/go-ceph v0.28.0 github.com/coreos/pkg v0.0.0-20230601102743-20bbbf26f4d8 diff --git a/go.sum b/go.sum index 45d6ecc62a84..7954adc96846 100644 --- a/go.sum +++ b/go.sum @@ -144,8 +144,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.44.164/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.54.15 h1:ErgCEVbzuSfuZl9nR+g8FFnzjgeJ/AqAGOEWn6tgAHo= -github.com/aws/aws-sdk-go v1.54.15/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go v1.54.19 h1:tyWV+07jagrNiCcGRzRhdtVjQs7Vy41NwsuOcl0IbVI= +github.com/aws/aws-sdk-go v1.54.19/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/banzaicloud/k8s-objectmatcher v1.8.0 h1:Nugn25elKtPMTA2br+JgHNeSQ04sc05MDPmpJnd1N2A= github.com/banzaicloud/k8s-objectmatcher v1.8.0/go.mod h1:p2LSNAjlECf07fbhDyebTkPUIYnU05G+WfGgkTmgeMg= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= diff --git a/pkg/operator/ceph/cluster/version_test.go b/pkg/operator/ceph/cluster/version_test.go index 5f72340834b9..8f4a2d65747b 100755 --- a/pkg/operator/ceph/cluster/version_test.go +++ b/pkg/operator/ceph/cluster/version_test.go @@ -165,6 +165,8 @@ func TestMinVersion(t *testing.T) { assert.NoError(t, c.validateCephVersion(v)) v = &cephver.CephVersion{Major: 18} assert.NoError(t, c.validateCephVersion(v)) + v = &cephver.CephVersion{Major: 19} + assert.NoError(t, c.validateCephVersion(v)) } func TestSupportedVersion(t *testing.T) { @@ -183,8 +185,12 @@ func TestSupportedVersion(t *testing.T) { v = &cephver.CephVersion{Major: 18, Minor: 2, Extra: 0} assert.NoError(t, c.validateCephVersion(v)) - // Squid is not supported - v = &cephver.CephVersion{Major: 19, Minor: 1, Extra: 0} + // Squid is supported + v = &cephver.CephVersion{Major: 19, Minor: 2, Extra: 0} + assert.NoError(t, c.validateCephVersion(v)) + + // Tentacle release is not supported + v = &cephver.CephVersion{Major: 20, Minor: 1, Extra: 0} assert.Error(t, c.validateCephVersion(v)) // Unsupported versions are now valid diff --git a/pkg/operator/ceph/version/version.go b/pkg/operator/ceph/version/version.go index 92d764d6e2fc..705258102da1 100644 --- a/pkg/operator/ceph/version/version.go +++ b/pkg/operator/ceph/version/version.go @@ -48,9 +48,11 @@ var ( Reef = CephVersion{18, 0, 0, 0, ""} // Squid ceph version Squid = CephVersion{19, 0, 0, 0, ""} + // Tentacle ceph version + Tentacle = CephVersion{20, 0, 0, 0, ""} // supportedVersions are production-ready versions that rook supports - supportedVersions = []CephVersion{Quincy, Reef} + supportedVersions = []CephVersion{Quincy, Reef, Squid} // unsupportedVersions are possibly Ceph pin-point release that introduced breaking changes and not recommended unsupportedVersions []CephVersion @@ -90,6 +92,8 @@ func (v *CephVersion) ReleaseName() string { return "reef" case Squid.Major: return "squid" + case Tentacle.Major: + return "tentacle" default: return unknownVersionString } @@ -180,6 +184,11 @@ func (v *CephVersion) IsSquid() bool { return v.isRelease(Squid) } +// IsTentacle checks if the Ceph version is Tentacle +func (v *CephVersion) IsTentacle() bool { + return v.isRelease(Tentacle) +} + // IsAtLeast checks a given Ceph version is at least a given one func (v *CephVersion) IsAtLeast(other CephVersion) bool { if v.Major > other.Major { @@ -203,12 +212,22 @@ func (v *CephVersion) IsAtLeast(other CephVersion) bool { return true } -// IsAtLeastReef check that the Ceph version is at least Reef +// IsAtLeastTentacle checks that the Ceph version is at least Tentacle +func (v *CephVersion) IsAtLeastTentacle() bool { + return v.IsAtLeast(Tentacle) +} + +// IsAtLeastSquid checks that the Ceph version is at least Squid +func (v *CephVersion) IsAtLeastSquid() bool { + return v.IsAtLeast(Squid) +} + +// IsAtLeastReef checks that the Ceph version is at least Reef func (v *CephVersion) IsAtLeastReef() bool { return v.IsAtLeast(Reef) } -// IsAtLeastQuincy check that the Ceph version is at least Quincy +// IsAtLeastQuincy checks that the Ceph version is at least Quincy func (v *CephVersion) IsAtLeastQuincy() bool { return v.IsAtLeast(Quincy) } diff --git a/pkg/operator/ceph/version/version_test.go b/pkg/operator/ceph/version/version_test.go index 36e2e895c7c7..1a95a3f1cd9e 100644 --- a/pkg/operator/ceph/version/version_test.go +++ b/pkg/operator/ceph/version/version_test.go @@ -101,7 +101,7 @@ func TestSupported(t *testing.T) { for _, v := range supportedVersions { assert.True(t, v.Supported()) } - assert.False(t, Squid.Supported()) + assert.False(t, Tentacle.Supported()) } func TestIsRelease(t *testing.T) { diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 680925465974..9d7a725b7fad 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -45,9 +45,11 @@ const ( // test with the latest releases quincyTestImage = "quay.io/ceph/ceph:v17" reefTestImage = "quay.io/ceph/ceph:v18" + squidTestImage = "quay.io/ceph/ceph:v19" // test with the current development versions quincyDevelTestImage = "quay.io/ceph/daemon-base:latest-quincy-devel" reefDevelTestImage = "quay.io/ceph/daemon-base:latest-reef-devel" + squidDevelTestImage = "quay.io/ceph/daemon-base:latest-squid-devel" // test with the latest Ceph main image mainTestImage = "quay.io/ceph/daemon-base:latest-main-devel" cephOperatorLabel = "app=rook-ceph-operator" @@ -71,6 +73,8 @@ var ( QuincyDevelVersion = cephv1.CephVersionSpec{Image: quincyDevelTestImage} ReefVersion = cephv1.CephVersionSpec{Image: reefTestImage} ReefDevelVersion = cephv1.CephVersionSpec{Image: reefDevelTestImage} + SquidVersion = cephv1.CephVersionSpec{Image: squidTestImage} + SquidDevelVersion = cephv1.CephVersionSpec{Image: squidDevelTestImage} MainVersion = cephv1.CephVersionSpec{Image: mainTestImage, AllowUnsupported: true} volumeReplicationBaseURL = fmt.Sprintf("https://raw.githubusercontent.com/csi-addons/kubernetes-csi-addons/%s/config/crd/bases/", volumeReplicationVersion) volumeReplicationCRDURL = volumeReplicationBaseURL + "replication.storage.openshift.io_volumereplications.yaml" @@ -97,8 +101,10 @@ func ReturnCephVersion() cephv1.CephVersionSpec { return QuincyDevelVersion case "reef-devel": return ReefDevelVersion + case "squid-devel": + return SquidDevelVersion default: - return ReefDevelVersion + return SquidDevelVersion } } diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go index 98db0ee65e86..9c10a4cdb86e 100644 --- a/tests/integration/ceph_upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -168,6 +168,17 @@ func (s *UpgradeSuite) testUpgrade(useHelm bool, initialCephVersion v1.CephVersi s.verifyFilesAfterUpgrade(newFile, rbdFilesToRead, cephfsFilesToRead) logger.Infof("Verified upgrade from quincy to reef") + // + // Upgrade from reef to squid + // + logger.Infof("*** UPGRADING CEPH FROM REEF TO SQUID ***") + s.gatherLogs(s.settings.OperatorNamespace, "_before_squid_upgrade") + s.upgradeCephVersion(installer.SquidVersion.Image, numOSDs) + // Verify reading and writing to the test clients + newFile = "post-squid-upgrade-file" + s.verifyFilesAfterUpgrade(newFile, rbdFilesToRead, cephfsFilesToRead) + logger.Infof("Verified upgrade from reef to squid") + checkCephObjectUser(&s.Suite, s.helper, s.k8sh, s.namespace, installer.ObjectStoreName, objectUserID, true, false) } @@ -237,6 +248,39 @@ func (s *UpgradeSuite) TestUpgradeCephToReefDevel() { checkCephObjectUser(&s.Suite, s.helper, s.k8sh, s.namespace, installer.ObjectStoreName, objectUserID, true, false) } +func (s *UpgradeSuite) TestUpgradeCephToSquidDevel() { + s.baseSetup(false, installer.SquidVersion) + + objectUserID := "upgraded-user" + preFilename := "pre-upgrade-file" + s.settings.CephVersion = installer.SquidVersion + numOSDs, rbdFilesToRead, cephfsFilesToRead := s.deployClusterforUpgrade(objectUserID, preFilename) + clusterInfo := client.AdminTestClusterInfo(s.namespace) + requireBlockImagesRemoved := false + defer func() { + blockTestDataCleanUp(s.helper, s.k8sh, &s.Suite, clusterInfo, installer.BlockPoolName, installer.BlockPoolSCName, blockName, rbdPodName, requireBlockImagesRemoved) + cleanupFilesystemConsumer(s.helper, s.k8sh, &s.Suite, s.namespace, filePodName) + cleanupFilesystem(s.helper, s.k8sh, &s.Suite, s.namespace, installer.FilesystemName) + _ = s.helper.ObjectUserClient.Delete(s.namespace, objectUserID) + _ = s.helper.BucketClient.DeleteObc(obcName, installer.ObjectStoreSCName, bucketPrefix, maxObject, false) + _ = s.helper.BucketClient.DeleteBucketStorageClass(s.namespace, installer.ObjectStoreName, installer.ObjectStoreSCName, "Delete") + objectStoreCleanUp(&s.Suite, s.helper, s.k8sh, s.settings.Namespace, installer.ObjectStoreName) + }() + + // + // Upgrade from squid to squid devel + // + logger.Infof("*** UPGRADING CEPH FROM SQUID STABLE TO SQUID DEVEL ***") + s.gatherLogs(s.settings.OperatorNamespace, "_before_squid_upgrade") + s.upgradeCephVersion(installer.SquidDevelVersion.Image, numOSDs) + // Verify reading and writing to the test clients + newFile := "post-squid-upgrade-file" + s.verifyFilesAfterUpgrade(newFile, rbdFilesToRead, cephfsFilesToRead) + logger.Infof("verified upgrade from squid stable to squid devel") + + checkCephObjectUser(&s.Suite, s.helper, s.k8sh, s.namespace, installer.ObjectStoreName, objectUserID, true, false) +} + func (s *UpgradeSuite) deployClusterforUpgrade(objectUserID, preFilename string) (int, []string, []string) { // // Create block, object, and file storage before the upgrade