From 5014d5bcf271aa952e1ce8d4da7cead5853c5b4b Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Thu, 15 Aug 2024 14:06:33 -0600 Subject: [PATCH] core: add annotations and labels to detect version jobs The jobs to detect the ceph and csi versions now can have custom annotationso and labels added to them. They are very short-lived jobs, but they may need custom annotations in some environments to run. Signed-off-by: Travis Nielsen --- Documentation/CRDs/specification.md | 2 ++ deploy/examples/cluster.yaml | 5 +++++ pkg/apis/ceph.rook.io/v1/annotations.go | 5 +++++ pkg/apis/ceph.rook.io/v1/annotations_test.go | 9 +++++++-- pkg/apis/ceph.rook.io/v1/keys.go | 1 + pkg/apis/ceph.rook.io/v1/labels.go | 4 ++++ pkg/apis/ceph.rook.io/v1/labels_test.go | 10 ++++++++-- pkg/operator/ceph/controller/network.go | 2 ++ pkg/operator/ceph/controller/version.go | 3 +++ pkg/operator/ceph/csi/controller.go | 6 ++++++ pkg/operator/ceph/csi/spec.go | 5 +++++ 11 files changed, 48 insertions(+), 4 deletions(-) diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index fc32110fceb7..fd4346f7344d 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -7482,6 +7482,8 @@ string

"clusterMetadata"

+

"cmdreporter"

+

"crashcollector"

"dashboard"

diff --git a/deploy/examples/cluster.yaml b/deploy/examples/cluster.yaml index 6e9619774ed9..6127c79d4fee 100644 --- a/deploy/examples/cluster.yaml +++ b/deploy/examples/cluster.yaml @@ -194,9 +194,14 @@ spec: annotations: # all: # mon: + # mgr: # osd: + # exporter: + # crashcollector: # cleanup: # prepareosd: + # cmdreporter is for jobs to detect ceph and csi versions, and check network status + # cmdreporter: # clusterMetadata annotations will be applied to only `rook-ceph-mon-endpoints` configmap and the `rook-ceph-mon` and `rook-ceph-admin-keyring` secrets. # And clusterMetadata annotations will not be merged with `all` annotations. # clusterMetadata: diff --git a/pkg/apis/ceph.rook.io/v1/annotations.go b/pkg/apis/ceph.rook.io/v1/annotations.go index b2d77dc2d06a..9610420be28c 100644 --- a/pkg/apis/ceph.rook.io/v1/annotations.go +++ b/pkg/apis/ceph.rook.io/v1/annotations.go @@ -72,6 +72,11 @@ func GetCephExporterAnnotations(a AnnotationsSpec) Annotations { return mergeAllAnnotationsWithKey(a, KeyCephExporter) } +// GetCmdReporterAnnotations returns the Annotations for jobs detecting versions +func GetCmdReporterAnnotations(a AnnotationsSpec) Annotations { + return mergeAllAnnotationsWithKey(a, KeyCmdReporter) +} + func GetClusterMetadataAnnotations(a AnnotationsSpec) Annotations { return a[KeyClusterMetadata] } diff --git a/pkg/apis/ceph.rook.io/v1/annotations_test.go b/pkg/apis/ceph.rook.io/v1/annotations_test.go index e3012a482d54..b832690c3567 100644 --- a/pkg/apis/ceph.rook.io/v1/annotations_test.go +++ b/pkg/apis/ceph.rook.io/v1/annotations_test.go @@ -58,8 +58,9 @@ func TestCephAnnotationsMerge(t *testing.T) { // Merge with "all" testAnnotations = AnnotationsSpec{ - "all": {"allkey1": "allval1", "allkey2": "allval2"}, - "mgr": {"mgrkey": "mgrval"}, + "all": {"allkey1": "allval1", "allkey2": "allval2"}, + "mgr": {"mgrkey": "mgrval"}, + "cmdreporter": {"myversions": "detect"}, } a = GetMonAnnotations(testAnnotations) assert.Equal(t, "allval1", a["allkey1"]) @@ -70,6 +71,10 @@ func TestCephAnnotationsMerge(t *testing.T) { assert.Equal(t, "allval1", a["allkey1"]) assert.Equal(t, "allval2", a["allkey2"]) assert.Equal(t, 3, len(a)) + b := GetCmdReporterAnnotations(testAnnotations) + assert.Equal(t, "detect", b["myversions"]) + assert.Equal(t, "allval1", b["allkey1"]) + assert.Equal(t, "allval2", b["allkey2"]) } func TestAnnotationsSpec(t *testing.T) { diff --git a/pkg/apis/ceph.rook.io/v1/keys.go b/pkg/apis/ceph.rook.io/v1/keys.go index 05395f1e3423..88467d31f3d5 100644 --- a/pkg/apis/ceph.rook.io/v1/keys.go +++ b/pkg/apis/ceph.rook.io/v1/keys.go @@ -32,4 +32,5 @@ const ( KeyCrashCollector KeyType = "crashcollector" KeyClusterMetadata KeyType = "clusterMetadata" KeyCephExporter KeyType = "exporter" + KeyCmdReporter KeyType = "cmdreporter" ) diff --git a/pkg/apis/ceph.rook.io/v1/labels.go b/pkg/apis/ceph.rook.io/v1/labels.go index ad4bb14660d4..6825838f8354 100644 --- a/pkg/apis/ceph.rook.io/v1/labels.go +++ b/pkg/apis/ceph.rook.io/v1/labels.go @@ -87,6 +87,10 @@ func GetCephExporterLabels(a LabelsSpec) Labels { return mergeAllLabelsWithKey(a, KeyCephExporter) } +func GetCmdReporterLabels(a LabelsSpec) Labels { + return mergeAllLabelsWithKey(a, KeyCmdReporter) +} + func mergeAllLabelsWithKey(a LabelsSpec, name KeyType) Labels { all := a.All() if all != nil { diff --git a/pkg/apis/ceph.rook.io/v1/labels_test.go b/pkg/apis/ceph.rook.io/v1/labels_test.go index c4810045310e..86668fb25f66 100644 --- a/pkg/apis/ceph.rook.io/v1/labels_test.go +++ b/pkg/apis/ceph.rook.io/v1/labels_test.go @@ -58,8 +58,9 @@ func TestCephLabelsMerge(t *testing.T) { // Merge with "all" testLabels = LabelsSpec{ - "all": {"allkey1": "allval1", "allkey2": "allval2"}, - "mgr": {"mgrkey": "mgrval"}, + "all": {"allkey1": "allval1", "allkey2": "allval2"}, + "mgr": {"mgrkey": "mgrval"}, + "cmdreporter": {"detect": "myversion"}, } a = GetMonLabels(testLabels) assert.Equal(t, "allval1", a["allkey1"]) @@ -70,6 +71,11 @@ func TestCephLabelsMerge(t *testing.T) { assert.Equal(t, "allval1", a["allkey1"]) assert.Equal(t, "allval2", a["allkey2"]) assert.Equal(t, 3, len(a)) + a = GetCmdReporterLabels(testLabels) + assert.Equal(t, "myversion", a["detect"]) + assert.Equal(t, "allval1", a["allkey1"]) + assert.Equal(t, "allval2", a["allkey2"]) + assert.Equal(t, 3, len(a)) } func TestLabelsSpec(t *testing.T) { diff --git a/pkg/operator/ceph/controller/network.go b/pkg/operator/ceph/controller/network.go index 4c64d37ab03f..b97cb454af1f 100644 --- a/pkg/operator/ceph/controller/network.go +++ b/pkg/operator/ceph/controller/network.go @@ -264,6 +264,8 @@ func discoverAddressRanges( job.Spec.Template.Annotations = map[string]string{ nadv1.NetworkAttachmentAnnot: netSelectionValue, } + cephv1.GetCmdReporterAnnotations(clusterSpec.Annotations).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta) + cephv1.GetCmdReporterLabels(clusterSpec.Labels).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta) // use osd placement for net canaries b/c osd pods are present on both public and cluster nets cephv1.GetOSDPlacement(clusterSpec.Placement).ApplyToPodSpec(&job.Spec.Template.Spec) diff --git a/pkg/operator/ceph/controller/version.go b/pkg/operator/ceph/controller/version.go index 1585cef77695..915672fda7d2 100644 --- a/pkg/operator/ceph/controller/version.go +++ b/pkg/operator/ceph/controller/version.go @@ -86,6 +86,9 @@ func DetectCephVersion(ctx context.Context, rookImage, namespace, jobName string cephv1.GetMonPlacement(cephClusterSpec.Placement).ApplyToPodSpec(&job.Spec.Template.Spec) job.Spec.Template.Spec.Affinity.PodAntiAffinity = nil + cephv1.GetCmdReporterAnnotations(cephClusterSpec.Annotations).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta) + cephv1.GetCmdReporterLabels(cephClusterSpec.Labels).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta) + stdout, stderr, retcode, err := versionReporter.Run(ctx, detectCephVersionTimeout) if err != nil { return nil, errors.Wrap(err, "failed to complete ceph version job") diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go index bd9912f9e595..db55853597da 100644 --- a/pkg/operator/ceph/csi/controller.go +++ b/pkg/operator/ceph/csi/controller.go @@ -58,6 +58,8 @@ type ReconcileCSI struct { opManagerContext context.Context opConfig opcontroller.OperatorConfig clustersWithHolder []ClusterDetail + // the first cluster CR which will determine some settings for the csi driver + firstCephCluster *cephv1.ClusterSpec } // ClusterDetail is a struct that holds the information of a cluster, it knows its internals (like @@ -276,6 +278,10 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e return reconcile.Result{}, nil } + if r.firstCephCluster == nil { + r.firstCephCluster = &cephClusters.Items[i].Spec + } + // Load cluster info for later use in updating the ceph-csi configmap clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cluster.Namespace, &cephClusters.Items[i].Spec) if err != nil { diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index a4eef9cc0faf..3d9924c313f7 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -25,6 +25,7 @@ import ( "strings" "time" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/operator/ceph/cluster/telemetry" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" @@ -874,6 +875,10 @@ func (r *ReconcileCSI) validateCSIVersion(ownerInfo *k8sutil.OwnerInfo) (*CephCS job.Spec.Template.Spec.Affinity = &corev1.Affinity{ NodeAffinity: getNodeAffinity(r.opConfig.Parameters, provisionerNodeAffinityEnv, &corev1.NodeAffinity{}), } + if r.firstCephCluster != nil { + cephv1.GetCmdReporterAnnotations(r.firstCephCluster.Annotations).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta) + cephv1.GetCmdReporterLabels(r.firstCephCluster.Labels).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta) + } stdout, _, retcode, err := versionReporter.Run(r.opManagerContext, timeout) if err != nil {