Merge pull request rook#14576 from travisn/version-detect-annotation

core: Add annotations and labels to detect version jobs

Documentation/CRDs/specification.md

@@ -7482,6 +7482,8 @@ string
 <td></td>
 </tr><tr><td><p>&#34;clusterMetadata&#34;</p></td>
 <td></td>
+</tr><tr><td><p>&#34;cmdreporter&#34;</p></td>
+<td></td>
 </tr><tr><td><p>&#34;crashcollector&#34;</p></td>
 <td></td>
 </tr><tr><td><p>&#34;dashboard&#34;</p></td>

deploy/examples/cluster.yaml

@@ -194,9 +194,14 @@ spec:
   annotations:
   #   all:
   #   mon:
+  #   mgr:
   #   osd:
+  #   exporter:
+  #   crashcollector:
   #   cleanup:
   #   prepareosd:
+  # cmdreporter is for jobs to detect ceph and csi versions, and check network status
+  #   cmdreporter:
   # clusterMetadata annotations will be applied to only `rook-ceph-mon-endpoints` configmap and the `rook-ceph-mon` and `rook-ceph-admin-keyring` secrets.
   # And clusterMetadata annotations will not be merged with `all` annotations.
   #    clusterMetadata:

pkg/apis/ceph.rook.io/v1/annotations.go

@@ -72,6 +72,11 @@ func GetCephExporterAnnotations(a AnnotationsSpec) Annotations {
 	return mergeAllAnnotationsWithKey(a, KeyCephExporter)
 }
 
+// GetCmdReporterAnnotations returns the Annotations for jobs detecting versions
+func GetCmdReporterAnnotations(a AnnotationsSpec) Annotations {
+	return mergeAllAnnotationsWithKey(a, KeyCmdReporter)
+}
+
 func GetClusterMetadataAnnotations(a AnnotationsSpec) Annotations {
 	return a[KeyClusterMetadata]
 }

pkg/apis/ceph.rook.io/v1/annotations_test.go

@@ -58,8 +58,9 @@ func TestCephAnnotationsMerge(t *testing.T) {
 
 	// Merge with "all"
 	testAnnotations = AnnotationsSpec{
-		"all": {"allkey1": "allval1", "allkey2": "allval2"},
-		"mgr": {"mgrkey": "mgrval"},
+		"all":         {"allkey1": "allval1", "allkey2": "allval2"},
+		"mgr":         {"mgrkey": "mgrval"},
+		"cmdreporter": {"myversions": "detect"},
 	}
 	a = GetMonAnnotations(testAnnotations)
 	assert.Equal(t, "allval1", a["allkey1"])
@@ -70,6 +71,10 @@ func TestCephAnnotationsMerge(t *testing.T) {
 	assert.Equal(t, "allval1", a["allkey1"])
 	assert.Equal(t, "allval2", a["allkey2"])
 	assert.Equal(t, 3, len(a))
+	b := GetCmdReporterAnnotations(testAnnotations)
+	assert.Equal(t, "detect", b["myversions"])
+	assert.Equal(t, "allval1", b["allkey1"])
+	assert.Equal(t, "allval2", b["allkey2"])
 }
 
 func TestAnnotationsSpec(t *testing.T) {

pkg/apis/ceph.rook.io/v1/keys.go

@@ -32,4 +32,5 @@ const (
 	KeyCrashCollector  KeyType = "crashcollector"
 	KeyClusterMetadata KeyType = "clusterMetadata"
 	KeyCephExporter    KeyType = "exporter"
+	KeyCmdReporter     KeyType = "cmdreporter"
 )

pkg/apis/ceph.rook.io/v1/labels.go

@@ -87,6 +87,10 @@ func GetCephExporterLabels(a LabelsSpec) Labels {
 	return mergeAllLabelsWithKey(a, KeyCephExporter)
 }
 
+func GetCmdReporterLabels(a LabelsSpec) Labels {
+	return mergeAllLabelsWithKey(a, KeyCmdReporter)
+}
+
 func mergeAllLabelsWithKey(a LabelsSpec, name KeyType) Labels {
 	all := a.All()
 	if all != nil {

pkg/apis/ceph.rook.io/v1/labels_test.go

@@ -58,8 +58,9 @@ func TestCephLabelsMerge(t *testing.T) {
 
 	// Merge with "all"
 	testLabels = LabelsSpec{
-		"all": {"allkey1": "allval1", "allkey2": "allval2"},
-		"mgr": {"mgrkey": "mgrval"},
+		"all":         {"allkey1": "allval1", "allkey2": "allval2"},
+		"mgr":         {"mgrkey": "mgrval"},
+		"cmdreporter": {"detect": "myversion"},
 	}
 	a = GetMonLabels(testLabels)
 	assert.Equal(t, "allval1", a["allkey1"])
@@ -70,6 +71,11 @@ func TestCephLabelsMerge(t *testing.T) {
 	assert.Equal(t, "allval1", a["allkey1"])
 	assert.Equal(t, "allval2", a["allkey2"])
 	assert.Equal(t, 3, len(a))
+	a = GetCmdReporterLabels(testLabels)
+	assert.Equal(t, "myversion", a["detect"])
+	assert.Equal(t, "allval1", a["allkey1"])
+	assert.Equal(t, "allval2", a["allkey2"])
+	assert.Equal(t, 3, len(a))
 }
 
 func TestLabelsSpec(t *testing.T) {

pkg/operator/ceph/controller/network.go

@@ -264,6 +264,8 @@ func discoverAddressRanges(
 	job.Spec.Template.Annotations = map[string]string{
 		nadv1.NetworkAttachmentAnnot: netSelectionValue,
 	}
+	cephv1.GetCmdReporterAnnotations(clusterSpec.Annotations).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta)
+	cephv1.GetCmdReporterLabels(clusterSpec.Labels).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta)
 
 	// use osd placement for net canaries b/c osd pods are present on both public and cluster nets
 	cephv1.GetOSDPlacement(clusterSpec.Placement).ApplyToPodSpec(&job.Spec.Template.Spec)

pkg/operator/ceph/controller/version.go

@@ -86,6 +86,9 @@ func DetectCephVersion(ctx context.Context, rookImage, namespace, jobName string
 	cephv1.GetMonPlacement(cephClusterSpec.Placement).ApplyToPodSpec(&job.Spec.Template.Spec)
 	job.Spec.Template.Spec.Affinity.PodAntiAffinity = nil
 
+	cephv1.GetCmdReporterAnnotations(cephClusterSpec.Annotations).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta)
+	cephv1.GetCmdReporterLabels(cephClusterSpec.Labels).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta)
+
 	stdout, stderr, retcode, err := versionReporter.Run(ctx, detectCephVersionTimeout)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to complete ceph version job")

pkg/operator/ceph/csi/controller.go

@@ -58,6 +58,8 @@ type ReconcileCSI struct {
 	opManagerContext   context.Context
 	opConfig           opcontroller.OperatorConfig
 	clustersWithHolder []ClusterDetail
+	// the first cluster CR which will determine some settings for the csi driver
+	firstCephCluster *cephv1.ClusterSpec
 }
 
 // ClusterDetail is a struct that holds the information of a cluster, it knows its internals (like
@@ -276,6 +278,10 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
 			return reconcile.Result{}, nil
 		}
 
+		if r.firstCephCluster == nil {
+			r.firstCephCluster = &cephClusters.Items[i].Spec
+		}
+
 		// Load cluster info for later use in updating the ceph-csi configmap
 		clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cluster.Namespace, &cephClusters.Items[i].Spec)
 		if err != nil {

pkg/operator/ceph/csi/spec.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 	"time"
 
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/operator/ceph/cluster/telemetry"
 	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
 	"github.com/rook/rook/pkg/operator/k8sutil"
@@ -874,6 +875,10 @@ func (r *ReconcileCSI) validateCSIVersion(ownerInfo *k8sutil.OwnerInfo) (*CephCS
 	job.Spec.Template.Spec.Affinity = &corev1.Affinity{
 		NodeAffinity: getNodeAffinity(r.opConfig.Parameters, provisionerNodeAffinityEnv, &corev1.NodeAffinity{}),
 	}
+	if r.firstCephCluster != nil {
+		cephv1.GetCmdReporterAnnotations(r.firstCephCluster.Annotations).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta)
+		cephv1.GetCmdReporterLabels(r.firstCephCluster.Labels).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta)
+	}
 
 	stdout, _, retcode, err := versionReporter.Run(r.opManagerContext, timeout)
 	if err != nil {