Configure Cassandra nodes to lookup their IP address by their fully q…

…ualified domain name

* Also ensure that they only ever use the Kubernetes seed provider service
* And ensure that that service publishes the seed IP addresses as soon as they are known,
* so that seeds them selves can find themselves when they are starting up.

Fixes: #319

pkg/controllers/cassandra/nodepool/resource.go

@@ -57,6 +57,7 @@ func StatefulSetForCluster(
 				Type: apps.RollingUpdateStatefulSetStrategyType,
 			},
 			PodManagementPolicy: apps.OrderedReadyPodManagement,
+			ServiceName:         statefulSetName,
 			Template: apiv1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels: nodePoolLabels,
@@ -195,6 +196,34 @@ func StatefulSetForCluster(
 									Name:  "HEAP_NEWSIZE",
 									Value: "100M",
 								},
+								// Deliberately set to a single space to force Cassandra to do a host name lookup.
+								// See https://github.com/apache/cassandra/blob/cassandra-3.11.2/conf/cassandra.yaml#L592
+								{
+									Name:  "CASSANDRA_LISTEN_ADDRESS",
+									Value: " ",
+								},
+								{
+									Name:  "CASSANDRA_BROADCAST_ADDRESS",
+									Value: " ",
+								},
+								{
+									Name:  "CASSANDRA_RPC_ADDRESS",
+									Value: " ",
+								},
+								// Set a non-existent default seed.
+								// The Kubernetes Seed Provider will fall back to a default seed host if it can't look up seeds via the CASSANDRA_SERVICE.
+								// And if the CASSANDRA_SEEDS environment variable is not set, it defaults to localhost.
+								// Which could cause confusion if a non-seed node is temporarily unable to lookup the seed nodes from the service.
+								// We want the list of seeds to be strictly controlled by the service.
+								// See:
+								// https://github.com/docker-library/cassandra/blame/master/3.11/docker-entrypoint.sh#L31 and
+								// https://github.com/apache/cassandra/blob/cassandra-3.11.2/conf/cassandra.yaml#L416 and
+								// https://github.com/kubernetes/examples/blob/cabf8b8e4739e576837111e156763d19a64a3591/cassandra/java/src/main/java/io/k8s/cassandra/KubernetesSeedProvider.java#L69 and
+								// https://github.com/kubernetes/examples/blob/cabf8b8e4739e576837111e156763d19a64a3591/cassandra/go/main.go#L51
+								{
+									Name:  "CASSANDRA_SEEDS",
+									Value: "black-hole-dns-name",
+								},
 								{
 									Name:  "CASSANDRA_ENDPOINT_SNITCH",
 									Value: cassSnitch,

pkg/controllers/cassandra/service/control.go

@@ -20,6 +20,10 @@ import (
 const (
 	SeedLabelKey   = "navigator.jetstack.io/cassandra-seed"
 	SeedLabelValue = "true"
+
+	// This field is deprecated. v1.Service.PublishNotReadyAddresses will replace it.
+	// See https://github.com/kubernetes/kubernetes/blob/v1.10.0/pkg/controller/endpoint/endpoints_controller.go#L68
+	tolerateUnreadyEndpointsAnnotationKey = "service.alpha.kubernetes.io/tolerate-unready-endpoints"
 )
 
 type serviceFactory func(*v1alpha1.CassandraCluster) *apiv1.Service
@@ -93,8 +97,11 @@ func SeedsServiceForCluster(cluster *v1alpha1.CassandraCluster) *apiv1.Service {
 	labels := util.ClusterLabels(cluster)
 	service := &apiv1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:            util.SeedsServiceName(cluster),
-			Namespace:       cluster.Namespace,
+			Name:      util.SeedsServiceName(cluster),
+			Namespace: cluster.Namespace,
+			Annotations: map[string]string{
+				tolerateUnreadyEndpointsAnnotationKey: "true",
+			},
 			OwnerReferences: []metav1.OwnerReference{util.NewControllerRef(cluster)},
 			Labels:          labels,
 		},
@@ -106,6 +113,10 @@ func SeedsServiceForCluster(cluster *v1alpha1.CassandraCluster) *apiv1.Service {
 			// But without it, DNS records are not registered.
 			// See https://github.com/kubernetes/kubernetes/issues/55158
 			Ports: []apiv1.ServicePort{{Port: 65535}},
+			// This ensures that seed node addresses are published immediately.
+			// And are available during C* node start up.
+			// Even for seed nodes them selves.
+			PublishNotReadyAddresses: true,
 		},
 	}
 	// Only mark nodes explicitly labeled as seeds as seed nodes