doc(contrib): add better docs for helm and LB (#3920)

contrib/helm/cds/README.md

@@ -25,7 +25,7 @@ This chart bootstraps a [CDS](https://github.com/ovh/cds) deployment on a [Kuber
 
 It starts a PostgreSQL server, a Redis server and an Elasticsearch server using the helm built-in dependency system. It also starts all µServices that CDS needed to use all CDS features :
 
-+ Hatchery over Kubernetes
++ Hatchery over Kubernetes (Only over the same kubernetes cluster and same namespace)
 + VCS service
 + Hooks service
 + Elasticsearch service

contrib/helm/cds/config.toml

@@ -1,137 +1,3 @@
-[hatchery]
-
-  # Hatchery Kubernetes.
-  [hatchery.kubernetes]
-
-    # Certificate authority data (content, not path and not base64 encoded) for tls kubernetes (optional if no tls needed)
-    # certAuthorityData = ""
-
-    # Client certificate data (content, not path and not base64 encoded) for tls kubernetes (optional if no tls needed)
-    # clientCertData = ""
-
-    # Client certificate data (content, not path and not base64 encoded) for tls kubernetes (optional if no tls needed)
-    # clientKeyData = ""
-
-    # Worker default memory in Mo
-    defaultMemory = 1024
-
-    # Kubernetes config file in yaml
-    kubernetesConfigFile = "kubeconfig.yaml"
-
-    # Address of kubernetes master
-    kubernetesMasterURL = "https://1.1.1.1:8443"
-
-    # Kubernetes namespace in which workers are spawned
-    namespace = "cds"
-
-    # Nb Workers to provision
-    nbProvision = 1
-
-    # Password to connect to kubernetes cluster (optional if config file is set)
-    # password = ""
-
-    # Token to connect to kubernetes cluster (optional if config file is set)
-    # token = ""
-
-    # Username to connect to kubernetes cluster (optional if config file is set)
-    # username = ""
-
-    # Worker TTL (minutes)
-    workerTTL = 10
-
-    [hatchery.kubernetes.commonConfiguration]
-
-      # Name of Hatchery
-      name = ""
-
-      # URL of this Hatchery
-      url = "http://localhost:8086"
-
-      [hatchery.kubernetes.commonConfiguration.api]
-
-        # Maximum allowed consecutives failures on heatbeat routine
-        maxHeartbeatFailures = 10
-
-        # Request CDS API: timeout in seconds
-        requestTimeout = 10
-
-        # CDS Token to reach CDS API. See https://ovh.github.io/cds/advanced/advanced.worker.token/
-        token = ""
-
-        [hatchery.kubernetes.commonConfiguration.api.grpc]
-
-          # sslInsecureSkipVerify, set to true if you use a self-signed SSL on CDS API
-          # insecure = false
-          # url = "http://localhost:8082"
-
-        [hatchery.kubernetes.commonConfiguration.api.http]
-
-          # sslInsecureSkipVerify, set to true if you use a self-signed SSL on CDS API
-          # insecure = false
-
-          # CDS API URL
-          # url = "http://localhost:8081"
-
-      ######################
-      # CDS Hatchery HTTP Configuration
-      #######################
-      [hatchery.kubernetes.commonConfiguration.http]
-
-        # Listen address without port, example: 127.0.0.1
-        # addr = ""
-        port = 8086
-
-      # Hatchery Log Configuration
-      [hatchery.kubernetes.commonConfiguration.logOptions]
-
-        [hatchery.kubernetes.commonConfiguration.logOptions.spawnOptions]
-
-          # log critical if spawn take more than this value (in seconds)
-          thresholdCritical = 480
-
-          # log warning if spawn take more than this value (in seconds)
-          thresholdWarning = 360
-
-      [hatchery.kubernetes.commonConfiguration.provision]
-
-        # Disabled provisioning. Format:true or false
-        disabled = false
-
-        # Check provisioning each n Seconds
-        frequency = 30
-
-        # if worker is queued less than this value (seconds), hatchery does not take care of it
-        graceTimeQueued = 4
-
-        # Maximum allowed simultaneous workers provisioning
-        maxConcurrentProvisioning = 10
-
-        # Maximum allowed simultaneous workers
-        maxWorker = 10
-
-        # Check if some worker model have to be registered each n Seconds
-        registerFrequency = 60
-
-        # Worker Log Configuration
-        [hatchery.kubernetes.commonConfiguration.provision.workerLogsOptions]
-
-          [hatchery.kubernetes.commonConfiguration.provision.workerLogsOptions.graylog]
-
-            # Example: X-OVH-TOKEN. You can use many keys: aaa,bbb
-            extraKey = ""
-
-            # value for extraKey field. For many keys: valueaaa,valuebbb
-            extraValue = ""
-
-            # Example: thot.ovh.com
-            host = ""
-
-            # Example: 12202
-            port = 0
-
-            # tcp or udp
-            protocol = "tcp"
-
 ######################
 # CDS Hooks Settings
 #######################

contrib/helm/cds/templates/NOTES.txt

@@ -19,7 +19,8 @@
   NOTE: It may take a few minutes for the LoadBalancer IP to be available.
         Watch the status with: 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ template "cds.fullname" . }}-ui'
 
-  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "cds.fullname" . }}-ui -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "cds.fullname" . }}-ui -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')
+  # It could take time
   echo http://$SERVICE_IP/
 {{- else if contains "ClusterIP"  .Values.ui.serviceType }}
 

contrib/helm/cds/templates/hatchery-k8s-deployment.yaml

@@ -30,8 +30,6 @@ spec:
             secretKeyRef:
               name: {{ template "cds.fullname" . }}
               key: cds-k8s_config_file
-        - name: CDS_HATCHERY_KUBERNETES_KUBERNETESMASTERURL
-          value: https://donotuse
         - name: CDS_HATCHERY_KUBERNETES_NAMESPACE
           value: "{{ .Values.hatchery.k8s_namespace }}"
         - name: CDS_HATCHERY_KUBERNETES_KUBERNETESCONFIGFILE

contrib/helm/cds/values.yaml

@@ -163,7 +163,7 @@ ingress:
 
 # hatchery kubernetes
 hatchery:
-  k8s_namespace: default
+  k8s_namespace: cds
 
 # Ui
 ui:

engine/hatchery/kubernetes/kill_workers.go

@@ -12,7 +12,7 @@ import (
 )
 
 func (h *HatcheryKubernetes) killAwolWorkers() error {
-	pods, err := h.k8sClient.CoreV1().Pods(h.Config.KubernetesNamespace).List(metav1.ListOptions{LabelSelector: LABEL_WORKER})
+	pods, err := h.k8sClient.CoreV1().Pods(h.Config.Namespace).List(metav1.ListOptions{LabelSelector: LABEL_WORKER})
 	if err != nil {
 		return err
 	}

engine/hatchery/kubernetes/kubernetes.go

@@ -94,12 +94,12 @@ func (h *HatcheryKubernetes) ApplyConfiguration(cfg interface{}) error {
 	}
 	h.k8sClient = clientset
 
-	if h.Config.KubernetesNamespace != apiv1.NamespaceDefault {
-		if _, err := clientset.CoreV1().Namespaces().Get(h.Config.KubernetesNamespace, metav1.GetOptions{}); err != nil {
+	if h.Config.Namespace != apiv1.NamespaceDefault {
+		if _, err := clientset.CoreV1().Namespaces().Get(h.Config.Namespace, metav1.GetOptions{}); err != nil {
 			ns := apiv1.Namespace{}
-			ns.SetName(h.Config.KubernetesNamespace)
+			ns.SetName(h.Config.Namespace)
 			if _, errC := clientset.CoreV1().Namespaces().Create(&ns); errC != nil {
-				return sdk.WrapError(errC, "Cannot create namespace %s in kubernetes", h.Config.KubernetesNamespace)
+				return sdk.WrapError(errC, "Cannot create namespace %s in kubernetes", h.Config.Namespace)
 			}
 		}
 	}
@@ -168,7 +168,7 @@ func (h *HatcheryKubernetes) CheckConfiguration(cfg interface{}) error {
 		return fmt.Errorf("please enter a name in your kubernetes hatchery configuration")
 	}
 
-	if hconfig.KubernetesNamespace == "" {
+	if hconfig.Namespace == "" {
 		return fmt.Errorf("please enter a valid kubernetes namespace")
 	}
 
@@ -326,6 +326,7 @@ func (h *HatcheryKubernetes) SpawnWorker(ctx context.Context, spawnArgs hatchery
 	podSchema := apiv1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:                       name,
+			Namespace:                  h.Config.Namespace,
 			DeletionGracePeriodSeconds: &gracePeriodSecs,
 			Labels: map[string]string{
 				LABEL_WORKER:        label,
@@ -400,7 +401,7 @@ func (h *HatcheryKubernetes) SpawnWorker(ctx context.Context, spawnArgs hatchery
 		podSchema.Spec.HostAliases[0].Hostnames[i+1] = strings.ToLower(serv.Name)
 	}
 
-	pod, err := h.k8sClient.CoreV1().Pods(h.Config.KubernetesNamespace).Create(&podSchema)
+	pod, err := h.k8sClient.CoreV1().Pods(h.Config.Namespace).Create(&podSchema)
 
 	log.Debug("hatchery> kubernetes> SpawnWorker> %s > Pod created", name)
 
@@ -410,9 +411,9 @@ func (h *HatcheryKubernetes) SpawnWorker(ctx context.Context, spawnArgs hatchery
 // WorkersStarted returns the number of instances started but
 // not necessarily register on CDS yet
 func (h *HatcheryKubernetes) WorkersStarted() []string {
-	list, err := h.k8sClient.CoreV1().Pods(h.Config.KubernetesNamespace).List(metav1.ListOptions{LabelSelector: LABEL_HATCHERY_NAME})
+	list, err := h.k8sClient.CoreV1().Pods(h.Config.Namespace).List(metav1.ListOptions{LabelSelector: LABEL_HATCHERY_NAME})
 	if err != nil {
-		log.Warning("WorkersStarted> unable to list pods on namespace %s", h.Config.KubernetesNamespace)
+		log.Warning("WorkersStarted> unable to list pods on namespace %s", h.Config.Namespace)
 		return nil
 	}
 	workerNames := make([]string, 0, list.Size())
@@ -428,7 +429,7 @@ func (h *HatcheryKubernetes) WorkersStarted() []string {
 // WorkersStartedByModel returns the number of instances of given model started but
 // not necessarily register on CDS yet
 func (h *HatcheryKubernetes) WorkersStartedByModel(model *sdk.Model) int {
-	list, err := h.k8sClient.CoreV1().Pods(h.Config.KubernetesNamespace).List(metav1.ListOptions{LabelSelector: LABEL_WORKER_MODEL})
+	list, err := h.k8sClient.CoreV1().Pods(h.Config.Namespace).List(metav1.ListOptions{LabelSelector: LABEL_WORKER_MODEL})
 	if err != nil {
 		log.Error("WorkersStartedByModel> Cannot get list of workers started (%s)", err)
 		return 0

engine/hatchery/kubernetes/services.go

@@ -14,7 +14,7 @@ import (
 )
 
 func (h *HatcheryKubernetes) getServicesLogs() error {
-	pods, err := h.k8sClient.CoreV1().Pods(h.Config.KubernetesNamespace).List(metav1.ListOptions{LabelSelector: LABEL_SERVICE_JOB_ID})
+	pods, err := h.k8sClient.CoreV1().Pods(h.Config.Namespace).List(metav1.ListOptions{LabelSelector: LABEL_SERVICE_JOB_ID})
 	if err != nil {
 		return err
 	}
@@ -45,7 +45,7 @@ func (h *HatcheryKubernetes) getServicesLogs() error {
 				continue
 			}
 			logsOpts := apiv1.PodLogOptions{SinceSeconds: &sinceSeconds, Container: container.Name, Timestamps: true}
-			logs, errLogs := h.k8sClient.CoreV1().Pods(h.Config.KubernetesNamespace).GetLogs(podName, &logsOpts).DoRaw()
+			logs, errLogs := h.k8sClient.CoreV1().Pods(h.Config.Namespace).GetLogs(podName, &logsOpts).DoRaw()
 			if errLogs != nil {
 				log.Error("getServicesLogs> cannot get logs for container %s in pod %s, err : %v", container.Name, podName, errLogs)
 				continue

engine/hatchery/kubernetes/types.go

@@ -31,8 +31,8 @@ type HatcheryConfiguration struct {
 	WorkerTTL int `mapstructure:"workerTTL" toml:"workerTTL" default:"10" commented:"false" comment:"Worker TTL (minutes)" json:"workerTTL"`
 	// DefaultMemory Worker default memory
 	DefaultMemory int `mapstructure:"defaultMemory" toml:"defaultMemory" default:"1024" commented:"false" comment:"Worker default memory in Mo" json:"defaultMemory"`
-	// KubernetesNamespace is the kubernetes namespace in which workers are spawned"
-	KubernetesNamespace string `mapstructure:"namespace" toml:"namespace" default:"default" commented:"false" comment:"Kubernetes namespace in which workers are spawned" json:"namespace"`
+	// Namespace is the kubernetes namespace in which workers are spawned"
+	Namespace string `mapstructure:"namespace" toml:"namespace" default:"cds" commented:"false" comment:"Kubernetes namespace in which workers are spawned" json:"namespace"`
 	// KubernetesMasterURL Address of kubernetes master
 	KubernetesMasterURL string `mapstructure:"kubernetesMasterURL" toml:"kubernetesMasterURL" default:"https://1.1.1.1:8443" commented:"false" comment:"Address of kubernetes master" json:"kubernetesMasterURL"`
 	// KubernetesConfigFile Kubernetes config file in yaml