Add Kubernetes 1.11 support (#256)

* Extend e2e tests to test against all supported Kubernetes versions on all providers
* Use the hyperkube image from `k8s.gcr.io` as the quay.io repo doesn't seem to be maintained

.circleci/config.yml

@@ -1,6 +1,6 @@
 defaults: &defaults
   docker:
-    - image: golang:1.10.1-alpine
+    - image: golang:1.10.3
   working_directory: /go/src/github.com/kubermatic/machine-controller
 
 version: 2
@@ -19,22 +19,22 @@ jobs:
     steps:
       - restore_cache:
           key: repo-{{ .Environment.CIRCLE_SHA1 }}
-      - run: apk update && apk add git && go get -u github.com/golang/dep/cmd/dep
+      - run: go get -u github.com/golang/dep/cmd/dep
       - run: dep status
 
   test:
     <<: *defaults
     steps:
       - restore_cache:
           key: repo-{{ .Environment.CIRCLE_SHA1 }}
-      - run: go test ./...
+      - run: go test -race ./...
 
   build:
     <<: *defaults
     steps:
       - restore_cache:
           key: repo-{{ .Environment.CIRCLE_SHA1 }}
-      - run: apk update && apk add make bash && make machine-controller
+      - run: make machine-controller
       - save_cache:
           key: machine-controller-{{ .Revision }}
           paths:
@@ -47,7 +47,7 @@ jobs:
       - run:
           name: Install dependencies
           command: >-
-              apk update && apk add make bash git openssh-client jq rsync curl &&
+              apt update && apt install -y jq rsync unzip &&
               curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.10.0/bin/linux/amd64/kubectl &&
               chmod +x kubectl &&
               mv kubectl /usr/local/bin
@@ -64,7 +64,7 @@ jobs:
           name:  Run E2E test
           command: >-
               export KUBECONFIG=$GOPATH/src/github.com/kubermatic/machine-controller/.kubeconfig &&
-              go test -tags=e2e -parallel 24 -v -timeout 20m  ./test/e2e/... -identifier=$CIRCLE_BUILD_NUM ||
+              go test -race -tags=e2e -parallel 240 -v -timeout 30m  ./test/e2e/... -identifier=$CIRCLE_BUILD_NUM ||
               (kubectl logs -n kube-system $(kubectl get pods -n kube-system|egrep '^machine-controller'|
               awk '{print $1}'); exit 1)
           no_output_timeout: 25m
@@ -87,7 +87,6 @@ jobs:
       - restore_cache:
           key: machine-controller-{{ .Revision }}
       - setup_remote_docker
-      - run: apk update && apk add make bash git
       - run: |
           set -e
           export GIT_TAG=$CIRCLE_TAG

Makefile

@@ -59,7 +59,8 @@ test-unit-docker:
 			make test-unit
 
 test-unit: vendor
-			go test ./...
+			@#The `-race` flag requires CGO
+			CGO_ENABLED=1 go test -race ./...
 
 e2e-cluster:
 	make -C test/tools/integration apply

README.md

@@ -13,7 +13,6 @@
 
 # Features
 ## What works
-- Kubernetes v1.8.5, v1.9.0, and v1.10.2
 - Creation of worker nodes on AWS, Digitalocean, Openstack, Azure and Hetzner cloud
 - Using Ubuntu, CoreOS/RedHat ContainerLinux or CentOS 7 distributions
 - Using Ubuntu with [CRI-O](https://github.com/kubernetes-incubator/cri-o) container runtime instead of Docker
@@ -94,9 +93,9 @@ locally, the following steps are required:
 Now you can either
 
 * Run the tests for all providers via
-  `go test -tags=e2e -parallel 24 -v -timeout 20m  ./test/e2e/... -identifier $USER`
+  `go test -race -tags=e2e -parallel 240 -v -timeout 30m  ./test/e2e/... -identifier $USER`
 * Check `test/e2e/provisioning/all_e2e_test.go` for the available tests, then run only a specific one via
-  `go test -tags=e2e -parallel 24 -v -timeout 20m  ./test/e2e/... -identifier $USER -run $TESTNAME`
+  `go test -race -tags=e2e -parallel 24 -v -timeout 20m  ./test/e2e/... -identifier $USER -run $TESTNAME`
 
 __Note:__ All e2e tests require corresponding credentials to be present, check
  [`test/e2e/provisioning/all_e2e_test.go`](test/e2e/provisioning/all_e2e_test.go) for details

examples/machine-controller.yaml

@@ -24,6 +24,38 @@ subjects:
   kind: Group
   name: system:bootstrappers:machine-controller:default-node-token
 ---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: machine-controller:kubelet-config-1.11
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resourceNames:
+  - kubelet-config-1.11
+  resources:
+  - configmaps
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: machine-controller:kubelet-config-1.11
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: machine-controller:kubelet-config-1.11
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:nodes
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:bootstrappers:machine-controller:default-node-token
+---
 apiVersion: apps/v1beta2
 kind: Deployment
 metadata:

pkg/containerruntime/crio/supported_versions.go

@@ -22,6 +22,8 @@ func GetOfficiallySupportedVersions(kubernetesVersion string) ([]string, error)
 		return []string{"1.9"}, nil
 	case "1.10":
 		return []string{"1.10"}, nil
+	case "1.11":
+		return []string{"1.10"}, nil
 	default:
 		return nil, errors.NoSupportedVersionsAvailableErr
 	}

pkg/containerruntime/docker/supported_versions.go

@@ -16,7 +16,7 @@ func GetOfficiallySupportedVersions(kubernetesVersion string) ([]string, error)
 
 	majorMinorString := fmt.Sprintf("%d.%d", v.Major(), v.Minor())
 	switch majorMinorString {
-	case "1.8", "1.9", "1.10":
+	case "1.8", "1.9", "1.10", "1.11":
 		return []string{"1.11.2", "1.12.6", "1.13.1", "17.03.2"}, nil
 	default:
 		return nil, errors.NoSupportedVersionsAvailableErr

pkg/controller/machine.go

@@ -22,6 +22,7 @@ import (
 	"net"
 	"regexp"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/golang/glog"
@@ -90,7 +91,8 @@ type Controller struct {
 	metrics            *MetricsCollection
 	kubeconfigProvider KubeconfigProvider
 
-	validationCache map[string]bool
+	validationCache      map[string]bool
+	validationCacheMutex sync.Mutex
 
 	name string
 }
@@ -353,9 +355,16 @@ func (c *Controller) syncHandler(key string) error {
 	if err != nil {
 		return fmt.Errorf("failed to userdata provider for '%s': %v", providerConfig.OperatingSystem, err)
 	}
-	if machine, err = c.defaultContainerRuntime(machine, userdataProvider); err != nil {
-		return fmt.Errorf("failed to default the container runtime version: %v", err)
+
+	// We use a new variable here to be able to put the Event on the machine even thought
+	// c.defaultContainerRuntime returns a nil pointer for the machine in case of an error
+	machineWithDefaultedContainerRuntime, err := c.defaultContainerRuntime(machine, userdataProvider)
+	if err != nil {
+		errorMessage := fmt.Sprintf("failed to default the container runtime version: %v", err)
+		c.recorder.Event(machine, corev1.EventTypeWarning, "ContainerRuntimeDefaultingFailed", errorMessage)
+		return errors.New(errorMessage)
 	}
+	machine = machineWithDefaultedContainerRuntime
 
 	// case 3.2: creates an instance if there is no node associated with the given machine
 	if machine.Status.NodeRef == nil {
@@ -479,7 +488,9 @@ func (c *Controller) ensureInstanceExistsForMachine(prov cloud.Provider, machine
 			return fmt.Errorf("invalid provider config: %v", err)
 		}
 		c.recorder.Event(machine, corev1.EventTypeNormal, "ValidationSucceeded", "Validation succeeded")
+		c.validationCacheMutex.Lock()
 		c.validationCache[cacheKey] = true
+		c.validationCacheMutex.Unlock()
 	} else {
 		glog.V(6).Infof("Skipping validation as the machine was already successfully validated before")
 	}
@@ -489,7 +500,9 @@ func (c *Controller) ensureInstanceExistsForMachine(prov cloud.Provider, machine
 	if err != nil {
 		//First invalidate the validation cache to make sure we run the validation on the next sync.
 		//This might happen in case the user invalidates his provider credentials...
+		c.validationCacheMutex.Lock()
 		c.validationCache[cacheKey] = false
+		c.validationCacheMutex.Unlock()
 
 		// case 2.1: instance was not found and we are going to create one
 		if err == cloudprovidererrors.ErrInstanceNotFound {

pkg/userdata/centos/userdata.go

@@ -238,4 +238,9 @@ packages:
 - nfs-utils
 - bash-completion # Have mercy for the poor operators
 - sudo
+{{- if semverCompare "=1.8.X" .KubeletVersion }}
+# There is a dependency issue in the rpm repo for 1.8, if the cni package is not explicitly
+# specified, installation of the kube packages fails
+- kubernetes-cni-0.5.1-1
+{{- end }}
 `

pkg/userdata/coreos/testdata/docker-1.12.6-auto-update-openstack-kubelet-v-version-prefix.golden

@@ -97,7 +97,7 @@
         "name": "docker.service"
       },
       {
-        "contents": "[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nEnvironment=KUBELET_IMAGE_TAG=v1.9.2_coreos.0\nEnvironment=\"RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \\\n  --volume=resolv,kind=host,source=/etc/resolv.conf \\\n  --mount volume=resolv,target=/etc/resolv.conf \\\n  --volume cni-bin,kind=host,source=/opt/cni/bin \\\n  --mount volume=cni-bin,target=/opt/cni/bin \\\n  --volume cni-conf,kind=host,source=/etc/cni/net.d \\\n  --mount volume=cni-conf,target=/etc/cni/net.d \\\n  --volume etc-kubernetes,kind=host,source=/etc/kubernetes \\\n  --mount volume=etc-kubernetes,target=/etc/kubernetes \\\n  --volume var-log,kind=host,source=/var/log \\\n  --mount volume=var-log,target=/var/log \\\n  --volume var-lib-calico,kind=host,source=/var/lib/calico \\\n  --mount volume=var-lib-calico,target=/var/lib/calico\"\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid\nExecStart=/usr/lib/coreos/kubelet-wrapper \\\n  --container-runtime=docker \\\n  --allow-privileged=true \\\n  --cni-bin-dir=/opt/cni/bin \\\n  --cni-conf-dir=/etc/cni/net.d \\\n  --cluster-dns=10.10.10.10 \\\n  --cluster-domain=cluster.local \\\n  --authentication-token-webhook=true \\\n  --hostname-override=node1 \\\n  --network-plugin=cni \\\n  --cloud-provider=openstack \\\n  --cloud-config=/etc/kubernetes/cloud-config \\\n  --cert-dir=/etc/kubernetes/ \\\n  --pod-manifest-path=/etc/kubernetes/manifests \\\n  --resolv-conf=/etc/resolv.conf \\\n  --rotate-certificates=true \\\n  --kubeconfig=/etc/kubernetes/kubeconfig \\\n  --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \\\n  --lock-file=/var/run/lock/kubelet.lock \\\n  --exit-on-lock-contention \\\n  --read-only-port 0 \\\n  --authorization-mode=Webhook \\\n  --anonymous-auth=false \\\n  --client-ca-file=/etc/kubernetes/ca.crt\nExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n",
+        "contents": "[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nEnvironment=KUBELET_IMAGE=docker://k8s.gcr.io/hyperkube-amd64:v1.9.2\nEnvironment=\"RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \\\n  --insecure-options=image \\\n  --volume=resolv,kind=host,source=/etc/resolv.conf \\\n  --mount volume=resolv,target=/etc/resolv.conf \\\n  --volume cni-bin,kind=host,source=/opt/cni/bin \\\n  --mount volume=cni-bin,target=/opt/cni/bin \\\n  --volume cni-conf,kind=host,source=/etc/cni/net.d \\\n  --mount volume=cni-conf,target=/etc/cni/net.d \\\n  --volume etc-kubernetes,kind=host,source=/etc/kubernetes \\\n  --mount volume=etc-kubernetes,target=/etc/kubernetes \\\n  --volume var-log,kind=host,source=/var/log \\\n  --mount volume=var-log,target=/var/log \\\n  --volume var-lib-calico,kind=host,source=/var/lib/calico \\\n  --mount volume=var-lib-calico,target=/var/lib/calico\"\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid\nExecStart=/usr/lib/coreos/kubelet-wrapper \\\n  --container-runtime=docker \\\n  --allow-privileged=true \\\n  --cni-bin-dir=/opt/cni/bin \\\n  --cni-conf-dir=/etc/cni/net.d \\\n  --cluster-dns=10.10.10.10 \\\n  --cluster-domain=cluster.local \\\n  --authentication-token-webhook=true \\\n  --hostname-override=node1 \\\n  --network-plugin=cni \\\n  --cloud-provider=openstack \\\n  --cloud-config=/etc/kubernetes/cloud-config \\\n  --cert-dir=/etc/kubernetes/ \\\n  --pod-manifest-path=/etc/kubernetes/manifests \\\n  --resolv-conf=/etc/resolv.conf \\\n  --rotate-certificates=true \\\n  --kubeconfig=/etc/kubernetes/kubeconfig \\\n  --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \\\n  --lock-file=/var/run/lock/kubelet.lock \\\n  --exit-on-lock-contention \\\n  --read-only-port 0 \\\n  --authorization-mode=Webhook \\\n  --anonymous-auth=false \\\n  --client-ca-file=/etc/kubernetes/ca.crt\nExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n",
         "dropins": [
           {
             "contents": "[Unit]\nRequires=docker.service\nAfter=docker.service\n",

pkg/userdata/coreos/testdata/docker-1.12.6-auto-update-openstack-multiple-dns.golden

@@ -97,7 +97,7 @@
         "name": "docker.service"
       },
       {
-        "contents": "[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nEnvironment=KUBELET_IMAGE_TAG=v1.9.2_coreos.0\nEnvironment=\"RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \\\n  --volume=resolv,kind=host,source=/etc/resolv.conf \\\n  --mount volume=resolv,target=/etc/resolv.conf \\\n  --volume cni-bin,kind=host,source=/opt/cni/bin \\\n  --mount volume=cni-bin,target=/opt/cni/bin \\\n  --volume cni-conf,kind=host,source=/etc/cni/net.d \\\n  --mount volume=cni-conf,target=/etc/cni/net.d \\\n  --volume etc-kubernetes,kind=host,source=/etc/kubernetes \\\n  --mount volume=etc-kubernetes,target=/etc/kubernetes \\\n  --volume var-log,kind=host,source=/var/log \\\n  --mount volume=var-log,target=/var/log \\\n  --volume var-lib-calico,kind=host,source=/var/lib/calico \\\n  --mount volume=var-lib-calico,target=/var/lib/calico\"\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid\nExecStart=/usr/lib/coreos/kubelet-wrapper \\\n  --container-runtime=docker \\\n  --allow-privileged=true \\\n  --cni-bin-dir=/opt/cni/bin \\\n  --cni-conf-dir=/etc/cni/net.d \\\n  --cluster-dns=10.10.10.10,10.10.10.11,10.10.10.12 \\\n  --cluster-domain=cluster.local \\\n  --authentication-token-webhook=true \\\n  --hostname-override=node1 \\\n  --network-plugin=cni \\\n  --cloud-provider=openstack \\\n  --cloud-config=/etc/kubernetes/cloud-config \\\n  --cert-dir=/etc/kubernetes/ \\\n  --pod-manifest-path=/etc/kubernetes/manifests \\\n  --resolv-conf=/etc/resolv.conf \\\n  --rotate-certificates=true \\\n  --kubeconfig=/etc/kubernetes/kubeconfig \\\n  --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \\\n  --lock-file=/var/run/lock/kubelet.lock \\\n  --exit-on-lock-contention \\\n  --read-only-port 0 \\\n  --authorization-mode=Webhook \\\n  --anonymous-auth=false \\\n  --client-ca-file=/etc/kubernetes/ca.crt\nExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n",
+        "contents": "[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nEnvironment=KUBELET_IMAGE=docker://k8s.gcr.io/hyperkube-amd64:v1.9.2\nEnvironment=\"RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \\\n  --insecure-options=image \\\n  --volume=resolv,kind=host,source=/etc/resolv.conf \\\n  --mount volume=resolv,target=/etc/resolv.conf \\\n  --volume cni-bin,kind=host,source=/opt/cni/bin \\\n  --mount volume=cni-bin,target=/opt/cni/bin \\\n  --volume cni-conf,kind=host,source=/etc/cni/net.d \\\n  --mount volume=cni-conf,target=/etc/cni/net.d \\\n  --volume etc-kubernetes,kind=host,source=/etc/kubernetes \\\n  --mount volume=etc-kubernetes,target=/etc/kubernetes \\\n  --volume var-log,kind=host,source=/var/log \\\n  --mount volume=var-log,target=/var/log \\\n  --volume var-lib-calico,kind=host,source=/var/lib/calico \\\n  --mount volume=var-lib-calico,target=/var/lib/calico\"\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid\nExecStart=/usr/lib/coreos/kubelet-wrapper \\\n  --container-runtime=docker \\\n  --allow-privileged=true \\\n  --cni-bin-dir=/opt/cni/bin \\\n  --cni-conf-dir=/etc/cni/net.d \\\n  --cluster-dns=10.10.10.10,10.10.10.11,10.10.10.12 \\\n  --cluster-domain=cluster.local \\\n  --authentication-token-webhook=true \\\n  --hostname-override=node1 \\\n  --network-plugin=cni \\\n  --cloud-provider=openstack \\\n  --cloud-config=/etc/kubernetes/cloud-config \\\n  --cert-dir=/etc/kubernetes/ \\\n  --pod-manifest-path=/etc/kubernetes/manifests \\\n  --resolv-conf=/etc/resolv.conf \\\n  --rotate-certificates=true \\\n  --kubeconfig=/etc/kubernetes/kubeconfig \\\n  --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \\\n  --lock-file=/var/run/lock/kubelet.lock \\\n  --exit-on-lock-contention \\\n  --read-only-port 0 \\\n  --authorization-mode=Webhook \\\n  --anonymous-auth=false \\\n  --client-ca-file=/etc/kubernetes/ca.crt\nExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n",
         "dropins": [
           {
             "contents": "[Unit]\nRequires=docker.service\nAfter=docker.service\n",