Merge pull request k0sproject#603 from makhov/use-default-api-port-in…

…-container

Always use default 6443 port for kube-apiserver in the container

internal/controller/infrastructure/remote_machine_controller.go

@@ -372,6 +372,9 @@ func (r *RemoteMachineController) getSSHKey(ctx context.Context, rm *infrastruct
 }
 
 func (r *RemoteMachineController) getBootstrapData(ctx context.Context, machine *clusterv1.Machine) ([]byte, error) {
+	if machine.Spec.Bootstrap.DataSecretName == nil {
+		return nil, fmt.Errorf("wait for bootstap secret for the machine: %s", machine.Name)
+	}
 	secret := &v1.Secret{}
 	key := client.ObjectKey{
 		Namespace: machine.Namespace,

internal/controller/k0smotron.io/jointokenrequest_controller.go

@@ -23,6 +23,7 @@ import (
 	"encoding/base64"
 	"fmt"
 	"io"
+	"net/url"
 	"strings"
 	"time"
 
@@ -33,6 +34,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/clientcmd/api"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -113,12 +115,18 @@ func (r *JoinTokenRequestReconciler) Reconcile(ctx context.Context, req ctrl.Req
 		return ctrl.Result{Requeue: true, RequeueAfter: time.Minute}, err
 	}
 
-	if err := r.reconcileSecret(ctx, jtr, token); err != nil {
+	newToken, newKubeconfig, err := ReplaceTokenPort(token, cluster)
+	if err != nil {
+		r.updateStatus(ctx, jtr, "Failed update token URL")
+		return ctrl.Result{Requeue: true, RequeueAfter: time.Minute}, err
+	}
+
+	if err := r.reconcileSecret(ctx, jtr, newToken); err != nil {
 		r.updateStatus(ctx, jtr, "Failed creating secret")
 		return ctrl.Result{Requeue: true, RequeueAfter: time.Minute}, err
 	}
 
-	tokenID, err := getTokenID(token, jtr.Spec.Role)
+	tokenID, err := getTokenID(newKubeconfig, jtr.Spec.Role)
 	if err != nil {
 		r.updateStatus(ctx, jtr, "Failed getting token id")
 		return ctrl.Result{Requeue: true, RequeueAfter: time.Minute}, err
@@ -191,17 +199,46 @@ func (r *JoinTokenRequestReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Complete(r)
 }
 
-func getTokenID(token, role string) (string, error) {
+func replaceKubeconfigPort(in string, cluster km.Cluster) (string, *api.Config, error) {
+	cfg, err := clientcmd.Load([]byte(in))
+	if err != nil {
+		return "", nil, err
+	}
+
+	u, err := url.Parse(cfg.Clusters["k0s"].Server)
+	if err != nil {
+		return "", nil, err
+	}
+	parts := strings.Split(u.Host, ":")
+	u.Host = fmt.Sprintf("%s:%d", parts[0], cluster.Spec.Service.APIPort)
+
+	cfg.Clusters["k0s"].Server = u.String()
+
+	b, err := clientcmd.Write(*cfg)
+	if err != nil {
+		return "", nil, err
+	}
+
+	return string(b), cfg, nil
+}
+
+func ReplaceTokenPort(token string, cluster km.Cluster) (string, *api.Config, error) {
 	b, err := tokenDecode(token)
 	if err != nil {
-		return "", err
+		return "", nil, err
 	}
 
-	cfg, err := clientcmd.Load(b)
+	updatedKubeconfig, cfg, err := replaceKubeconfigPort(string(b), cluster)
 	if err != nil {
-		return "", err
+		return "", nil, err
 	}
 
+	newToken, err := tokenEncode([]byte(updatedKubeconfig))
+
+	return newToken, cfg, err
+}
+
+func getTokenID(cfg *api.Config, role string) (string, error) {
 	var userName string
 	switch role {
 	case "controller":
@@ -233,3 +270,24 @@ func tokenDecode(token string) ([]byte, error) {
 
 	return output, err
 }
+
+func tokenEncode(token []byte) (string, error) {
+	in := bytes.NewReader(token)
+
+	var outBuf bytes.Buffer
+	gz, err := gzip.NewWriterLevel(&outBuf, gzip.BestCompression)
+	if err != nil {
+		return "", err
+	}
+
+	_, err = io.Copy(gz, in)
+	gzErr := gz.Close()
+	if err != nil {
+		return "", err
+	}
+	if gzErr != nil {
+		return "", gzErr
+	}
+
+	return base64.StdEncoding.EncodeToString(outBuf.Bytes()), nil
+}

internal/controller/k0smotron.io/k0smotroncluster_configmap.go

@@ -214,7 +214,7 @@ func getV1Beta1Spec(kmc *km.Cluster, sans []string) map[string]interface{} {
 	v1beta1Spec := map[string]interface{}{
 		"api": map[string]interface{}{
 			"externalAddress": kmc.Spec.ExternalAddress,
-			"port":            kmc.Spec.Service.APIPort,
+			"port":            defaultKubeAPIPort,
 			"sans":            sans,
 		},
 		"konnectivity": map[string]interface{}{

internal/controller/k0smotron.io/k0smotroncluster_controller.go

@@ -36,6 +36,8 @@ import (
 	km "github.com/k0sproject/k0smotron/api/k0smotron.io/v1beta1"
 )
 
+const defaultKubeAPIPort = 6443
+
 var patchOpts []client.PatchOption = []client.PatchOption{
 	client.FieldOwner("k0smotron-operator"),
 	client.ForceOwnership,

internal/controller/k0smotron.io/k0smotroncluster_kubeconfig.go

@@ -42,6 +42,11 @@ func (r *ClusterReconciler) reconcileKubeConfigSecret(ctx context.Context, kmc k
 		return err
 	}
 
+	output, _, err = replaceKubeconfigPort(output, kmc)
+	if err != nil {
+		return err
+	}
+
 	logger.Info("Kubeconfig generated, creating the secret")
 
 	secret := v1.Secret{

internal/controller/k0smotron.io/k0smotroncluster_service.go

@@ -40,8 +40,8 @@ func (r *ClusterReconciler) generateService(kmc *km.Cluster) v1.Service {
 		name = kmc.GetNodePortServiceName()
 		ports = append(ports,
 			v1.ServicePort{
-				Port:       int32(kmc.Spec.Service.APIPort),
-				TargetPort: intstr.FromInt(kmc.Spec.Service.APIPort),
+				Port:       int32(defaultKubeAPIPort),
+				TargetPort: intstr.FromInt(defaultKubeAPIPort),
 				Name:       "api",
 				NodePort:   int32(kmc.Spec.Service.APIPort),
 			},
@@ -57,7 +57,7 @@ func (r *ClusterReconciler) generateService(kmc *km.Cluster) v1.Service {
 		ports = append(ports,
 			v1.ServicePort{
 				Port:       int32(kmc.Spec.Service.APIPort),
-				TargetPort: intstr.FromInt(kmc.Spec.Service.APIPort),
+				TargetPort: intstr.FromInt(defaultKubeAPIPort),
 				Name:       "api",
 			},
 			v1.ServicePort{
@@ -76,7 +76,7 @@ func (r *ClusterReconciler) generateService(kmc *km.Cluster) v1.Service {
 		ports = append(ports,
 			v1.ServicePort{
 				Port:       int32(kmc.Spec.Service.APIPort),
-				TargetPort: intstr.FromInt(kmc.Spec.Service.APIPort),
+				TargetPort: intstr.FromInt(defaultKubeAPIPort),
 				Name:       "api",
 			},
 			v1.ServicePort{

internal/controller/k0smotron.io/k0smotroncluster_statefulset.go

@@ -118,7 +118,7 @@ func (r *ClusterReconciler) generateStatefulSet(kmc *km.Cluster) (apps.StatefulS
 							{
 								Name:          "api",
 								Protocol:      v1.ProtocolTCP,
-								ContainerPort: int32(kmc.Spec.Service.APIPort),
+								ContainerPort: int32(defaultKubeAPIPort),
 							},
 							{
 								Name:          "konnectivity",

inttest/basic/basic_test.go

@@ -91,14 +91,14 @@ func (s *BasicSuite) TestK0sGetsUp() {
 	s.checkClusterStatus(s.Context(), rc)
 
 	s.T().Log("Generating k0smotron join token")
-	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-0", "kmc-test")
+	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-0", "kmc-test", 30443)
 	s.Require().NoError(err)
 
 	s.T().Log("joining worker to k0smotron cluster")
 	s.Require().NoError(s.RunWithToken(s.K0smotronNode(0), token))
 
 	s.T().Log("Starting portforward")
-	fw, err := util.GetPortForwarder(rc, "kmc-kmc-test-0", "kmc-test", 30443)
+	fw, err := util.GetPortForwarder(rc, "kmc-kmc-test-0", "kmc-test", 6443)
 	s.Require().NoError(err)
 
 	go fw.Start(s.Require().NoError)

inttest/capi-docker-machinedeployment/capi_docker_test.go

@@ -90,7 +90,7 @@ func (s *CAPIDockerSuite) TestCAPIDocker() {
 	s.Require().NoError(common.WaitForStatefulSet(s.ctx, s.client, "kmc-docker-md-test", "default"))
 
 	s.T().Log("Starting portforward")
-	fw, err := util.GetPortForwarder(s.restConfig, "kmc-docker-md-test-0", "default", 30443)
+	fw, err := util.GetPortForwarder(s.restConfig, "kmc-docker-md-test-0", "default", 6443)
 	s.Require().NoError(err)
 
 	go fw.Start(s.Require().NoError)

inttest/capi-docker/capi_docker_test.go

@@ -103,7 +103,7 @@ func (s *CAPIDockerSuite) TestCAPIDocker() {
 	s.checkControlPlaneStatus(s.ctx, s.restConfig)
 
 	s.T().Log("Starting portforward")
-	fw, err := util.GetPortForwarder(s.restConfig, "kmc-docker-test-0", "default", 30443)
+	fw, err := util.GetPortForwarder(s.restConfig, "kmc-docker-test-0", "default", 6443)
 	s.Require().NoError(err)
 
 	go fw.Start(s.Require().NoError)

inttest/capi-remote-machine-job-provision/capi_remote_machine_job_provision_test.go

@@ -133,7 +133,7 @@ func (s *RemoteMachineSuite) TestCAPIRemoteMachine() {
 	s.Require().NoError(common.WaitForStatefulSet(ctx, s.client, "kmc-remote-test", "default"))
 
 	s.T().Log("Starting portforward")
-	fw, err := util.GetPortForwarder(s.restConfig, "kmc-remote-test-0", "default", 30443)
+	fw, err := util.GetPortForwarder(s.restConfig, "kmc-remote-test-0", "default", 6443)
 	s.Require().NoError(err)
 
 	go fw.Start(s.Require().NoError)

inttest/capi-remote-machine/capi_remote_machine_test.go

@@ -134,7 +134,7 @@ func (s *RemoteMachineSuite) TestCAPIRemoteMachine() {
 	s.Require().NoError(common.WaitForStatefulSet(ctx, s.client, "kmc-remote-test", "default"))
 
 	s.T().Log("Starting portforward")
-	fw, err := util.GetPortForwarder(s.restConfig, "kmc-remote-test-0", "default", 30443)
+	fw, err := util.GetPortForwarder(s.restConfig, "kmc-remote-test-0", "default", 6443)
 	s.Require().NoError(err)
 
 	go fw.Start(s.Require().NoError)

inttest/config-update-hcp/config_update_test.go

@@ -67,7 +67,7 @@ func (s *ConfigUpdateSuite) TestK0sGetsUp() {
 	s.checkClusterStatus(s.Context(), rc)
 
 	s.T().Log("Starting portforward")
-	fw, err := util.GetPortForwarder(rc, "kmc-kmc-test-0", "kmc-test", 30443)
+	fw, err := util.GetPortForwarder(rc, "kmc-kmc-test-0", "kmc-test", 6443)
 	s.Require().NoError(err)
 
 	go fw.Start(s.Require().NoError)

inttest/ha-controller-etcd/ha_controller_etcd_test.go

@@ -56,7 +56,7 @@ func (s *HAControllerEtcdSuite) TestK0sGetsUp() {
 	s.Require().NoError(common.WaitForStatefulSet(s.Context(), kc, "kmc-kmc-test", "kmc-test"))
 
 	s.T().Log("Generating k0smotron join token")
-	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-0", "kmc-test")
+	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-0", "kmc-test", 30443)
 	s.Require().NoError(err)
 
 	s.T().Log("joining worker to k0smotron cluster")
@@ -65,15 +65,18 @@ func (s *HAControllerEtcdSuite) TestK0sGetsUp() {
 	s.T().Log("Starting portforward")
 	pod := s.getPod(s.Context(), kc)
 
-	fw, err := util.GetPortForwarder(rc, pod.Name, pod.Namespace, 30443)
+	fw, err := util.GetPortForwarder(rc, pod.Name, pod.Namespace, 6443)
 	s.Require().NoError(err)
 	go fw.Start(s.Require().NoError)
 	defer fw.Close()
 
 	<-fw.ReadyChan
 
+	localPort, err := fw.LocalPort()
+	s.Require().NoError(err)
+
 	s.T().Log("waiting for node to be ready")
-	kmcKC, err := util.GetKMCClientSet(s.Context(), kc, "kmc-test", "kmc-test", 30443)
+	kmcKC, err := util.GetKMCClientSet(s.Context(), kc, "kmc-test", "kmc-test", localPort)
 	s.Require().NoError(err)
 	s.Require().NoError(s.WaitForNodeReady(s.K0smotronNode(0), kmcKC))
 

inttest/ha-controller-secret/ha_controller_secret_test.go

@@ -64,7 +64,7 @@ func (s *HAControllerSecretSuite) TestK0sGetsUp() {
 	s.Require().NoError(common.WaitForStatefulSet(s.Context(), kc, "kmc-kmc-test-secret", "kmc-test"))
 
 	s.T().Log("Generating k0smotron join token")
-	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-secret-0", "kmc-test")
+	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-secret-0", "kmc-test", 30443)
 	s.Require().NoError(err)
 
 	s.T().Log("joining worker to k0smotron cluster")
@@ -73,15 +73,18 @@ func (s *HAControllerSecretSuite) TestK0sGetsUp() {
 	s.T().Log("Starting portforward")
 	pod := s.getPod(s.Context(), kc)
 
-	fw, err := util.GetPortForwarder(rc, pod.Name, pod.Namespace, 30443)
+	fw, err := util.GetPortForwarder(rc, pod.Name, pod.Namespace, 6443)
 	s.Require().NoError(err)
 	go fw.Start(s.Require().NoError)
 	defer fw.Close()
 
 	<-fw.ReadyChan
 	s.T().Log("portforward ready")
+	localPort, err := fw.LocalPort()
+	s.Require().NoError(err)
+
 	s.T().Log("getting child clientset")
-	kmcKC, err := util.GetKMCClientSet(s.Context(), kc, "kmc-test-secret", "kmc-test", 30443)
+	kmcKC, err := util.GetKMCClientSet(s.Context(), kc, "kmc-test-secret", "kmc-test", localPort)
 	s.Require().NoError(err)
 	s.T().Log("waiting for node to be ready")
 	s.Require().NoError(s.WaitForNodeReady(s.K0smotronNode(0), kmcKC))

inttest/ha-controller/ha_controller_test.go

@@ -60,7 +60,7 @@ func (s *HAControllerSuite) TestK0sGetsUp() {
 	s.Require().NoError(common.WaitForStatefulSet(s.Context(), kc, "kmc-kmc-test", "kmc-test"))
 
 	s.T().Log("Generating k0smotron join token")
-	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-0", "kmc-test")
+	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-0", "kmc-test", 30443)
 	s.Require().NoError(err)
 
 	s.T().Log("joining worker to k0smotron cluster")
@@ -69,15 +69,18 @@ func (s *HAControllerSuite) TestK0sGetsUp() {
 	s.T().Log("Starting portforward")
 	pod := s.getPod(s.Context(), kc)
 
-	fw, err := util.GetPortForwarder(rc, pod.Name, pod.Namespace, 30443)
+	fw, err := util.GetPortForwarder(rc, pod.Name, pod.Namespace, 6443)
 	s.Require().NoError(err)
 	go fw.Start(s.Require().NoError)
 	defer fw.Close()
 
 	<-fw.ReadyChan
 
+	localPort, err := fw.LocalPort()
+	s.Require().NoError(err)
+
 	s.T().Log("waiting for node to be ready")
-	kmcKC, err := util.GetKMCClientSet(s.Context(), kc, "kmc-test", "kmc-test", 30443)
+	kmcKC, err := util.GetKMCClientSet(s.Context(), kc, "kmc-test", "kmc-test", localPort)
 	s.Require().NoError(err)
 	s.Require().NoError(s.WaitForNodeReady(s.K0smotronNode(0), kmcKC))
 

inttest/hostpath/hostpath_test.go

@@ -63,22 +63,25 @@ func (s *HostPathSuite) TestK0sGetsUp() {
 	s.Require().NoError(common.WaitForStatefulSet(s.Context(), kc, "kmc-kmc-test", "kmc-test"))
 
 	s.T().Log("Generating k0smotron join token")
-	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-0", "kmc-test")
+	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-0", "kmc-test", 30443)
 	s.Require().NoError(err)
 
 	s.T().Log("joining worker to k0smotron cluster")
 	s.Require().NoError(s.RunWithToken(s.K0smotronNode(0), token))
 
 	s.T().Log("Starting portforward")
-	fw, err := util.GetPortForwarder(rc, "kmc-kmc-test-0", "kmc-test", 30443)
+	fw, err := util.GetPortForwarder(rc, "kmc-kmc-test-0", "kmc-test", 6443)
 	s.Require().NoError(err)
 	go fw.Start(s.Require().NoError)
 	defer fw.Close()
 
 	<-fw.ReadyChan
 
+	localPort, err := fw.LocalPort()
+	s.Require().NoError(err)
+
 	s.T().Log("waiting for node to be ready")
-	kmcKC, err := util.GetKMCClientSet(s.Context(), kc, "kmc-test", "kmc-test", 30443)
+	kmcKC, err := util.GetKMCClientSet(s.Context(), kc, "kmc-test", "kmc-test", localPort)
 	s.Require().NoError(err)
 	s.Require().NoError(s.WaitForNodeReady(s.K0smotronNode(0), kmcKC))
 

inttest/monitoring/monitoring_test.go

@@ -61,7 +61,7 @@ func (s *MonitoringSuite) TestK0sGetsUp() {
 	s.Require().NoError(common.WaitForStatefulSet(s.Context(), kc, "kmc-kmc-test", "kmc-test"))
 
 	s.T().Log("Generating k0smotron join token")
-	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-0", "kmc-test")
+	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-0", "kmc-test", 30443)
 	s.Require().NoError(err)
 
 	s.T().Log("joining worker to k0smotron cluster")
@@ -70,7 +70,7 @@ func (s *MonitoringSuite) TestK0sGetsUp() {
 	s.Require().NoError(common.WaitForDeployment(s.Context(), kc, "prometheus-server", "default"))
 
 	s.T().Log("Starting portforward")
-	fw, err := util.GetPortForwarder(rc, "kmc-kmc-test-0", "kmc-test", 30443)
+	fw, err := util.GetPortForwarder(rc, "kmc-kmc-test-0", "kmc-test", 6443)
 	s.Require().NoError(err)
 
 	go fw.Start(s.Require().NoError)

inttest/util/portforward.go

@@ -52,7 +52,7 @@ func GetPortForwarder(cfg *rest.Config, name string, namespace string, port int)
 
 	stopChan := make(chan struct{})
 	readyChan := make(chan struct{})
-	fw, err := portforward.New(dialer, []string{fmt.Sprintf("%d", port)}, stopChan, readyChan, io.Discard, os.Stderr)
+	fw, err := portforward.New(dialer, []string{fmt.Sprintf("42042:%d", port)}, stopChan, readyChan, io.Discard, os.Stderr)
 	if err != nil {
 		return nil, err
 	}