Merge pull request #11 from garutilorenzo/kubeapi_public_access

New feature: expose kubeapi to the internet
garutilorenzo · Aug 19, 2022 · 2be37fd · 2be37fd
2 parents 2cb6942 + 2a50910
commit 2be37fd
Show file tree

Hide file tree

Showing 8 changed files with 122 additions and 12 deletions.
diff --git a/README.md b/README.md
@@ -223,15 +223,16 @@ Once you have created the terraform.tfvars file edit the main.tf file (always in
 | `k3s_worker_pool_size`  | `no`  | Number of k3s workers deployed. Default 2  |
 | `install_nginx_ingress`  | `no`  | Boolean value, install kubernetes nginx ingress controller instead of Traefik. Default: true. For more information see [Nginx ingress controller](#nginx-ingress-controller) |
 nginx_ingress_controller_http_nodeport
-| `nginx_ingress_controller_http_nodeport`  | `30080`  | NodePort where nginx ingress will listen for http traffic  |
-| `nginx_ingress_controller_https_nodeport`  | `30443`  | NodePort where nginx ingress will listen for https traffic  |
+| `nginx_ingress_controller_http_nodeport`  | `no`  | NodePort where nginx ingress will listen for http traffic. Default 30080  |
+| `nginx_ingress_controller_https_nodeport`  | `no`  | NodePort where nginx ingress will listen for https traffic.  Default 30443 |
 | `install_longhorn`  | `no`  | Boolean value, install longhorn "Cloud native distributed block storage for Kubernetes". Default: true  |
 | `longhorn_release`  | `no`  | Longhorn release. Default: v1.2.3  |
 | `install_certmanager`  | `no`  | Boolean value, install [cert manager](https://cert-manager.io/) "Cloud native certificate management". Default: true  |
 | `certmanager_release`  | `no`  | Cert manager release. Default: v1.8.2  |
 | `certmanager_email_address`  | `no`  | Email address used for signing https certificates. Defaul: changeme@example.com  |
 | `unique_tag_key`  | `no`  | Unique tag name used for tagging all the deployed resources. Default: k3s-provisioner |
 | `unique_tag_value`  | `no`  | Unique value used with  unique_tag_key. Default: https://github.com/garutilorenzo/k3s-oci-cluster |
+| `expose_kubeapi`  | `no`  | Boolean value, default false. Expose or not the kubeapi server to the internet. Access is granted only from *my_public_ip_cidr* for security reasons. |
 | `PATH_TO_PUBLIC_KEY`     | `no`       | Path to your public ssh key (Default: "~/.ssh/id_rsa.pub) |
 
 #### Generate random token

diff --git a/data.tf b/data.tf
@@ -15,6 +15,8 @@ data "template_cloudinit_config" "k3s_server_tpl" {
       availability_domain                     = var.availability_domain,
       k3s_url                                 = oci_load_balancer_load_balancer.k3s_load_balancer.ip_address_details[0].ip_address,
       k3s_tls_san                             = oci_load_balancer_load_balancer.k3s_load_balancer.ip_address_details[0].ip_address,
+      expose_kubeapi                          = var.expose_kubeapi,
+      k3s_tls_san_public                      = local.public_lb_ip[0],
       install_longhorn                        = var.install_longhorn,
       longhorn_release                        = var.longhorn_release,
       nginx_ingress_controller_http_nodeport  = var.nginx_ingress_controller_http_nodeport,

diff --git a/files/k3s-install-server.sh b/files/k3s-install-server.sh
@@ -136,17 +136,21 @@ first_last="last"
 disable_traefik="--disable traefik"
 %{ endif }
 
+%{ if expose_kubeapi }
+tls_extra_san="--tls-san ${k3s_tls_san_public}"
+%{ endif }
+
 if [[ "$first_instance" == "$instance_id" ]]; then
   echo "I'm the first yeeee: Cluster init!"
   first_last="first"
-  until (curl -sfL https://get.k3s.io | K3S_TOKEN=${k3s_token} sh -s - --cluster-init $disable_traefik --node-ip $local_ip --advertise-address $local_ip --flannel-iface $flannel_iface --tls-san ${k3s_tls_san}); do
+  until (curl -sfL https://get.k3s.io | K3S_TOKEN=${k3s_token} sh -s - --cluster-init $disable_traefik --node-ip $local_ip --advertise-address $local_ip --flannel-iface $flannel_iface --tls-san ${k3s_tls_san} $tls_extra_san); do
     echo 'k3s did not install correctly'
     sleep 2
   done
 else
   echo ":( Cluster join"
   wait_lb
-  until (curl -sfL https://get.k3s.io | K3S_TOKEN=${k3s_token} sh -s - --server https://${k3s_url}:6443 $disable_traefik --node-ip $local_ip --advertise-address $local_ip --flannel-iface $flannel_iface --tls-san ${k3s_tls_san}); do
+  until (curl -sfL https://get.k3s.io | K3S_TOKEN=${k3s_token} sh -s - --server https://${k3s_url}:6443 $disable_traefik --node-ip $local_ip --advertise-address $local_ip --flannel-iface $flannel_iface --tls-san ${k3s_tls_san} $tls_extra_san); do
     echo 'k3s did not install correctly'
     sleep 2
   done
@@ -182,6 +186,13 @@ if [[ "$first_last" == "first" ]]; then
   kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/${certmanager_release}/cert-manager.yaml
   render_staging_issuer /root/staging_issuer.yaml
   render_prod_issuer /root/prod_issuer.yaml
+
+  # Wait cert-manager to be ready
+  until kubectl get pods -n cert-manager | grep 'Running'; do
+    echo 'Waiting for cert-manager to be ready'
+    sleep 15
+  done
+
   kubectl create -f /root/prod_issuer.yaml
   kubectl create -f /root/staging_issuer.yaml
 fi

diff --git a/k3slb.tf b/k3slb.tf
@@ -80,4 +80,42 @@ resource "oci_network_load_balancer_backend" "k3s_https_backend" {
   name                     = format("%s:%s", data.oci_core_instance_pool_instances.k3s_workers_instances.instances[count.index].id, var.https_lb_port)
   port                     = var.https_lb_port
   target_id                = data.oci_core_instance_pool_instances.k3s_workers_instances.instances[count.index].id
+}
+
+## kube-api
+
+resource "oci_network_load_balancer_listener" "k3s_kubeapi_listener" {
+  count                    = var.expose_kubeapi ? 1 : 0
+  default_backend_set_name = oci_network_load_balancer_backend_set.k3s_kubeapi_backend_set[count.index].name
+  name                     = "k3s_kubeapi_listener"
+  network_load_balancer_id = oci_network_load_balancer_network_load_balancer.k3s_public_lb.id
+  port                     = var.kube_api_port
+  protocol                 = "TCP"
+}
+
+resource "oci_network_load_balancer_backend_set" "k3s_kubeapi_backend_set" {
+  count = var.expose_kubeapi ? 1 : 0
+
+  health_checker {
+    protocol = "TCP"
+    port     = var.kube_api_port
+  }
+
+  name                     = "k3s_kubeapi_backend"
+  network_load_balancer_id = oci_network_load_balancer_network_load_balancer.k3s_public_lb.id
+  policy                   = "FIVE_TUPLE"
+  is_preserve_source       = true
+}
+
+resource "oci_network_load_balancer_backend" "k3s_kubeapi_backend" {
+  depends_on = [
+    oci_core_instance_pool.k3s_servers,
+  ]
+
+  count                    = var.expose_kubeapi ? var.k3s_server_pool_size : 0
+  backend_set_name         = oci_network_load_balancer_backend_set.k3s_kubeapi_backend_set[0].name
+  network_load_balancer_id = oci_network_load_balancer_network_load_balancer.k3s_public_lb.id
+  name                     = format("%s:%s", data.oci_core_instance_pool_instances.k3s_servers_instances.instances[count.index].id, var.kube_api_port)
+  port                     = var.kube_api_port
+  target_id                = data.oci_core_instance_pool_instances.k3s_servers_instances.instances[count.index].id
 }
diff --git a/locals.tf b/locals.tf
@@ -1,3 +1,3 @@
-# locals {
-#   k3s_int_lb_dns_name = format("%s.%s.%s.oraclevcn.com", replace(var.k3s_load_balancer_name, " ", "-"), var.oci_core_subnet_dns_label11, var.oci_core_vcn_dns_label)
-# }
+locals {
+  public_lb_ip = [for interface in oci_network_load_balancer_network_load_balancer.k3s_public_lb.ip_addresses : interface.ip_address if interface.is_public == true]
+}
diff --git a/nsg.tf b/nsg.tf
@@ -48,10 +48,30 @@ resource "oci_core_network_security_group_security_rule" "allow_https_from_all"
   }
 }
 
-resource "oci_core_network_security_group" "lb_to_instances" {
+resource "oci_core_network_security_group_security_rule" "allow_kubeapi_from_all" {
+  count                     = var.expose_kubeapi ? 1 : 0
+  network_security_group_id = oci_core_network_security_group.public_lb_nsg.id
+  direction                 = "INGRESS"
+  protocol                  = 6 # tcp
+
+  description = "Allow HTTPS from all"
+
+  source      = var.my_public_ip_cidr
+  source_type = "CIDR_BLOCK"
+  stateless   = false
+
+  tcp_options {
+    destination_port_range {
+      max = var.kube_api_port
+      min = var.kube_api_port
+    }
+  }
+}
+
+resource "oci_core_network_security_group" "lb_to_instances_http" {
   compartment_id = var.compartment_ocid
   vcn_id         = oci_core_vcn.default_oci_core_vcn.id
-  display_name   = "Public LB to Compute Instances NSG"
+  display_name   = "Public LB to K3s workers Compute Instances NSG"
 
   freeform_tags = {
     "provisioner"           = "terraform"
@@ -61,7 +81,7 @@ resource "oci_core_network_security_group" "lb_to_instances" {
 }
 
 resource "oci_core_network_security_group_security_rule" "nsg_to_instances_http" {
-  network_security_group_id = oci_core_network_security_group.lb_to_instances.id
+  network_security_group_id = oci_core_network_security_group.lb_to_instances_http.id
   direction                 = "INGRESS"
   protocol                  = 6 # tcp
 
@@ -80,7 +100,7 @@ resource "oci_core_network_security_group_security_rule" "nsg_to_instances_http"
 }
 
 resource "oci_core_network_security_group_security_rule" "nsg_to_instances_https" {
-  network_security_group_id = oci_core_network_security_group.lb_to_instances.id
+  network_security_group_id = oci_core_network_security_group.lb_to_instances_http.id
   direction                 = "INGRESS"
   protocol                  = 6 # tcp
 
@@ -96,4 +116,36 @@ resource "oci_core_network_security_group_security_rule" "nsg_to_instances_https
       min = var.https_lb_port
     }
   }
+}
+
+resource "oci_core_network_security_group" "lb_to_instances_kubeapi" {
+  compartment_id = var.compartment_ocid
+  vcn_id         = oci_core_vcn.default_oci_core_vcn.id
+  display_name   = "Public LB to K3s master Compute Instances NSG (kubeapi)"
+
+  freeform_tags = {
+    "provisioner"           = "terraform"
+    "environment"           = "${var.environment}"
+    "${var.unique_tag_key}" = "${var.unique_tag_value}"
+  }
+}
+
+resource "oci_core_network_security_group_security_rule" "nsg_to_instances_kubeapi" {
+  count                     = var.expose_kubeapi ? 1 : 0
+  network_security_group_id = oci_core_network_security_group.lb_to_instances_kubeapi.id
+  direction                 = "INGRESS"
+  protocol                  = 6 # tcp
+
+  description = "Allow kubeapi access from my_public_ip_cidr"
+
+  source      = oci_core_network_security_group.public_lb_nsg.id
+  source_type = "NETWORK_SECURITY_GROUP"
+  stateless   = false
+
+  tcp_options {
+    destination_port_range {
+      max = var.kube_api_port
+      min = var.kube_api_port
+    }
+  }
 }
diff --git a/template.tf b/template.tf
@@ -41,6 +41,7 @@ resource "oci_core_instance_configuration" "k3s_server_template" {
       create_vnic_details {
         assign_public_ip = true
         subnet_id        = oci_core_subnet.default_oci_core_subnet10.id
+        nsg_ids          = [oci_core_network_security_group.lb_to_instances_kubeapi.id]
       }
 
       display_name = "Ubuntu k3s server template"
@@ -106,7 +107,7 @@ resource "oci_core_instance_configuration" "k3s_worker_template" {
       create_vnic_details {
         assign_public_ip = true
         subnet_id        = oci_core_subnet.default_oci_core_subnet10.id
-        nsg_ids          = [oci_core_network_security_group.lb_to_instances.id]
+        nsg_ids          = [oci_core_network_security_group.lb_to_instances_http.id]
       }
 
       display_name = "Ubuntu k3s worker template"

diff --git a/vars.tf b/vars.tf
@@ -178,4 +178,9 @@ variable "install_longhorn" {
 variable "longhorn_release" {
   type    = string
   default = "v1.2.3"
+}
+
+variable "expose_kubeapi" {
+  type    = bool
+  default = false
 }