Merge pull request #628 from kube-hetzner/staging

Fix private interface and dns

agents.tf

@@ -75,6 +75,7 @@ resource "null_resource" "agents" {
   # Start the k3s agent and wait for it to have started
   provisioner "remote-exec" {
     inline = [
+      "/etc/cloud/rename_interface.sh",
       "systemctl start k3s-agent 2> /dev/null",
       <<-EOT
       timeout 120 bash <<EOF

control_planes.tf

@@ -136,6 +136,7 @@ resource "null_resource" "control_planes" {
   # Start the k3s server and wait for it to have started correctly
   provisioner "remote-exec" {
     inline = [
+      "/etc/cloud/rename_interface.sh",
       "systemctl start k3s 2> /dev/null",
       <<-EOT
       timeout 120 bash <<EOF

docs/terraform.md

@@ -89,7 +89,7 @@
 | <a name="input_calico_version"></a> [calico\_version](#input\_calico\_version) | Version of Calico. | `string` | `null` | no |
 | <a name="input_cert_manager_values"></a> [cert\_manager\_values](#input\_cert\_manager\_values) | Additional helm values file to pass to Cert-Manager as 'valuesContent' at the HelmChart. | `string` | `""` | no |
 | <a name="input_cilium_values"></a> [cilium\_values](#input\_cilium\_values) | Additional helm values file to pass to Cilium as 'valuesContent' at the HelmChart. | `string` | `""` | no |
-| <a name="input_cluster_autoscaler_image"></a> [cluster\_autoscaler\_image](#input\_cluster\_autoscaler\_image) | Image of Kubernetes Cluster Autoscaler for Hetzner Cloud to be used. | `string` | `"k8s.gcr.io/autoscaling/cluster-autoscaler"` | no |
+| <a name="input_cluster_autoscaler_image"></a> [cluster\_autoscaler\_image](#input\_cluster\_autoscaler\_image) | Image of Kubernetes Cluster Autoscaler for Hetzner Cloud to be used. | `string` | `"registry.k8s.io/autoscaling/cluster-autoscaler"` | no |
 | <a name="input_cluster_autoscaler_version"></a> [cluster\_autoscaler\_version](#input\_cluster\_autoscaler\_version) | Version of Kubernetes Cluster Autoscaler for Hetzner Cloud. Should be aligned with Kubernetes version | `string` | `"v1.25.0"` | no |
 | <a name="input_cluster_ipv4_cidr"></a> [cluster\_ipv4\_cidr](#input\_cluster\_ipv4\_cidr) | Internal Pod CIDR, used for the controller and currently for calico. | `string` | `"10.42.0.0/16"` | no |
 | <a name="input_cluster_name"></a> [cluster\_name](#input\_cluster\_name) | Name of the cluster. | `string` | `"k3s"` | no |

init.tf

@@ -47,6 +47,7 @@ resource "null_resource" "first_control_plane" {
   # Upon reboot start k3s and wait for it to be ready to receive commands
   provisioner "remote-exec" {
     inline = [
+      "/etc/cloud/rename_interface.sh",
       "systemctl start k3s",
       # prepare the post_install directory
       "mkdir -p /var/post_install",

kube.tf.example

@@ -212,7 +212,7 @@ module "kube-hetzner" {
   # * Example below:
   # autoscaler_nodepools = [
   #   {
-  #     name        = "autoscaler"
+  #     name        = "autoscaled-small"
   #     server_type = "cpx21" # must be same or better than the control_plane server type (regarding disk size)!
   #     location    = "fsn1"
   #     min_nodes   = 0
@@ -495,7 +495,7 @@ module "kube-hetzner" {
   /*   cilium_values = <<EOT
 ipam:
   mode: kubernetes
-devices: "enp7s0"
+devices: "eth1"
 k8s:
   requireIPv4PodCIDR: true
 kubeProxyReplacement: strict

locals.tf

@@ -319,7 +319,7 @@ locals {
 
   kubelet_arg                 = ["cloud-provider=external", "volume-plugin-dir=/var/lib/kubelet/volumeplugins"]
   kube_controller_manager_arg = "flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins"
-  flannel_iface               = "enp7s0"
+  flannel_iface               = "eth1"
 
   ingress_controller = var.ingress_controller
 
@@ -343,7 +343,7 @@ ipam:
  operator:
   clusterPoolIPv4PodCIDRList:
    - ${var.cluster_ipv4_cidr}
-devices: "enp7s0"
+devices: "eth1"
 %{if var.enable_wireguard~}
 l7Proxy: false
 encryption:

modules/host/locals.tf

@@ -10,7 +10,7 @@ locals {
   ssh_client_identity = var.ssh_private_key == null ? var.ssh_public_key : var.ssh_private_key
 
   # Final list of packages to install
-  needed_packages = join(" ", concat(["restorecond policycoreutils setools-console"], var.packages_to_install))
+  needed_packages = join(" ", concat(["restorecond policycoreutils setools-console bind-utils"], var.packages_to_install))
 
   # the hosts name with its unique suffix attached
   name = "${var.name}-${random_string.server.id}"

modules/host/main.tf

@@ -111,7 +111,7 @@ resource "hcloud_server" "server" {
 
     inline = [<<-EOT
       set -ex
-      transactional-update shell <<< "zypper --no-gpg-checks --non-interactive install https://github.com/kube-hetzner/terraform-hcloud-kube-hetzner/raw/master/.extra/k3s-selinux-next.rpm"
+      transactional-update shell <<< "zypper --no-gpg-checks --non-interactive install https://github.com/k3s-io/k3s-selinux/releases/download/v1.3.testing.4/k3s-selinux-1.3-4.sle.noarch.rpm"
       transactional-update --continue shell <<< "zypper --gpg-auto-import-keys install -y ${local.needed_packages}"
       sleep 1 && udevadm settle
       EOT

modules/host/templates/userdata.yaml.tpl

@@ -4,6 +4,25 @@ debug: True
 
 write_files:
 
+# Script to rename the private interface to eth1
+- path: /etc/cloud/rename_interface.sh
+  content: |
+    #!/bin/bash
+    set -euo pipefail
+
+    sleep 11
+
+    INTERFACE=$(ip link show | awk '/^3:/{print $2}' | sed 's/://g')
+    MAC=$(cat /sys/class/net/$INTERFACE/address)
+
+    cat <<EOF > /etc/udev/rules.d/70-persistent-net.rules
+    SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="$MAC", NAME="eth1"
+    EOF
+
+    ip link set $INTERFACE down
+    ip link set $INTERFACE name eth1
+    ip link set eth1 up
+
 # Disable ssh password authentication
 - content: |
     Port ${sshPort}
@@ -67,6 +86,21 @@ write_files:
   path: /etc/rancher/k3s/registries.yaml
 %{ endif }
 
+%{ if length(dnsServers) > 0 }
+# Set prepare for manual dns config
+- content: |
+    [main]
+    dns=none
+  path: /etc/NetworkManager/conf.d/dns.conf
+
+- content: |
+    %{ for server in dnsServers ~}
+    nameserver ${server}
+    %{ endfor }
+  path: /etc/resolv.conf
+  permissions: '0644'
+%{ endif }
+
 # Add ssh authorized keys
 ssh_authorized_keys:
 %{ for key in sshAuthorizedKeys ~}
@@ -88,15 +122,6 @@ runcmd:
 - [semodule, '-vi', '/etc/selinux/sshd_t.pp']
 %{ endif }
 
-# As above, make sure the hostname is not reset
-- [sed, '-i', 's/NETCONFIG_NIS_SETDOMAINNAME="yes"/NETCONFIG_NIS_SETDOMAINNAME="no"/g', /etc/sysconfig/network/config]
-- [sed, '-i', 's/DHCLIENT_SET_HOSTNAME="yes"/DHCLIENT_SET_HOSTNAME="no"/g', /etc/sysconfig/network/dhcp]
-
-%{ if length(dnsServers) > 0 }
-# We set the user provided DNS servers, or leave the value empty to default to Hetzners
-- [sed, '-i', 's/NETCONFIG_DNS_STATIC_SERVERS=""/NETCONFIG_DNS_STATIC_SERVERS="${join(" ", dnsServers)}"/g', /etc/sysconfig/network/config]
-%{ endif }
-
 # Bounds the amount of logs that can survive on the system
 - [sed, '-i', 's/#SystemMaxUse=/SystemMaxUse=3G/g', /etc/systemd/journald.conf]
 - [sed, '-i', 's/#MaxRetentionSec=/MaxRetentionSec=1week/g', /etc/systemd/journald.conf]
@@ -105,6 +130,14 @@ runcmd:
 - [sed, '-i', 's/NUMBER_LIMIT="2-10"/NUMBER_LIMIT="4"/g', /etc/snapper/configs/root]
 - [sed, '-i', 's/NUMBER_LIMIT_IMPORTANT="4-10"/NUMBER_LIMIT_IMPORTANT="3"/g', /etc/snapper/configs/root]
 
+%{ if length(dnsServers) > 0 }
+# Set the dns manually
+- [systemctl, 'reload', 'NetworkManager']
+%{ endif }
+
 # Disables unneeded services
 - [systemctl, 'restart', 'sshd']
 - [systemctl, disable, '--now', 'rebootmgr.service']
+
+# make rename_service executable
+- [chmod, '+x', '/etc/cloud/rename_interface.sh']

templates/autoscaler-cloudinit.yaml.tpl

@@ -4,6 +4,21 @@ debug: True
 
 write_files:
 
+# Script to rename the private interface to eth1
+- path: /etc/cloud/rename_interface.sh
+  content: |
+    #!/bin/bash
+    set -xeuo pipefail
+
+    sleep 11
+
+    INTERFACE=$(ip link show | awk '/^3:/{print $2}' | sed 's/://g')
+    MAC=$(cat /sys/class/net/$INTERFACE/address)
+
+    cat <<EOF > /etc/udev/rules.d/70-persistent-net.rules
+    SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="$MAC", NAME="eth1"
+    EOF
+
 # Disable ssh password authentication
 - content: |
     Port ${sshPort}
@@ -57,11 +72,41 @@ write_files:
   encoding: base64
   path: /etc/rancher/k3s/registries.yaml
 
+%{ if length(dnsServers) > 0 }
+# Set prepare for manual dns config
+- content: |
+    [main]
+    dns=none
+  path: /etc/NetworkManager/conf.d/dns.conf
+
+- content: |
+    %{ for server in dnsServers ~}
+    nameserver ${server}
+    %{ endfor }
+  path: /etc/resolv.conf
+  permissions: '0644'
+%{ endif }
+
 - content: |
-    set -ex
+    set -vx
     curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_CHANNEL=${k3s_channel} INSTALL_K3S_EXEC=agent sh -
     /sbin/semodule -v -i /usr/share/selinux/packages/k3s.pp
-  path: /tmp/install-k3s-agent.sh
+    systemctl start k3s-agent
+  path: /var/pre_install/install-k3s-agent.sh
+
+- content: |
+    [Unit]
+    Description=Run install-k3s-agent once at boot time
+    After=network-online.target
+
+    [Service]
+    Type=oneshot
+    ExecStart=/bin/sh /var/pre_install/install-k3s-agent.sh
+
+    [Install]
+    WantedBy=network-online.target
+  permissions: '0644'
+  path: /etc/systemd/system/install-k3s-agent.service
 
 # Add new authorized keys
 ssh_deletekeys: true
@@ -91,15 +136,6 @@ runcmd:
 - [semodule, '-vi', '/etc/selinux/sshd_t.pp']
 %{ endif }
 
-# As above, make sure the hostname is not reset
-- [sed, '-i', 's/NETCONFIG_NIS_SETDOMAINNAME="yes"/NETCONFIG_NIS_SETDOMAINNAME="no"/g', /etc/sysconfig/network/config]
-- [sed, '-i', 's/DHCLIENT_SET_HOSTNAME="yes"/DHCLIENT_SET_HOSTNAME="no"/g', /etc/sysconfig/network/dhcp]
-
-%{ if length(dnsServers) > 0 }
-# We set the user provided DNS servers, or leave the value empty to default to Hetzners
-- [sed, '-i', 's/NETCONFIG_DNS_STATIC_SERVERS=""/NETCONFIG_DNS_STATIC_SERVERS="${join(" ", dnsServers)}"/g', /etc/sysconfig/network/config]
-%{ endif }
-
 # Bounds the amount of logs that can survive on the system
 - [sed, '-i', 's/#SystemMaxUse=/SystemMaxUse=3G/g', /etc/systemd/journald.conf]
 - [sed, '-i', 's/#MaxRetentionSec=/MaxRetentionSec=1week/g', /etc/systemd/journald.conf]
@@ -108,12 +144,20 @@ runcmd:
 - [sed, '-i', 's/NUMBER_LIMIT="2-10"/NUMBER_LIMIT="4"/g', /etc/snapper/configs/root]
 - [sed, '-i', 's/NUMBER_LIMIT_IMPORTANT="4-10"/NUMBER_LIMIT_IMPORTANT="3"/g', /etc/snapper/configs/root]
 
-# Activate changes and disable unneeded services
-- [systemctl, 'restart', 'sshd']
+# Disable unneeded services
 - [systemctl, disable, '--now', 'rebootmgr.service']
 
-# install k3s
-- ['/bin/sh', '/tmp/install-k3s-agent.sh']
+%{ if length(dnsServers) > 0 }
+# Set the dns manually
+- [systemctl, 'reload', 'NetworkManager']
+%{ endif }
+
+# rename network interface
+- [chmod, '+x', '/etc/cloud/rename_interface.sh']
+- ['/etc/cloud/rename_interface.sh']
+
+# Enable install-k3s-agent service
+- [systemctl, enable, 'install-k3s-agent.service']
 
-# start k3s-agent service
-- [systemctl, 'start', 'k3s-agent']
+# Reboot to activate everything
+- [reboot]

variables.tf

@@ -112,7 +112,7 @@ variable "agent_nodepools" {
 
 variable "cluster_autoscaler_image" {
   type        = string
-  default     = "k8s.gcr.io/autoscaling/cluster-autoscaler"
+  default     = "registry.k8s.io/autoscaling/cluster-autoscaler"
   description = "Image of Kubernetes Cluster Autoscaler for Hetzner Cloud to be used."
 }