salt,docs,tests: Add procedure to change the Control Plane Ingress IP

Add a simple orchestrate and small procedure in the documentation to
change the Control Plane Ingress IP

Author: TeddyAndrieux

buildchain/buildchain/salt_tree.py

@@ -545,6 +545,7 @@ def _get_parts(self) -> Iterator[str]:
     Path("salt/metalk8s/orchestrate/downgrade/precheck.sls"),
     Path("salt/metalk8s/orchestrate/downgrade/pre.sls"),
     Path("salt/metalk8s/orchestrate/downgrade/post.sls"),
+    Path("salt/metalk8s/orchestrate/update-control-plane-ingress-ip.sls"),
     Path("salt/metalk8s/orchestrate/upgrade/init.sls"),
     Path("salt/metalk8s/orchestrate/upgrade/precheck.sls"),
     Path("salt/metalk8s/orchestrate/upgrade/pre.sls"),

docs/operation/changing_control_plane_ingress_ip.rst

@@ -0,0 +1,44 @@
+Changing the Control Plane Ingress IP
+=====================================
+
+#. On the bootstrap node, update the ``ip`` field from
+   ``networks.controlPlane.ingress`` in the Bootstrap configuration file.
+   (refer to :ref:`Bootstrap Configuration<Bootstrap Configuration>`)
+
+#. Refresh the pillar.
+
+   .. code-block:: console
+
+     $ salt-call saltutil.refresh_pillar wait=True
+
+#. Check that the change is taken into account.
+
+   .. code-block:: console
+
+      $ salt-call metalk8s_network.get_control_plane_ingress_ip
+      local:
+          <my-new-ip>
+
+#. On the bootstrap node, reconfigure ApiServer:
+
+   .. parsed-literal::
+
+     $ salt-call state.sls \\
+         metalk8s.kubernetes.apiserver \\
+         saltenv=metalk8s-|version|
+
+#. Reconfigure Control Plane conponents:
+
+   .. parsed-literal::
+
+      $ kubectl exec -n kube-system -c salt-master \\
+          --kubeconfig=/etc/kubernetes/admin.conf \\
+          $(kubectl --kubeconfig=/etc/kubernetes/admin.conf get pod \\
+          -l "app.kubernetes.io/name=salt-master" \\
+          --namespace=kube-system -o jsonpath='{.items[0].metadata.name}')  \\
+          -- salt-run state.orchestrate \\
+          metalk8s.orchestrate.update-control-plane-ingress-ip \\
+          saltenv=metalk8s-|version|
+
+#. You can :ref:`access the MetalK8s GUI <installation-services-admin-ui>`
+   using this new IP.

docs/operation/index.rst

@@ -19,6 +19,7 @@ do not have a working MetalK8s_ setup.
    disaster_recovery/index
    solutions
    changing_node_hostname
+   changing_control_plane_ingress_ip
    metalk8s-utils
    registry_ha
    listening_processes

salt/metalk8s/orchestrate/update-control-plane-ingress-ip.sls

@@ -0,0 +1,57 @@
+{%- set bootstrap_node = salt.metalk8s.minions_by_role('bootstrap') | first %}
+Check pillar content on {{ bootstrap_node }}:
+  salt.function:
+    - name: metalk8s.check_pillar_keys
+    - tgt: {{ bootstrap_node }}
+    - kwarg:
+        keys:
+          - metalk8s.endpoints.repositories
+        raise_error: False
+    - retry:
+        attempts: 5
+
+Regenerate Control Plane Ingress cert on {{ bootstrap_node }}:
+  salt.state:
+    - tgt: {{ bootstrap_node }}
+    - sls:
+      - metalk8s.addons.nginx-ingress-control-plane.certs
+    - saltenv: {{ saltenv }}
+    - require:
+      - salt: Check pillar content on {{ bootstrap_node }}
+
+Reconfigure Control Plane Ingress:
+  salt.runner:
+    - name: state.orchestrate
+    - mods:
+      - metalk8s.addons.nginx-ingress-control-plane.deployed
+    - saltenv: {{ saltenv }}
+    - require:
+      - salt: Regenerate Control Plane Ingress cert on {{ bootstrap_node }}
+
+Reconfigure Control Plane components:
+  salt.runner:
+  - name: state.orchestrate
+  - mods:
+    - metalk8s.addons.dex.deployed
+    - metalk8s.addons.prometheus-operator.deployed
+    - metalk8s.addons.ui.deployed
+  - saltenv: {{ saltenv }}
+  - require:
+    - salt: Reconfigure Control Plane Ingress
+
+{%- set master_nodes = salt.metalk8s.minions_by_role('master') %}
+{%- for node in master_nodes | sort %}
+
+Reconfigure apiserver on {{ node }}:
+  salt.state:
+    - tgt: {{ node }}
+    - sls:
+      - metalk8s.kubernetes.apiserver
+    - saltenv: {{ saltenv }}
+    - require:
+      - salt: Reconfigure Control Plane components
+    {%- if loop.previtem is defined %}
+      - salt: Reconfigure apiserver on {{ loop.previtem }}
+    {%- endif %}
+
+{%- endfor %}

tests/post/features/ingress.feature

@@ -18,3 +18,13 @@ Feature: Ingress
         And the node control-plane IP is not equal to its workload-plane IP
         When we perform an HTTP request on port 80 on a control-plane IP
         Then the server should not respond
+
+    Scenario: Change Control Plane Ingress IP to node-1 IP
+        Given the Kubernetes API is available
+        And we are on a multi node cluster
+        And pods with label 'app.kubernetes.io/name=ingress-nginx' are 'Ready'
+        When we update control plane ingress IP to node 'node-1' IP
+        And we wait for the rollout of 'daemonset/ingress-nginx-control-plane-controller' in namespace 'metalk8s-ingress' to complete
+        And we wait for the rollout of 'deploy/dex' in namespace 'metalk8s-auth' to complete
+        Then the control plane ingress IP is equal to node 'node-1' IP
+        And we are able to login to Dex as 'admin@metalk8s.invalid' using password 'password'

tests/post/steps/test_ingress.py

@@ -1,8 +1,11 @@
+import json
+import os
 import requests
 import requests.exceptions
 
 import pytest
 from pytest_bdd import given, parsers, scenario, then, when
+import testinfra
 
 from tests import utils
 
@@ -22,11 +25,30 @@ def test_access_http_services_on_control_plane_ip(host):
     pass
 
 
+@scenario("../features/ingress.feature", "Change Control Plane Ingress IP to node-1 IP")
+def test_change_cp_ingress_ip(host, teardown):
+    pass
+
+
 @pytest.fixture(scope="function")
 def context():
     return {}
 
 
+@pytest.fixture
+def teardown(context, host, ssh_config, version):
+    yield
+    if "bootstrap_to_restore" in context:
+        with host.sudo():
+            host.check_output(
+                "cp {} /etc/metalk8s/bootstrap.yaml".format(
+                    context["bootstrap_to_restore"]
+                )
+            )
+
+        re_configure_cp_ingress(host, version, ssh_config)
+
+
 @given("the node control-plane IP is not equal to its workload-plane IP")
 def node_control_plane_ip_is_not_equal_to_its_workload_plane_ip(host):
     data = utils.get_grain(host, "metalk8s")
@@ -67,6 +89,17 @@ def perform_request(host, context, protocol, port, plane):
         context["exception"] = exc
 
 
+@when(parsers.parse("we update control plane ingress IP to node '{node_name}' IP"))
+def update_cp_ingress_ip(host, context, ssh_config, version, node_name):
+    node = testinfra.get_host(node_name, ssh_config=ssh_config)
+    ip = utils.get_grain(node, "metalk8s:control_plane_ip")
+
+    bootstrap_patch = {"networks": {"controlPlane": {"ingress": {"ip": ip}}}}
+
+    patch_bootstrap_config(context, host, bootstrap_patch)
+    re_configure_cp_ingress(host, version, ssh_config)
+
+
 @then(
     parsers.re(r"the server returns (?P<status_code>\d+) '(?P<reason>.+)'"),
     converters=dict(status_code=int),
@@ -82,3 +115,48 @@ def server_returns(host, context, status_code, reason):
 def server_does_not_respond(host, context):
     assert "exception" in context
     assert isinstance(context["exception"], requests.exceptions.ConnectionError)
+
+
+@then(parsers.parse("the control plane ingress IP is equal to node '{node_name}' IP"))
+def check_cp_ingress_node_ip(control_plane_ingress_ip, node_name, ssh_config):
+    node = testinfra.get_host(node_name, ssh_config=ssh_config)
+    ip = utils.get_grain(node, "metalk8s:control_plane_ip")
+
+    assert control_plane_ingress_ip == ip
+
+
+def patch_bootstrap_config(context, host, patch):
+    with host.sudo():
+        cmd_ret = host.check_output("salt-call --out json --local temp.dir")
+
+    tmp_dir = json.loads(cmd_ret)["local"]
+
+    with host.sudo():
+        host.check_output("cp /etc/metalk8s/bootstrap.yaml {}".format(tmp_dir))
+
+    context["bootstrap_to_restore"] = os.path.join(tmp_dir, "bootstrap.yaml")
+
+    with host.sudo():
+        host.check_output(
+            "salt-call --local --retcode-passthrough state.single "
+            "file.serialize /etc/metalk8s/bootstrap.yaml "
+            "dataset='{}' "
+            "merge_if_exists=True".format(json.dumps(patch))
+        )
+
+
+def re_configure_cp_ingress(host, version, ssh_config):
+    with host.sudo():
+        host.check_output(
+            "salt-call --retcode-passthrough state.sls "
+            "metalk8s.kubernetes.apiserver saltenv=metalk8s-{}".format(version)
+        )
+
+    command = [
+        "salt-run",
+        "state.orchestrate",
+        "metalk8s.orchestrate.update-control-plane-ingress-ip",
+        "saltenv=metalk8s-{}".format(version),
+    ]
+
+    utils.run_salt_command(host, command, ssh_config)