Make Control Plane Ingress IP configurable

CHANGELOG.md

@@ -60,6 +60,12 @@
   LVMLogicalVolume volume in the UI
   (PR[#3410](https://github.com/scality/metalk8s/pull/3410))
 
+- [#2381](https://github.com/scality/metalk8s/issues/2381)) - Allow
+  configuring the Control Plane Ingress' external IP, to enable high
+  availability with failover of this (virtual) IP between control plane
+  nodes. This failover is not managed by MetalK8s.
+  (PR[#3415](https://github.com/scality/metalk8s/pull/3415))
+
 ### Breaking changes
 
 - [#2199](https://github.com/scality/metalk8s/issues/2199) - Prometheus label

buildchain/buildchain/salt_tree.py

@@ -427,7 +427,6 @@ def _get_parts(self) -> Iterator[str]:
     Path("salt/metalk8s/addons/nginx-ingress-control-plane/deployed/init.sls"),
     Path("salt/metalk8s/addons/nginx-ingress-control-plane/deployed/chart.sls"),
     Path("salt/metalk8s/addons/nginx-ingress-control-plane/deployed/tls-secret.sls"),
-    Path("salt/metalk8s/addons/nginx-ingress-control-plane/control-plane-ip.sls"),
     Path("salt/metalk8s/beacon/certificates.sls"),
     Path("salt/metalk8s/container-engine/containerd/configured.sls"),
     Path("salt/metalk8s/container-engine/containerd/files/50-metalk8s.conf.j2"),
@@ -546,6 +545,7 @@ def _get_parts(self) -> Iterator[str]:
     Path("salt/metalk8s/orchestrate/downgrade/precheck.sls"),
     Path("salt/metalk8s/orchestrate/downgrade/pre.sls"),
     Path("salt/metalk8s/orchestrate/downgrade/post.sls"),
+    Path("salt/metalk8s/orchestrate/update-control-plane-ingress-ip.sls"),
     Path("salt/metalk8s/orchestrate/upgrade/init.sls"),
     Path("salt/metalk8s/orchestrate/upgrade/precheck.sls"),
     Path("salt/metalk8s/orchestrate/upgrade/pre.sls"),

charts/ingress-nginx-control-plane.yaml

@@ -35,7 +35,7 @@ controller:
     type: ClusterIP
 
     externalIPs:
-    - '{%- endraw -%}{{ grains.metalk8s.control_plane_ip }}{%- raw -%}'
+    - '{%- endraw -%}{{ salt.metalk8s_network.get_control_plane_ingress_ip() }}{%- raw -%}'
 
     enableHttp: false
 

charts/kube-prometheus-stack.yaml

@@ -176,7 +176,7 @@ grafana:
 
   grafana.ini:
     server:
-      root_url: '__escape__(https://{{ "{{ grains.metalk8s.control_plane_ip }}" }}:8443/grafana)'
+      root_url: '__escape__({{ "{{ salt.metalk8s_network.get_control_plane_ingress_endpoint() }}" }}/grafana)'
     analytics:
       reporting_enabled: false
       check_for_updates: false
@@ -188,9 +188,9 @@ grafana:
       scopes: "openid profile email groups"
       client_id: "grafana-ui"
       client_secret: "4lqK98NcsWG5qBRHJUqYM1"
-      auth_url: '__escape__(https://{{ "{{ grains.metalk8s.control_plane_ip }}" }}:8443/oidc/auth)'
-      token_url:  '__escape__(https://{{ "{{ grains.metalk8s.control_plane_ip }}" }}:8443/oidc/token)'
-      api_url: '__escape__(https://{{ "{{ grains.metalk8s.control_plane_ip }}" }}:8443/oidc/userinfo)'
+      auth_url: '__escape__({{ "{{ salt.metalk8s_network.get_control_plane_ingress_endpoint() }}" }}/oidc/auth)'
+      token_url:  '__escape__({{ "{{ salt.metalk8s_network.get_control_plane_ingress_endpoint() }}" }}/oidc/token)'
+      api_url: '__escape__({{ "{{ salt.metalk8s_network.get_control_plane_ingress_endpoint() }}" }}/oidc/userinfo)'
       role_attribute_path: >-
         contains(`{% endraw %}{{ "{{ dex.spec.config.staticPasswords | map(attribute='email') | list | tojson }}" }}{% raw %}`, email) && 'Admin'
 

docs/installation/bootstrap.rst

@@ -47,6 +47,8 @@ Configuration
       networks:
         controlPlane:
           cidr: <CIDR-notation>
+          ingress:
+            ip: <IP-for-ingress>
         workloadPlane:
           cidr: <CIDR-notation>
           mtu: <network-MTU>
@@ -80,6 +82,16 @@ notation for it's various subfields.
         network. This is an :ref:`advanced configuration<multiple CIDR network>`
         which we do not recommend for non-experts.
 
+      For ``controlPlane`` entry, an ``ingress`` can also be provided. This
+      section allow to set the IP that will be used to connect to all the
+      control plane components, like MetalK8s-UI and the whole monitoring
+      stack. We suggest using a
+      `Virtual IP <https://en.wikipedia.org/wiki/Virtual_IP_address>`_ that
+      will sit on a working master Node. The default value for this
+      Ingress IP is the control plane IP of the Bootstrap node (which means
+      that if you lose the Bootstrap node, you no longer have access to any
+      control plane component).
+
       For ``workloadPlane`` entry an
       `MTU <https://en.wikipedia.org/wiki/Maximum_transmission_unit>`_ can
       also be provided, this MTU value should be the lowest MTU value accross

docs/installation/services.rst

@@ -12,13 +12,14 @@ and can be used for operating, extending and upgrading a MetalK8s cluster.
 
 Gather Required Information
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Get the control plane IP of the bootstrap node.
+Get the ingress control plane IP.
 
-.. code-block:: shell
+.. code-block:: console
 
-    root@bootstrap $ salt-call grains.get metalk8s:control_plane_ip
-    local:
-        <the control plane IP>
+    root@bootstrap $ kubectl --kubeconfig=/etc/kubernetes/admin.conf \
+      get svc -n metalk8s-ingress ingress-nginx-control-plane-controller \
+      -o=jsonpath='{.spec.externalIPs[0]}{"\n"}'
+    <the ingress control plane IP>
 
 Use MetalK8s UI
 ^^^^^^^^^^^^^^^

docs/operation/changing_control_plane_ingress_ip.rst

@@ -0,0 +1,44 @@
+Changing the Control Plane Ingress IP
+=====================================
+
+#. On the Bootstrap node, update the ``ip`` field from
+   ``networks.controlPlane.ingress`` in the Bootstrap configuration file.
+   (refer to :ref:`Bootstrap Configuration<Bootstrap Configuration>`)
+
+#. Refresh the pillar.
+
+   .. code-block:: console
+
+     $ salt-call saltutil.refresh_pillar wait=True
+
+#. Check that the change is taken into account.
+
+   .. code-block:: console
+
+      $ salt-call metalk8s_network.get_control_plane_ingress_ip
+      local:
+          <my-new-ip>
+
+#. On the Bootstrap node, reconfigure apiServer:
+
+   .. parsed-literal::
+
+     $ salt-call state.sls \\
+         metalk8s.kubernetes.apiserver \\
+         saltenv=metalk8s-|version|
+
+#. Reconfigure Control Plane components:
+
+   .. parsed-literal::
+
+      $ kubectl exec -n kube-system -c salt-master \\
+          --kubeconfig=/etc/kubernetes/admin.conf \\
+          $(kubectl --kubeconfig=/etc/kubernetes/admin.conf get pod \\
+          -l "app.kubernetes.io/name=salt-master" \\
+          --namespace=kube-system -o jsonpath='{.items[0].metadata.name}')  \\
+          -- salt-run state.orchestrate \\
+          metalk8s.orchestrate.update-control-plane-ingress-ip \\
+          saltenv=metalk8s-|version|
+
+#. You can :ref:`access the MetalK8s GUI <installation-services-admin-ui>`
+   using this new IP.

docs/operation/index.rst

@@ -19,6 +19,7 @@ do not have a working MetalK8s_ setup.
    disaster_recovery/index
    solutions
    changing_node_hostname
+   changing_control_plane_ingress_ip
    metalk8s-utils
    registry_ha
    listening_processes

salt/_modules/metalk8s_network.py

@@ -226,3 +226,34 @@ def routes():
             )
 
     return ret
+
+
+def get_control_plane_ingress_ip():
+    if "ingress" in __pillar__["networks"]["control_plane"]:
+        return __pillar__["networks"]["control_plane"]["ingress"]["ip"]
+
+    # Use Bootstrap Control Plane IP as Ingress Control plane IP
+    bootstrap_id = __salt__["metalk8s.minions_by_role"]("bootstrap")[0]
+
+    if __grains__["id"] == bootstrap_id:
+        return __grains__["metalk8s"]["control_plane_ip"]
+
+    if __opts__.get("__role") == "minion":
+        mine_ret = __salt__["mine.get"](tgt=bootstrap_id, fun="control_plane_ip")
+    else:
+        mine_ret = __salt__["saltutil.runner"](
+            "mine.get", tgt=bootstrap_id, fun="control_plane_ip"
+        )
+
+    if not isinstance(mine_ret, dict) or bootstrap_id not in mine_ret:
+        raise CommandExecutionError(
+            "Unable to get {} Control Plane IP: {}".format(bootstrap_id, mine_ret)
+        )
+
+    return mine_ret[bootstrap_id]
+
+
+def get_control_plane_ingress_endpoint():
+    return "https://{}:8443".format(
+        __salt__["metalk8s_network.get_control_plane_ingress_ip"]()
+    )

salt/metalk8s/addons/dex/config/dex.yaml.j2

@@ -10,6 +10,8 @@
     )
 %}
 
+{%- set control_plane_ingress_ep = salt.metalk8s_network.get_control_plane_ingress_endpoint() %}
+
 
 # Defaults for configuration of Dex (OIDC)
 apiVersion: addons.metalk8s.scality.com/v1alpha2
@@ -21,7 +23,7 @@ spec:
 
   # Dex server configuration
   config:
-    issuer: https://{{ grains.metalk8s.control_plane_ip }}:8443/oidc
+    issuer: {{ control_plane_ingress_ep }}/oidc
 
     storage:
       config:
@@ -65,12 +67,12 @@ spec:
     - id: metalk8s-ui
       name: MetalK8s UI
       redirectURIs:
-      - https://{{ grains.metalk8s.control_plane_ip }}:8443/{{ metalk8s_ui_config.spec.basePath.lstrip('/') }}
+      - {{ control_plane_ingress_ep }}/{{ metalk8s_ui_config.spec.basePath.lstrip('/') }}
       secret: ybrMJpVMQxsiZw26MhJzCjA2ut
     - id: grafana-ui
       name: Grafana UI
       redirectURIs:
-      - https://{{ grains.metalk8s.control_plane_ip }}:8443/grafana/login/generic_oauth
+      - {{ control_plane_ingress_ep }}/grafana/login/generic_oauth
       secret: 4lqK98NcsWG5qBRHJUqYM1
 
     enablePasswordDB: true

salt/metalk8s/addons/nginx-ingress-control-plane/certs/server.sls

@@ -21,7 +21,6 @@ Create Control-Plane Ingress server private key:
     - unless:
       - test -f "{{ private_key_path }}"
 
-{# TODO: add Ingress Service IP once stable (LoadBalancer probably) #}
 {%- set certSANs = [
     grains.fqdn,
     'localhost',
@@ -30,7 +29,7 @@ Create Control-Plane Ingress server private key:
     'nginx-ingress-control-plane.metalk8s-ingress',
     'nginx-ingress-control-plane.metalk8s-ingress.svc',
     'nginx-ingress-control-plane.metalk8s-ingress.svc.cluster.local',
-    grains.metalk8s.control_plane_ip,
+    salt.metalk8s_network.get_control_plane_ingress_ip(),
 ] %}
 
 Generate Control-Plane Ingress server certificate:

salt/metalk8s/addons/nginx-ingress-control-plane/deployed/chart.sls

@@ -283,7 +283,8 @@ metadata:
   namespace: metalk8s-ingress
 spec:
   externalIPs:
-  - '{%- endraw -%}{{ grains.metalk8s.control_plane_ip }}{%- raw -%}'
+  - '{%- endraw -%}{{ salt.metalk8s_network.get_control_plane_ingress_ip() }}{%- raw
+    -%}'
   ports:
   - name: https
     port: 8443

salt/metalk8s/addons/prometheus-operator/deployed/chart.sls

@@ -20826,15 +20826,15 @@ data:
     [auth]
     oauth_auto_login = true
     [auth.generic_oauth]
-    api_url = "{% endraw -%}https://{{ grains.metalk8s.control_plane_ip }}:8443/oidc/userinfo{%- raw %}"
-    auth_url = "{% endraw -%}https://{{ grains.metalk8s.control_plane_ip }}:8443/oidc/auth{%- raw %}"
+    api_url = "{% endraw -%}{{ salt.metalk8s_network.get_control_plane_ingress_endpoint() }}/oidc/userinfo{%- raw %}"
+    auth_url = "{% endraw -%}{{ salt.metalk8s_network.get_control_plane_ingress_endpoint() }}/oidc/auth{%- raw %}"
     client_id = grafana-ui
     client_secret = 4lqK98NcsWG5qBRHJUqYM1
     enabled = true
     role_attribute_path = contains(`{% endraw %}{{ dex.spec.config.staticPasswords | map(attribute='email') | list | tojson }}{% raw %}`, email) && 'Admin'
     scopes = openid profile email groups
     tls_skip_verify_insecure = true
-    token_url = "{% endraw -%}https://{{ grains.metalk8s.control_plane_ip }}:8443/oidc/token{%- raw %}"
+    token_url = "{% endraw -%}{{ salt.metalk8s_network.get_control_plane_ingress_endpoint() }}/oidc/token{%- raw %}"
     [grafana_net]
     url = https://grafana.net
     [log]
@@ -20845,7 +20845,7 @@ data:
     plugins = /var/lib/grafana/plugins
     provisioning = /etc/grafana/provisioning
     [server]
-    root_url = "{% endraw -%}https://{{ grains.metalk8s.control_plane_ip }}:8443/grafana{%- raw %}"
+    root_url = "{% endraw -%}{{ salt.metalk8s_network.get_control_plane_ingress_endpoint() }}/grafana{%- raw %}"
 kind: ConfigMap
 metadata:
   labels:

salt/metalk8s/addons/ui/config/metalk8s-shell-ui-config.yaml.j2

@@ -19,7 +19,7 @@ kind: ShellUIConfig
 spec:
   oidc:
     providerUrl: "/oidc"
-    redirectUrl: "https://{{ grains.metalk8s.control_plane_ip }}:8443/{{ metalk8s_ui_config.spec.basePath.lstrip('/') }}"
+    redirectUrl: "{{ salt.metalk8s_network.get_control_plane_ingress_endpoint() }}/{{ metalk8s_ui_config.spec.basePath.lstrip('/') }}"
     clientId: "metalk8s-ui"
     responseType: "id_token"
     scopes: "openid profile email groups offline_access audience:server:client_id:oidc-auth-client"

salt/metalk8s/addons/ui/deployed/ui-configuration.sls

@@ -55,7 +55,7 @@ metalk8s-ui-config ConfigMap already exist:
   {%- endif %}
 
   {%- set stripped_base_path = metalk8s_ui.spec.basePath.strip('/') %}
-  {%- set cp_ingress_url = "https://" ~ grains.metalk8s.control_plane_ip ~ ":8443" %}
+  {%- set cp_ingress_url = salt.metalk8s_network.get_control_plane_ingress_endpoint() %}
   {%- set metalk8s_ui_url = cp_ingress_url ~ '/' ~ stripped_base_path ~
                             ('/' if stripped_base_path else '') %}
 

salt/metalk8s/addons/ui/deployed/ui.sls.in

@@ -1,11 +1,6 @@
 include:
 - .namespace
 
-{%- from "metalk8s/addons/nginx-ingress-control-plane/control-plane-ip.sls"
-    import ingress_control_plane with context
-%}
-
-
 {%- set metalk8s_ui_defaults = salt.slsutil.renderer(
         'salt://metalk8s/addons/ui/config/metalk8s-ui-config.yaml', saltenv=saltenv
     )

salt/metalk8s/defaults.yaml

@@ -74,9 +74,6 @@ networks:
       control_plane_ip:8080:
         expected: nginx
         description: MetalK8s repository
-      control_plane_ip:8443:
-        expected: kube-proxy
-        description: Control plane nginx ingress
     master:
       0.0.0.0:6443:
         expected: kube-apiserver
@@ -87,6 +84,9 @@ networks:
       127.0.0.1:7443:
         expected: nginx
         description: Apiserver proxy
+      ingress_control_plane_ip:8443:
+        expected: kube-proxy
+        description: Control plane nginx ingress
       control_plane_ip:10257:
         expected: kube-controller-manager
         description: Kubernetes controller manager

salt/metalk8s/kubernetes/apiserver/installed.sls

@@ -2,9 +2,6 @@
 {%- from "metalk8s/map.jinja" import certificates with context %}
 {%- from "metalk8s/map.jinja" import metalk8s with context %}
 {%- from "metalk8s/map.jinja" import networks with context %}
-{%- from "metalk8s/addons/nginx-ingress-control-plane/control-plane-ip.sls"
-    import ingress_control_plane with context
-%}
 
 {%- set encryption_k8s_path = "/etc/kubernetes/encryption.conf" %}
 
@@ -98,7 +95,7 @@ Create kube-apiserver Pod manifest:
         # }
           - --encryption-provider-config={{ encryption_k8s_path }}
           - --cors-allowed-origins=.*
-          - --oidc-issuer-url=https://{{ ingress_control_plane }}/oidc
+          - --oidc-issuer-url={{ salt.metalk8s_network.get_control_plane_ingress_endpoint() }}/oidc
           - --oidc-client-id=oidc-auth-client
           - --oidc-ca-file=/etc/metalk8s/pki/nginx-ingress/ca.crt
           - --oidc-username-claim=email

salt/metalk8s/orchestrate/bootstrap/init.sls

@@ -132,6 +132,8 @@ Update pillar on bootstrap minion after highstate:
   salt.function:
   - name: saltutil.refresh_pillar
   - tgt: {{ pillar.bootstrap_id }}
+  - kwarg:
+      wait: true
   - require:
     - salt: Configure bootstrap Node object