chore: add end-to-end test #41 (#43)

molecule/end-to-end/converge.yml

@@ -0,0 +1,79 @@
+---
+
+- name: Configure host
+  hosts: localhost
+  connection: local
+  become: yes
+  pre_tasks:
+    - name: Set proper nf_conntrack_max
+      ansible.posix.sysctl:
+        name: net.netfilter.nf_conntrack_max
+        value: '393216'
+        sysctl_set: true
+        state: present
+        reload: true
+
+- name: Configure k8s instances
+  hosts:
+    - control_plane
+    - workers
+  gather_facts: false
+  tasks:
+    - name: Install Python
+      ansible.builtin.raw: apt update && apt install -y python3-pip python3 python3-apt
+      changed_when: false
+    - name: Install prerequisites
+      ansible.builtin.apt:
+        name:
+          - kmod  # modprobe and lsmod
+        update_cache: true
+
+- name: Pre configure all hosts
+  hosts: all
+  gather_facts: false
+  tasks:
+    - name: Install prerequisites
+      ansible.builtin.apt:
+        name:
+          - iproute2  # provides network facts
+          - net-tools
+          - curl
+        update_cache: true
+    - name: Collect facts
+      ansible.builtin.setup:
+
+- name: Configure certs
+  hosts:
+    - etcd
+    - etcd_clients
+  gather_facts: true
+  become_method: su
+  roles:
+    - { role: andrewrothstein.cfssl }
+    - { role: cloudlabsinfra.etcd_cluster_certificates }
+
+- name: Configure etcd
+  hosts: etcd
+  become: yes
+  gather_facts: true
+  roles:
+    - { role: andrewrothstein.etcd }
+    - { role: cloudlabsinfra.etcd_cluster }
+
+- name: Configure load balancers
+  hosts: lb
+  become: yes
+  become_method: su
+  gather_facts: true
+  roles:
+    - { role: geerlingguy.haproxy }
+
+- name: Converge
+  hosts:
+    - control_plane
+    - workers
+  become: yes
+  become_method: su
+  gather_facts: false
+  roles:
+    - { role: cloudlabsinfra.k8s_cluster }

molecule/end-to-end/molecule.yml

@@ -0,0 +1,239 @@
+---
+
+dependency:
+  name: galaxy
+driver:
+  name: docker
+platforms:
+  # etcd
+  - &default_platform_common
+    name: etcd-instance-01
+    hostname: etcd-instance-01
+    image: mpaivabarbosa/molecule-systemd-ubuntu:20.04
+    groups:
+      - etcd
+    command: /sbin/init
+    security_opts:
+      - seccomp=unconfined
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    privileged: true
+    pre_build_image: true
+    override_command: false
+    keep_volumes: false
+    # https://github.com/ansible-community/molecule-docker/blob/main/src/molecule_docker/driver.py
+    docker_networks:
+      - name: k8s_cluster
+        ipam_config:
+          - subnet: '172.18.0.0/24'
+            gateway: '172.18.0.254'
+    networks:
+      - name: k8s_cluster
+  - <<: *default_platform_common
+    name: etcd-instance-02
+    hostname: etcd-instance-02
+  # load balancers
+  - <<: *default_platform_common
+    name: lb-etcd
+    hostname: lb-etcd
+    groups:
+      - lb
+    networks:
+      - name: k8s_cluster
+        ipv4_address: "172.18.0.100"
+  - <<: *default_platform_common
+    name: lb-master
+    hostname: lb-master
+    groups:
+      - lb
+    networks:
+      - name: k8s_cluster
+        ipv4_address: "172.18.0.200"
+  # --------
+  # k8s
+  - &default_platform
+    name: k8s-control-plane-01
+    hostname: k8s-control-plane-01
+    image: kindest/node:v1.26.3
+    groups:
+      - control_plane
+      - etcd_clients
+    volumes:
+      - /lib/modules:/lib/modules:ro
+      - /var/lib/containerd
+    privileged: true
+    pre_build_image: true
+    keep_volumes: false
+    sysctls:
+      net.bridge.bridge-nf-call-iptables: 1
+      net.bridge.bridge-nf-call-ip6tables: 1
+      net.ipv4.ip_forward: 1
+    docker_networks:
+      - name: k8s_cluster
+        ipam_config:
+          - subnet: '172.18.0.0/24'
+            gateway: '172.18.0.254'
+    networks:
+      - name: k8s_cluster
+    etc_hosts:
+      'etcd.cloudlabsinfra.local': '172.18.0.100'  # dns name of etcd load balancer
+      'control-plane.cloudlabsinfra.local': '172.18.0.200'  # dns name of control-plane load balancer
+  - <<: *default_platform
+    name: k8s-control-plane-02
+    hostname: k8s-control-plane-02
+  - <<: *default_platform
+    name: k8s-worker-01
+    hostname: k8s-worker-01
+    groups:
+      - workers
+provisioner:
+  name: ansible
+  inventory:
+    host_vars:
+      k8s-control-plane-01:
+        # cloudlabsinfra.k8s_cluster role related variables
+        k8s_cluster_initial_master: true
+        k8s_cluster_kubernetes_version: 1.26.0
+        # we can't use default cluster configuration here because it doesn't have 'etcd' section
+        k8s_cluster_cluster_configuration:
+          etcd:
+            external:
+              endpoints: ["https://{{ etcd_frontend_name }}:2379"]
+              caFile: "/etc/ssl/private/ca.pem"
+              certFile: "/etc/ssl/private/client.pem"
+              keyFile: "/etc/ssl/private/client-key.pem"
+          networking:
+            serviceSubnet: 10.96.0.0/12
+            podSubnet: 10.244.0.0/16
+            dnsDomain: cluster.local
+          kubernetesVersion: "{{ k8s_cluster_kubernetes_version }}"
+          controlPlaneEndpoint: "{{ k8s_cluster_control_plane_endpoint }}:6443"
+      lb-etcd:
+        haproxy_frontend_name: 'etcd'
+        haproxy_frontend_bind_address: '*'
+        haproxy_frontend_port: 2379
+        haproxy_frontend_mode: 'tcp'
+        haproxy_backend_name: 'etcd'
+        haproxy_backend_mode: 'tcp'
+        haproxy_backend_balance_method: 'roundrobin'
+        haproxy_backend_httpchk: ''
+        haproxy_backend_servers:
+          - name: etcd-instance-01
+            address: "{{ hostvars['etcd-instance-01']['ansible_facts']['default_ipv4']['address'] }}:2379"
+          - name: etcd-instance-02
+            address: "{{ hostvars['etcd-instance-02']['ansible_facts']['default_ipv4']['address'] }}:2379"
+      lb-master:
+        haproxy_frontend_name: 'master'
+        haproxy_frontend_bind_address: '*'
+        haproxy_frontend_port: 6443
+        haproxy_frontend_mode: 'tcp'
+        haproxy_backend_name: 'master'
+        haproxy_backend_mode: 'tcp'
+        haproxy_backend_balance_method: 'roundrobin'
+        haproxy_backend_httpchk: ''
+        haproxy_backend_servers:
+          - name: k8s-control-plane-01
+            address: "{{ hostvars['k8s-control-plane-01']['ansible_facts']['default_ipv4']['address'] }}:6443"
+          - name: k8s-control-plane-02
+            address: "{{ hostvars['k8s-control-plane-02']['ansible_facts']['default_ipv4']['address'] }}:6443"
+    group_vars:
+      all:
+        # required for control-plane nodes and etcd as well
+        etcd_frontend_name: "etcd.cloudlabsinfra.local"
+        k8s_cluster_control_plane_endpoint: "control-plane.cloudlabsinfra.local"
+      control_plane:
+        # default value of variable below is 'worker'
+        k8s_cluster_node_type: "master"
+        # we can't use default join configuration here because it doesn't have 'controlPlane' section
+        k8s_cluster_join_configuration:
+          nodeRegistration:
+            ignorePreflightErrors:
+              - SystemVerification
+          discovery:
+            bootstrapToken:
+              token: "{{ k8s_cluster_join_token }}"
+              apiServerEndpoint: "{{ k8s_cluster_control_plane_endpoint }}:6443"
+              caCertHashes:
+                - "sha256:{{ k8s_cluster_root_ca_hash }}"
+              unsafeSkipCAVerification: false
+          controlPlane:
+            localAPIEndpoint:
+              advertiseAddress: "{{ hostvars[inventory_hostname]['ansible_facts']['default_ipv4']['address'] }}"
+              bindPort: 6443
+      etcd_clients:
+        etcd_cert_matrix:
+          - profile_name: client
+            output_name: client
+            csr:
+              CN: client
+              hosts: []
+              key:
+                algo: ecdsa
+                size: 256
+              names:
+                - C: RU
+                  L: Moscow
+                  O: Organization
+                  OU: Organizational Unit
+                  ST: Moscow region
+      etcd:
+        # cloudlabsinfra.etcd_cluster_certificates role related variables
+        etcd_cert_dir: /etc/ssl/private
+        # cloudlabsinfra.etcd_cluster role related variables
+        # client/server
+        etcd_trusted_ca_file: "{{ etcd_conf_dir }}/ca.pem"
+        etcd_key_file: "{{ etcd_conf_dir }}/server-key.pem"
+        etcd_cert_file: "{{ etcd_conf_dir }}/server.pem"
+        etcd_client_cert_auth: 'true'
+        # peer
+        etcd_peer_trusted_ca_file: "{{ etcd_trusted_ca_file }}"
+        etcd_peer_key_file: "{{ etcd_conf_dir }}/peer-key.pem"
+        etcd_peer_cert_file: "{{ etcd_conf_dir }}/peer.pem"
+        etcd_peer_client_cert_auth: 'true'
+        etcd_remote_cert_files:
+          - "{{ etcd_cert_dir }}/ca.pem"
+          - "{{ etcd_cert_dir }}/server-key.pem"
+          - "{{ etcd_cert_dir }}/server.pem"
+          - "{{ etcd_cert_dir }}/peer-key.pem"
+          - "{{ etcd_cert_dir }}/peer.pem"
+        etcd_cert_matrix:
+          - profile_name: server
+            output_name: server
+            csr: &default_csr
+              CN: "{{ inventory_hostname }}"
+              hosts:
+                - "{{ inventory_hostname }}"
+                - "{{ ansible_default_ipv4.address }}"
+                - "{{ etcd_frontend_name }}"
+              key:
+                algo: ecdsa
+                size: 256
+              names:
+                - C: RU
+                  L: Moscow
+                  O: Organization
+                  OU: Organizational Unit
+                  ST: Moscow region
+          - profile_name: peer
+            output_name: peer
+            csr:
+              <<: *default_csr
+          - profile_name: client
+            output_name: client
+            csr:
+              <<: *default_csr
+              CN: client
+              hosts: []
+scenario:
+  test_sequence:
+    - dependency
+    - cleanup
+    - destroy
+    - syntax
+    - create
+    - prepare
+    - converge
+    - side_effect
+    - verify
+    - cleanup
+    - destroy

molecule/end-to-end/verify.yml

@@ -0,0 +1,9 @@
+---
+
+- name: Verify control plane
+  hosts: control_plane
+  gather_facts: false
+  tasks:
+    - name: Include verify common
+      ansible.builtin.include_tasks:
+        file: ../verify-common.yml

requirements.yml

@@ -6,6 +6,9 @@
 - name: geerlingguy.containerd
   version: 1.3.1
 
+- name: geerlingguy.haproxy
+  version: 1.3.1
+
 - name: andrewrothstein.etcd
   version: v2.3.1