diff --git a/Makefile b/Makefile index 016e6225..9c7d4373 100644 --- a/Makefile +++ b/Makefile @@ -83,7 +83,9 @@ partition: partition-bake .PHONY: partition-bake partition-bake: docker pull $(MINI_LAB_VM_IMAGE) - +ifeq ($(MINI_LAB_FLAVOR),sonic) + docker pull $(MINI_LAB_SONIC_IMAGE) +endif @if ! sudo $(CONTAINERLAB) --topo $(LAB_TOPOLOGY) inspect | grep -i leaf01 > /dev/null; then \ sudo --preserve-env $(CONTAINERLAB) deploy --topo $(LAB_TOPOLOGY) --reconfigure && \ ./scripts/deactivate_offloading.sh; fi diff --git a/deploy_partition.yaml b/deploy_partition.yaml index c885de9d..7a59227e 100644 --- a/deploy_partition.yaml +++ b/deploy_partition.yaml @@ -16,14 +16,6 @@ tags: sonic - name: Configure Cumulus switches - hosts: leaves:&cumulus - roles: - - name: metal-roles/partition/roles/leaf - tags: leaf - - name: internet - tags: internet - -- name: Deploy docker on Cumulus switches hosts: leaves:&cumulus pre_tasks: # the following task is not required as long as we do not install something from the cumulus repositories, for which all the keys are expired now @@ -38,6 +30,8 @@ - name: install apt-transport shell: dpkg -i /root/https-deps/*.deb && apt-get install -fy roles: + - name: cumulus + tags: cumulus - name: metal-roles/partition/roles/docker-on-cumulus tags: docker-on-cumulus diff --git a/inventories/partition.yaml b/inventories/partition.yaml index 23c0490e..b1734a81 100644 --- a/inventories/partition.yaml +++ b/inventories/partition.yaml @@ -38,3 +38,8 @@ leaves: ansible_python_interpreter: /usr/bin/python ansible_user: root ansible_ssh_private_key_file: "{{ playbook_dir }}/files/ssh/id_rsa" + +cumulus: + vars: + router_enable_static_route_leak: true +sonic: diff --git a/roles/cumulus/defaults/main.yaml b/roles/cumulus/defaults/main.yaml new file mode 100644 index 00000000..8d7b1500 --- /dev/null +++ b/roles/cumulus/defaults/main.yaml @@ -0,0 +1,5 @@ +--- +router_enable_mgmt_vrf: true +router_enable_static_route_leak: false + +router_nameservers: [] diff --git a/roles/cumulus/files/99control_plane_catch_all.rules b/roles/cumulus/files/99control_plane_catch_all.rules new file mode 100644 index 00000000..d469ae8e --- /dev/null +++ b/roles/cumulus/files/99control_plane_catch_all.rules @@ -0,0 +1,36 @@ +# +# Note: These are catch-all rules that shall be last in the over all rule set. +# + +INGRESS_INTF = swp+ + +INGRESS_CHAIN = INPUT + + + +[iptables] + +-A $INGRESS_CHAIN --in-interface $INGRESS_INTF -m addrtype --dst-type LOCAL -j POLICE --set-mode pkt --set-rate 1000 --set-burst 10000 --set-class 2 + +-A $INGRESS_CHAIN --in-interface $INGRESS_INTF -m addrtype --dst-type IPROUTER -j POLICE --set-mode pkt --set-rate 30000 --set-burst 70000 --set-class 2 + +-A $INGRESS_CHAIN --in-interface $INGRESS_INTF -j SETCLASS --class 0 + + +[ip6tables] + +-A $INGRESS_CHAIN --in-interface $INGRESS_INTF -m addrtype --dst-type LOCAL -j POLICE --set-mode pkt --set-rate 1000 --set-burst 1000 --set-class 2 + +-A $INGRESS_CHAIN --in-interface $INGRESS_INTF -m addrtype --dst-type IPROUTER -j POLICE --set-mode pkt --set-rate 400 --set-burst 100 --set-class 2 + +-A $INGRESS_CHAIN --in-interface $INGRESS_INTF -j SETCLASS --class 0 + + +[ebtables] + +-A $INGRESS_CHAIN -p ipv4 --in-interface $INGRESS_INTF -j ACCEPT +-A $INGRESS_CHAIN -p ipv6 --in-interface $INGRESS_INTF -j ACCEPT +-A $INGRESS_CHAIN --in-interface $INGRESS_INTF -j setclass --class 0 +# ipv4 multicast misses +-A $INGRESS_CHAIN -p ipv4 -d 01:00:5e:00:00:00/ff:ff:ff:80:00:00 -j police --set-mode pkt --set-rate 100 --set-burst 100 +-A $INGRESS_CHAIN -j police --set-mode pkt --set-rate 100 --set-burst 100 diff --git a/roles/cumulus/files/bridgemac.json b/roles/cumulus/files/bridgemac.json new file mode 100644 index 00000000..14b83eda --- /dev/null +++ b/roles/cumulus/files/bridgemac.json @@ -0,0 +1,7 @@ +{ + "bridge": { + "module_globals": { + "bridge_mac_iface": ["eth0", "eth1"] + } + } +} diff --git a/roles/cumulus/files/daemons b/roles/cumulus/files/daemons new file mode 100644 index 00000000..c86f9822 --- /dev/null +++ b/roles/cumulus/files/daemons @@ -0,0 +1,2 @@ +bgpd=yes +zebra=yes \ No newline at end of file diff --git a/roles/cumulus/files/frr-validation@.service b/roles/cumulus/files/frr-validation@.service new file mode 100644 index 00000000..d2e9e276 --- /dev/null +++ b/roles/cumulus/files/frr-validation@.service @@ -0,0 +1,10 @@ +[Unit] +Description=Trigger a validation run of a frr configuration file %I + +[Service] +Type=oneshot +ExecStart=/usr/bin/vtysh --dryrun --inputfile %I +StandardOutput=journal + +[Install] +WantedBy=multi-user.target diff --git a/roles/internet/files/frr.tpl b/roles/cumulus/files/frr.tpl similarity index 100% rename from roles/internet/files/frr.tpl rename to roles/cumulus/files/frr.tpl diff --git a/roles/cumulus/files/ifreload.service b/roles/cumulus/files/ifreload.service new file mode 100644 index 00000000..a71205a4 --- /dev/null +++ b/roles/cumulus/files/ifreload.service @@ -0,0 +1,10 @@ +[Unit] +Description=Trigger Interface Reload with ifreload + +[Service] +Type=oneshot +ExecStart=/sbin/ifreload -v -a +StandardOutput=journal + +[Install] +WantedBy=multi-user.target diff --git a/roles/cumulus/files/interfaces-validation@.service b/roles/cumulus/files/interfaces-validation@.service new file mode 100644 index 00000000..9df7795b --- /dev/null +++ b/roles/cumulus/files/interfaces-validation@.service @@ -0,0 +1,10 @@ +[Unit] +Description=Trigger a validation of a network interfaces file %I + +[Service] +Type=oneshot +ExecStart=/sbin/ifup --syntax-check --verbose --all --interfaces %I +StandardOutput=journal + +[Install] +WantedBy=multi-user.target diff --git a/roles/cumulus/files/lldpd.d/portsubtype.conf b/roles/cumulus/files/lldpd.d/portsubtype.conf new file mode 100644 index 00000000..c54ba139 --- /dev/null +++ b/roles/cumulus/files/lldpd.d/portsubtype.conf @@ -0,0 +1,2 @@ +configure lldp portidsubtype macaddress + diff --git a/roles/cumulus/files/lldpd.d/tx-interval.conf b/roles/cumulus/files/lldpd.d/tx-interval.conf new file mode 100644 index 00000000..44c7ec2b --- /dev/null +++ b/roles/cumulus/files/lldpd.d/tx-interval.conf @@ -0,0 +1 @@ +configure lldp tx-interval 10 diff --git a/roles/cumulus/handlers/main.yaml b/roles/cumulus/handlers/main.yaml new file mode 100644 index 00000000..7732c61d --- /dev/null +++ b/roles/cumulus/handlers/main.yaml @@ -0,0 +1,63 @@ +--- +- name: reload interfaces + shell: sleep 3; ifreload -a + async: 1 + poll: 0 + notify: wait for new connection + +- name: wait for new connection + wait_for_connection: + connect_timeout: 20 + sleep: 5 + delay: 5 + timeout: 300 + +- name: reload systemd + systemd: + daemon_reload: yes + +- name: reload sysctl + command: sysctl --system + +- name: restart switchd + service: + name: switchd.service + enabled: true + state: restarted + +- name: reload interfaces + shell: sleep 3; ifreload -a + async: 1 + poll: 0 + notify: wait for new connection + +- name: wait for new connection + wait_for_connection: + connect_timeout: 20 + sleep: 5 + delay: 5 + timeout: 300 + +- name: reload frr + service: + name: frr + enabled: true + state: reloaded + +- name: restart frr + service: + name: frr + enabled: true + state: restarted + +- name: lldpd restart + service: + name: lldpd + enabled: true + state: restarted + +- name: restart ntp@mgmt + service: + name: ntp@mgmt + enabled: true + state: restarted diff --git a/roles/internet/tasks/main.yaml b/roles/cumulus/tasks/internet.yaml similarity index 94% rename from roles/internet/tasks/main.yaml rename to roles/cumulus/tasks/internet.yaml index 9503f15c..0cab4702 100644 --- a/roles/internet/tasks/main.yaml +++ b/roles/cumulus/tasks/internet.yaml @@ -12,7 +12,7 @@ src: internet.intf.j2 dest: /etc/network/interfaces.d/internet.intf -# add the vniInternet inferface to the bridge and configure the new vid at the bridge +# add the vniInternet interface to the bridge and configure the new vid at the bridge # otherwise ifreload will not work - name: add internet vni to bridge command: net add bridge bridge ports vniInternet diff --git a/roles/cumulus/tasks/main.yaml b/roles/cumulus/tasks/main.yaml new file mode 100644 index 00000000..c75a9fd1 --- /dev/null +++ b/roles/cumulus/tasks/main.yaml @@ -0,0 +1,31 @@ +--- +- import_tasks: router.yaml + +- name: flush handlers + meta: flush_handlers + +- name: masquerade for eth0 + iptables: + table: nat + chain: POSTROUTING + out_interface: eth0 + jump: MASQUERADE + +- name: check for static route in mgmt vrf + command: ip r s vrf mgmt + register: route_check + changed_when: false + +- name: ensure that static route for return path to pxe network is present + command: "ip r a 10.0.1.0/24 vrf mgmt via {{ dhcp_server_ip }} dev vlan4000" + when: + - '"10.0.1.0/24" not in route_check.stdout' + - dhcp_server_ip is defined + +- name: create bridgemac.json + copy: + src: bridgemac.json + dest: /etc/network/ifupdown2/policy.d/bridgemac.json + notify: reload interfaces + +- import_tasks: internet.yaml diff --git a/roles/cumulus/tasks/mgmt_vrf.yaml b/roles/cumulus/tasks/mgmt_vrf.yaml new file mode 100644 index 00000000..5451e7bd --- /dev/null +++ b/roles/cumulus/tasks/mgmt_vrf.yaml @@ -0,0 +1,22 @@ +--- +- name: check if mgmt vrf is active + shell: vrf list | grep mgmt + changed_when: false + failed_when: false + register: mgmt_vrf_exists + +- name: activate mgmt vrf; drops connections + nclu: + commands: + - add vrf mgmt + commit: true + async: 1 + poll: 0 + when: mgmt_vrf_exists.rc != 0 + +- name: wait for new connection + wait_for_connection: + connect_timeout: 20 + sleep: 2 + delay: 6 + timeout: 60 diff --git a/roles/cumulus/tasks/router.yaml b/roles/cumulus/tasks/router.yaml new file mode 100644 index 00000000..734a48d6 --- /dev/null +++ b/roles/cumulus/tasks/router.yaml @@ -0,0 +1,85 @@ +--- +- name: configure mgmt vrf + import_tasks: mgmt_vrf.yaml + when: router_enable_mgmt_vrf + +- name: configure switch plane + import_tasks: switch_plane.yaml + when: ports is defined + +- name: flush handlers + meta: flush_handlers + +- name: install services + copy: + src: "{{ item }}" + dest: "/etc/systemd/system/{{ item }}" + notify: reload systemd + with_items: + - frr-validation@.service + - interfaces-validation@.service + - ifreload.service + +- name: copy lldpd configs + copy: + src: lldpd.d/ + dest: /etc/lldpd.d/ + notify: lldpd restart + +- name: check if lldpd has the correct portidsubtype setting + shell: lldpcli show configuration | grep subtype + register: lldpd_subtype_check + changed_when: false + +- name: trigger lldpd restart if portidsubtype setting is wrong + service: + name: lldpd + state: restarted + when: ("macaddress" not in lldpd_subtype_check.stdout) + +- name: populate service facts + service_facts: + +- name: render interfaces configuration + template: + src: interfaces.j2 + dest: /etc/network/interfaces + validate: '/sbin/ifup --syntax-check --all --interfaces %s' + notify: reload interfaces + when: "ansible_facts.services['metal-core.service'] is not defined" + +- name: render custom interfaces configuration section + copy: + content: "{{ custom_interface_section }}" + dest: /etc/network/interfaces.d/99_custom.intf + validate: '/sbin/ifup --syntax-check --all --interfaces %s' + notify: reload interfaces + when: custom_interface_section is defined + +- name: render resolv.conf + template: + src: resolv.conf.j2 + dest: /etc/resolv.conf + notify: reload interfaces + +- name: enable frr daemons + copy: + src: daemons + dest: /etc/frr/daemons + notify: restart frr + +- name: render frr configuration + template: + src: frr.conf.j2 + dest: /etc/frr/frr.conf + validate: '/usr/bin/vtysh --dryrun --inputfile %s' + tags: frr + register: frr_rendered + notify: reload frr + when: "ansible_facts.services['metal-core.service'] is not defined" + +- name: set hostname + nclu: + commands: + - add hostname {{ metal_partition_id }}-{{ inventory_hostname }} + commit: true diff --git a/roles/cumulus/tasks/switch_plane.yaml b/roles/cumulus/tasks/switch_plane.yaml new file mode 100644 index 00000000..6ccb1203 --- /dev/null +++ b/roles/cumulus/tasks/switch_plane.yaml @@ -0,0 +1,14 @@ +--- +- name: render ports.conf + template: + src: ports.conf.j2 + dest: /etc/cumulus/ports.conf + notify: restart switchd + +- name: enable static route leak to apply hardware support + replace: + path: /etc/cumulus/switchd.conf + regexp: '#vrf_route_leak_enable = FALSE' + replace: 'vrf_route_leak_enable = TRUE' + when: router_enable_static_route_leak + notify: restart switchd diff --git a/roles/cumulus/templates/frr.conf.j2 b/roles/cumulus/templates/frr.conf.j2 new file mode 100644 index 00000000..384314af --- /dev/null +++ b/roles/cumulus/templates/frr.conf.j2 @@ -0,0 +1,41 @@ +#jinja2: lstrip_blocks: "True", trim_blocks: "True" +frr version 4.0+cl3u9 +frr defaults datacenter +hostname {{ ansible_hostname }} +username cumulus nopassword +! +service integrated-vtysh-config +! +log syslog informational +! +vrf mgmt + ip route 10.0.1.0/24 {{ ansible_host }} nexthop-vrf default + exit-vrf +! +router bgp {{ asn }} + bgp router-id {{ lo }} + neighbor FABRIC peer-group + neighbor FABRIC remote-as external + {% for iface in uplinks %} + neighbor {{ iface.name }} interface peer-group FABRIC + {% endfor %} + ! + address-family ipv4 unicast + neighbor FABRIC activate + redistribute connected route-map LOOPBACKS + exit-address-family + ! + address-family l2vpn evpn + neighbor FABRIC activate + advertise-all-vni + exit-address-family +! +route-map LOOPBACKS permit 10 + match interface lo +! +{% if metal_partition_mgmt_gateway %} +ip route 0.0.0.0/0 {{ metal_partition_mgmt_gateway }} nexthop-vrf mgmt +! +{% endif %} +line vty +! diff --git a/roles/cumulus/templates/interfaces.j2 b/roles/cumulus/templates/interfaces.j2 new file mode 100644 index 00000000..b65f7590 --- /dev/null +++ b/roles/cumulus/templates/interfaces.j2 @@ -0,0 +1,57 @@ +# This file describes the network interfaces available on your system +# and how to activate them. For more information, see interfaces(5). + +source /etc/network/interfaces.d/*.intf + +# The loopback network interface +auto lo +iface lo inet loopback + address {{ lo }}/32 + +# The primary network interface +auto eth0 +iface eth0 inet dhcp + vrf mgmt + +auto mgmt +iface mgmt + address 127.0.0.1/8 + vrf-table auto + +{% for iface in interfaces %} +auto {{ iface.name }} +iface {{ iface.name }} + mtu {{ mtu.default }} + bridge-access 4000 + +{% endfor %} +{% for iface in uplinks %} +auto {{ iface.name }} +iface {{ iface.name }} + mtu {{ mtu.vxlan }} + +{% endfor %} + +auto bridge +iface bridge + bridge-ports {% for iface in interfaces %}{{ iface.name }} {% endfor %}vni104000 + bridge-vids 4000 + bridge-vlan-aware yes + +auto vlan4000 +iface vlan4000 + mtu {{ mtu.default }} + address {{ metal_core_cidr }} + vlan-id 4000 + vlan-raw-device bridge + +auto vni104000 +iface vni104000 + mtu {{ mtu.default }} + bridge-access 4000 + bridge-learning off + mstpctl-bpduguard yes + mstpctl-portbpdufilter yes + vxlan-id 104000 + vxlan-local-tunnelip {{ lo }} + diff --git a/roles/internet/templates/internet.intf.j2 b/roles/cumulus/templates/internet.intf.j2 similarity index 100% rename from roles/internet/templates/internet.intf.j2 rename to roles/cumulus/templates/internet.intf.j2 diff --git a/roles/cumulus/templates/ports.conf.j2 b/roles/cumulus/templates/ports.conf.j2 new file mode 100644 index 00000000..238f4970 --- /dev/null +++ b/roles/cumulus/templates/ports.conf.j2 @@ -0,0 +1,5 @@ +# ports.conf -- +# = [4x10G|4x25G|2x50G|40G|50G|100G] +{% for key, value in ports|dictsort %} +{{ key }}={{ value }} +{% endfor %} diff --git a/roles/cumulus/templates/resolv.conf.j2 b/roles/cumulus/templates/resolv.conf.j2 new file mode 100644 index 00000000..41c31ff2 --- /dev/null +++ b/roles/cumulus/templates/resolv.conf.j2 @@ -0,0 +1,3 @@ +{% for ns in router_nameservers %} +nameserver {{ ns }} +{% endfor %}