From 28fcd9cd259fb4e84d79e098e373351c11e895f9 Mon Sep 17 00:00:00 2001 From: e-minguez Date: Wed, 2 Jul 2025 17:53:41 +0200 Subject: [PATCH] Added some Edge 3.3.1 examples --- .../README.md | 43 +++++++ ...(slmicro-base-image-being-built-with-kiwi) | 0 .../custom/scripts/01-growfs.sh | 15 +++ .../eib.yaml | 33 +++++ .../network/configure-network.sh | 32 +++++ .../edge-331-mgmt-cluster-metal3/README.md | 28 +++++ ...(slmicro-base-image-being-built-with-kiwi) | 0 .../custom/files/basic-setup.sh | 25 ++++ .../custom/files/metal3.sh | 88 +++++++++++++ .../custom/files/mgmt-stack-setup.service | 30 +++++ .../custom/files/rancher.sh | 62 +++++++++ .../custom/scripts/99-alias.sh | 4 + .../custom/scripts/99-mgmt-setup.sh | 12 ++ .../custom/scripts/99-register.sh | 8 ++ .../edge-331-mgmt-cluster-metal3/eib.yaml | 96 ++++++++++++++ .../kubernetes/config/server.yaml | 5 + .../kubernetes/helm/values/certmanager.yaml | 1 + .../kubernetes/helm/values/metal3.yaml | 10 ++ .../kubernetes/helm/values/rancher.yaml | 3 + .../kubernetes/manifests/cluster.yaml | 118 ++++++++++++++++++ .../kubernetes/manifests/downstream-host.yaml | 27 ++++ .../kubernetes/manifests/ingress-ippool.yaml | 13 ++ .../kubernetes/manifests/ingress-l2adv.yaml | 8 ++ .../server/manifests/rke2-cilium-config.yaml | 9 ++ .../server/manifests/rke2-ingress-config.yaml | 17 +++ 25 files changed, 687 insertions(+) create mode 100644 slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/README.md create mode 100644 slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/base-images/REPLACEME(slmicro-base-image-being-built-with-kiwi) create mode 100755 slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/custom/scripts/01-growfs.sh create mode 100644 slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/eib.yaml create mode 100644 slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/network/configure-network.sh create mode 100644 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/README.md create mode 100644 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/base-images/REPLACEME(slmicro-base-image-being-built-with-kiwi) create mode 100755 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/basic-setup.sh create mode 100755 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/metal3.sh create mode 100644 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/mgmt-stack-setup.service create mode 100755 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/rancher.sh create mode 100755 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/scripts/99-alias.sh create mode 100755 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/scripts/99-mgmt-setup.sh create mode 100755 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/scripts/99-register.sh create mode 100644 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/eib.yaml create mode 100644 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/config/server.yaml create mode 100644 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/helm/values/certmanager.yaml create mode 100644 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/helm/values/metal3.yaml create mode 100644 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/helm/values/rancher.yaml create mode 100644 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/manifests/cluster.yaml create mode 100644 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/manifests/downstream-host.yaml create mode 100644 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/manifests/ingress-ippool.yaml create mode 100644 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/manifests/ingress-l2adv.yaml create mode 100644 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/os-files/var/lib/rancher/rke2/server/manifests/rke2-cilium-config.yaml create mode 100644 slemicro/eib-examples/edge-331-mgmt-cluster-metal3/os-files/var/lib/rancher/rke2/server/manifests/rke2-ingress-config.yaml diff --git a/slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/README.md b/slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/README.md new file mode 100644 index 0000000..76ebc6f --- /dev/null +++ b/slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/README.md @@ -0,0 +1,43 @@ +# Example deployment of a SUSE Edge 3.3.1 downstream single-node cluster + +You need to have sushy-emulator running for BMC management via libvirt (see the [create_vm.sh](https://github.com/suse-edge/misc/blob/main/baremetal_vm/create_vm.sh) script for inspiration) + +- Find and replace the "REPLACEME" strings. +- Follow the SUSE Edge documentation on how to [Build an updated SUSE Linux Micro image](https://documentation.suse.com/suse-edge/3.3/html/edge/guides-kiwi-builder-images.html). You can use the "Base" profile. +- Drop the resulting image in the `base-images` folder. +- Create the EIB image as: + +``` +./create_eib.sh -e eib-examples/edge-331-downstream-cluster-single-node-metal3/ -f vm1-downstream +``` + +- Copy the raw image to a webserver and generate the sha256sum: + +``` +cp eib-examples/edge-331-downstream-cluster-single-node-metal3/331-downstream-cluster.raw /path/to/my/webserver/files/ +pushd /path/to/my/webserver/files/ +sha256sum 331-downstream-cluster.raw > 331-downstream-cluster.raw.sha256 +popd +``` + +- Create an empty VM: + +``` +./create_empty_vm.sh -f vm1-downstream -s "40" +``` + +- The VM will be provisioned by the [management cluster](../edge-331-mgmt-cluster-metal3) + +The vm1-downstream file looks like: + +``` +VMFOLDER="/var/lib/libvirt/images/" +VMNAME="vm1-downstream" +CPUS="8" +MEMORY="10240" +MACADDRESS="00:00:00:10:01:01" +LIBVIRT_DISK_SETTINGS="bus=virtio,cache=unsafe" +EIB_IMAGE="registry.suse.com/edge/3.3/edge-image-builder:1.2.1" +``` + +Please adjust to your enviornment according to your needs. diff --git a/slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/base-images/REPLACEME(slmicro-base-image-being-built-with-kiwi) b/slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/base-images/REPLACEME(slmicro-base-image-being-built-with-kiwi) new file mode 100644 index 0000000..e69de29 diff --git a/slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/custom/scripts/01-growfs.sh b/slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/custom/scripts/01-growfs.sh new file mode 100755 index 0000000..6c0a6ea --- /dev/null +++ b/slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/custom/scripts/01-growfs.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Bugzilla - https://bugzilla.suse.com/show_bug.cgi?id=1217430 +growfs() { + mnt="$1" + dev="$(findmnt --fstab --target ${mnt} --evaluate --real --output SOURCE --noheadings)" + # /dev/sda3 -> /dev/sda, /dev/nvme0n1p3 -> /dev/nvme0n1 + parent_dev="/dev/$(lsblk --nodeps -rno PKNAME "${dev}")" + # Last number in the device name: /dev/nvme0n1p42 -> 42 + partnum="$(echo "${dev}" | sed 's/^.*[^0-9]\([0-9]\+\)$/\1/')" + ret=0 + growpart "$parent_dev" "$partnum" || ret=$? + [ $ret -eq 0 ] || [ $ret -eq 1 ] || exit 1 + /usr/lib/systemd/systemd-growfs "$mnt" +} +growfs / diff --git a/slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/eib.yaml b/slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/eib.yaml new file mode 100644 index 0000000..91be22a --- /dev/null +++ b/slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/eib.yaml @@ -0,0 +1,33 @@ +apiVersion: 1.1 +image: + imageType: raw + arch: x86_64 + baseImage: REPLACEME(slmicro-base-image-being-built-with-kiwi) + outputImageName: 331-downstream-cluster.raw +operatingSystem: + time: + timezone: Europe/Madrid + kernelArgs: + - ignition.platform.id=openstack + - net.ifnames=1 + rawConfiguration: + diskSize: 3G + packages: + packageList: + - jq + - qemu-guest-agent + - openssh-server-config-rootlogin + sccRegistrationCode: REPLACEME(scc-registration-code) + systemd: + disable: + - rebootmgr.service + - transactional-update.timer + - transactional-update-cleanup.timer + enable: + - qemu-guest-agent + users: + - username: root + createHomeDir: true + encryptedPassword: REPLACEME(root-encrypted-password) + sshKeys: + - ssh-rsa REPLACEME(sshkey) diff --git a/slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/network/configure-network.sh b/slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/network/configure-network.sh new file mode 100644 index 0000000..a7079f3 --- /dev/null +++ b/slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/network/configure-network.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -eux + +# Attempt to statically configure a nic in the case where we find a network_data.json +# In a configuration drive + +CONFIG_DRIVE=$(blkid --label config-2 || true) +if [ -z "${CONFIG_DRIVE}" ]; then + echo "No config-2 device found, skipping network configuration" + exit 0 +fi + +mount -o ro $CONFIG_DRIVE /mnt + +NETWORK_DATA_FILE="/mnt/openstack/latest/network_data.json" + +if [ ! -f "${NETWORK_DATA_FILE}" ]; then + umount /mnt + echo "No network_data.json found, skipping network configuration" + exit 0 +fi + +# FIXME: we can probably improve this, but there's no jq in the ramdisk +DESIRED_HOSTNAME=$(cat /mnt/openstack/latest/meta_data.json | tr ',{}' '\n' | grep '\"metal3-name\"' | sed 's/.*\"metal3-name\": \"\(.*\)\"/\1/') + +mkdir -p /tmp/nmc/{desired,generated} +cp ${NETWORK_DATA_FILE} /tmp/nmc/desired/${DESIRED_HOSTNAME}.yaml +umount /mnt + +./nmc generate --config-dir /tmp/nmc/desired --output-dir /tmp/nmc/generated +./nmc apply --config-dir /tmp/nmc/generated diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/README.md b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/README.md new file mode 100644 index 0000000..9b5d1df --- /dev/null +++ b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/README.md @@ -0,0 +1,28 @@ +# Example deployment of a SUSE Edge 3.3.1 management cluster + +- Find and replace the "REPLACEME" strings. +- Follow the SUSE Edge documentation on how to [Build an updated SUSE Linux Micro image](https://documentation.suse.com/suse-edge/3.3/html/edge/guides-kiwi-builder-images.html). You can use the "Base" profile. +- Drop the resulting image in the `base-images` folder. +- Create the Management Cluster as: + +``` +./create_eib.sh -e eib-examples/edge-331-mgmt-cluster-metal3/ -f vm1 +for vm in vm1 vm2 vm3 ; do + ./create_vm_with_image.sh -i eib-examples/edge-331-mgmt-cluster-metal3/331-mgmt-cluster.raw -f ${vm} +done +``` + +The vm files look like: + +``` +VMFOLDER="/var/lib/libvirt/images/" +VMNAME="vm1" +CPUS="10" +MEMORY="10240" +# +1 to the latest octet per VM +MACADDRESS="00:00:00:00:01:01" +LIBVIRT_DISK_SETTINGS="bus=virtio,cache=unsafe" +EIB_IMAGE="registry.suse.com/edge/3.3/edge-image-builder:1.2.1" +``` + +Please adjust to your enviornment according to your needs. diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/base-images/REPLACEME(slmicro-base-image-being-built-with-kiwi) b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/base-images/REPLACEME(slmicro-base-image-being-built-with-kiwi) new file mode 100644 index 0000000..e69de29 diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/basic-setup.sh b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/basic-setup.sh new file mode 100755 index 0000000..7207317 --- /dev/null +++ b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/basic-setup.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Pre-requisites. Cluster already running +export KUBECTL="/var/lib/rancher/rke2/bin/kubectl" +export KUBECONFIG="/etc/rancher/rke2/rke2.yaml" + +################## +# METAL3 DETAILS # +################## +export METAL3_CHART_TARGETNAMESPACE="metal3-system" + +########### +# METALLB # +########### +export METALLBNAMESPACE="metallb-system" + +########### +# RANCHER # +########### +export RANCHER_CHART_TARGETNAMESPACE="cattle-system" +export RANCHER_FINALPASSWORD="adminadminadmin" + +die(){ + echo ${1} 1>&2 + exit ${2} +} \ No newline at end of file diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/metal3.sh b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/metal3.sh new file mode 100755 index 0000000..ebc24bf --- /dev/null +++ b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/metal3.sh @@ -0,0 +1,88 @@ +#!/bin/bash +set -euo pipefail + +BASEDIR="$(dirname "$0")" +source ${BASEDIR}/basic-setup.sh + +METAL3LOCKNAMESPACE="default" +METAL3LOCKCMNAME="metal3-lock" + +trap 'catch $? $LINENO' EXIT + +catch() { + if [ "$1" != "0" ]; then + echo "Error $1 occurred on $2" + ${KUBECTL} delete configmap ${METAL3LOCKCMNAME} -n ${METAL3LOCKNAMESPACE} + fi +} + +# Get or create the lock to run all those steps just in a single node +# As the first node is created WAY before the others, this should be enough +# TODO: Investigate if leases is better +if [ $(${KUBECTL} get cm -n ${METAL3LOCKNAMESPACE} ${METAL3LOCKCMNAME} -o name | wc -l) -lt 1 ]; then + ${KUBECTL} create configmap ${METAL3LOCKCMNAME} -n ${METAL3LOCKNAMESPACE} --from-literal foo=bar +else + exit 0 +fi + +# Wait for metal3 +while ! ${KUBECTL} wait --for condition=ready -n ${METAL3_CHART_TARGETNAMESPACE} $(${KUBECTL} get pods -n ${METAL3_CHART_TARGETNAMESPACE} -l app.kubernetes.io/name=metal3-ironic -o name) --timeout=10s; do sleep 2 ; done + +# Get the ironic IP +IRONICIP=$(${KUBECTL} get cm -n ${METAL3_CHART_TARGETNAMESPACE} ironic-bmo -o jsonpath='{.data.IRONIC_IP}') + +# If LoadBalancer, use metallb, else it is NodePort +if [ $(${KUBECTL} get svc -n ${METAL3_CHART_TARGETNAMESPACE} metal3-metal3-ironic -o jsonpath='{.spec.type}') == "LoadBalancer" ]; then + # Wait for metallb + while ! ${KUBECTL} wait --for condition=ready -n ${METALLBNAMESPACE} $(${KUBECTL} get pods -n ${METALLBNAMESPACE} -l app.kubernetes.io/component=controller -o name) --timeout=10s; do sleep 2 ; done + + # Don't create the ippool if already created + ${KUBECTL} get ipaddresspool -n ${METALLBNAMESPACE} ironic-ip-pool -o name || cat <<-EOF | ${KUBECTL} apply -f - + apiVersion: metallb.io/v1beta1 + kind: IPAddressPool + metadata: + name: ironic-ip-pool + namespace: ${METALLBNAMESPACE} + spec: + addresses: + - ${IRONICIP}/32 + serviceAllocation: + priority: 100 + serviceSelectors: + - matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [metal3-ironic]} + EOF + + # Same for L2 Advs + ${KUBECTL} get L2Advertisement -n ${METALLBNAMESPACE} ironic-ip-pool-l2-adv -o name || cat <<-EOF | ${KUBECTL} apply -f - + apiVersion: metallb.io/v1beta1 + kind: L2Advertisement + metadata: + name: ironic-ip-pool-l2-adv + namespace: ${METALLBNAMESPACE} + spec: + ipAddressPools: + - ironic-ip-pool + EOF +fi + +# If rancher is deployed +if [ $(${KUBECTL} get pods -n ${RANCHER_CHART_TARGETNAMESPACE} -l app=rancher -o name | wc -l) -ge 1 ]; then + cat <<-EOF | ${KUBECTL} apply -f - + apiVersion: management.cattle.io/v3 + kind: Feature + metadata: + name: embedded-cluster-api + spec: + value: false + EOF + + # Disable Rancher webhooks for CAPI + ${KUBECTL} delete --ignore-not-found=true mutatingwebhookconfiguration.admissionregistration.k8s.io mutating-webhook-configuration + ${KUBECTL} delete --ignore-not-found=true validatingwebhookconfigurations.admissionregistration.k8s.io validating-webhook-configuration + ${KUBECTL} wait --for=delete namespace/cattle-provisioning-capi-system --timeout=300s +fi + +# Clean up the lock cm + +${KUBECTL} delete configmap ${METAL3LOCKCMNAME} -n ${METAL3LOCKNAMESPACE} diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/mgmt-stack-setup.service b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/mgmt-stack-setup.service new file mode 100644 index 0000000..da92b42 --- /dev/null +++ b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/mgmt-stack-setup.service @@ -0,0 +1,30 @@ +[Unit] +Description=Setup Management stack components +Wants=network-online.target +# It requires rke2 or k3s running, but it won't fail if those services are not present +After=network.target network-online.target rke2-server.service k3s.service +# At least, the basic-setup.sh one needs to be present +ConditionPathExists=/opt/mgmt/bin/basic-setup.sh + +[Service] +User=root +Type=forking +# Metal3 can take A LOT to download the IPA image +TimeoutStartSec=1800 + +ExecStartPre=/bin/sh -c "echo 'Setting up Management components...'" +# Scripts are executed in StartPre because Start can only run a single on +ExecStartPre=/opt/mgmt/bin/rancher.sh +ExecStartPre=/opt/mgmt/bin/metal3.sh +ExecStart=/bin/sh -c "echo 'Finished setting up Management components'" +RemainAfterExit=yes +KillMode=process +# Disable & delete everything +ExecStartPost=rm -f /opt/mgmt/bin/rancher.sh +ExecStartPost=rm -f /opt/mgmt/bin/metal3.sh +ExecStartPost=rm -f /opt/mgmt/bin/basic-setup.sh +ExecStartPost=/bin/sh -c "systemctl disable mgmt-stack-setup.service" +ExecStartPost=rm -f /etc/systemd/system/mgmt-stack-setup.service + +[Install] +WantedBy=multi-user.target diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/rancher.sh b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/rancher.sh new file mode 100755 index 0000000..21a93dc --- /dev/null +++ b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/rancher.sh @@ -0,0 +1,62 @@ +#!/bin/bash +set -euo pipefail + +BASEDIR="$(dirname "$0")" +source ${BASEDIR}/basic-setup.sh + +RANCHERLOCKNAMESPACE="default" +RANCHERLOCKCMNAME="rancher-lock" + +if [ -z "${RANCHER_FINALPASSWORD}" ]; then + # If there is no final password, then finish the setup right away + exit 0 +fi + +trap 'catch $? $LINENO' EXIT + +catch() { + if [ "$1" != "0" ]; then + echo "Error $1 occurred on $2" + ${KUBECTL} delete configmap ${RANCHERLOCKCMNAME} -n ${RANCHERLOCKNAMESPACE} + fi +} + +# Get or create the lock to run all those steps just in a single node +# As the first node is created WAY before the others, this should be enough +# TODO: Investigate if leases is better +if [ $(${KUBECTL} get cm -n ${RANCHERLOCKNAMESPACE} ${RANCHERLOCKCMNAME} -o name | wc -l) -lt 1 ]; then + ${KUBECTL} create configmap ${RANCHERLOCKCMNAME} -n ${RANCHERLOCKNAMESPACE} --from-literal foo=bar +else + exit 0 +fi + +# Wait for rancher to be deployed +while ! ${KUBECTL} wait --for condition=ready -n ${RANCHER_CHART_TARGETNAMESPACE} $(${KUBECTL} get pods -n ${RANCHER_CHART_TARGETNAMESPACE} -l app=rancher -o name) --timeout=10s; do sleep 2 ; done +until ${KUBECTL} get ingress -n ${RANCHER_CHART_TARGETNAMESPACE} rancher > /dev/null 2>&1; do sleep 10; done + +RANCHERBOOTSTRAPPASSWORD=$(${KUBECTL} get secret -n ${RANCHER_CHART_TARGETNAMESPACE} bootstrap-secret -o jsonpath='{.data.bootstrapPassword}' | base64 -d) +RANCHERHOSTNAME=$(${KUBECTL} get ingress -n ${RANCHER_CHART_TARGETNAMESPACE} rancher -o jsonpath='{.spec.rules[0].host}') + +# Skip the whole process if things have been set already +if [ -z $(${KUBECTL} get settings.management.cattle.io first-login -ojsonpath='{.value}') ]; then + # Add the protocol + RANCHERHOSTNAME="https://${RANCHERHOSTNAME}" + TOKEN="" + while [ -z "${TOKEN}" ]; do + # Get token + sleep 2 + TOKEN=$(curl -sk -X POST ${RANCHERHOSTNAME}/v3-public/localProviders/local?action=login -H 'content-type: application/json' -d "{\"username\":\"admin\",\"password\":\"${RANCHERBOOTSTRAPPASSWORD}\"}" | jq -r .token) + done + + # Set password + #curl -sk ${RANCHERHOSTNAME}/v3/users?action=changepassword -H 'content-type: application/json' -H "Authorization: Bearer $TOKEN" -d "{\"currentPassword\":\"${RANCHERBOOTSTRAPPASSWORD}\",\"newPassword\":\"${RANCHER_FINALPASSWORD}\"}" + + # Create a temporary API token (ttl=60 minutes) + APITOKEN=$(curl -sk ${RANCHERHOSTNAME}/v3/token -H 'content-type: application/json' -H "Authorization: Bearer ${TOKEN}" -d '{"type":"token","description":"automation","ttl":3600000}' | jq -r .token) + + curl -sk ${RANCHERHOSTNAME}/v3/settings/server-url -H 'content-type: application/json' -H "Authorization: Bearer ${APITOKEN}" -X PUT -d "{\"name\":\"server-url\",\"value\":\"${RANCHERHOSTNAME}\"}" + curl -sk ${RANCHERHOSTNAME}/v3/settings/telemetry-opt -X PUT -H 'content-type: application/json' -H 'accept: application/json' -H "Authorization: Bearer ${APITOKEN}" -d '{"value":"out"}' +fi + +# Clean up the lock cm +${KUBECTL} delete configmap ${RANCHERLOCKCMNAME} -n ${RANCHERLOCKNAMESPACE} diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/scripts/99-alias.sh b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/scripts/99-alias.sh new file mode 100755 index 0000000..6685995 --- /dev/null +++ b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/scripts/99-alias.sh @@ -0,0 +1,4 @@ +#!/bin/bash +echo "alias k=kubectl" >> /etc/profile.local +echo "alias kubectl=/var/lib/rancher/rke2/bin/kubectl" >> /etc/profile.local +echo "export KUBECONFIG=/etc/rancher/rke2/rke2.yaml" >> /etc/profile.local \ No newline at end of file diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/scripts/99-mgmt-setup.sh b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/scripts/99-mgmt-setup.sh new file mode 100755 index 0000000..9bdc519 --- /dev/null +++ b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/scripts/99-mgmt-setup.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Copy the scripts from combustion to the final location +mkdir -p /opt/mgmt/bin/ +for script in basic-setup.sh rancher.sh metal3.sh; do + cp ${script} /opt/mgmt/bin/ + chmod 744 ${script} +done + +# Copy the systemd unit file and enable it at boot +cp mgmt-stack-setup.service /etc/systemd/system/mgmt-stack-setup.service +systemctl enable mgmt-stack-setup.service diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/scripts/99-register.sh b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/scripts/99-register.sh new file mode 100755 index 0000000..0ab8fca --- /dev/null +++ b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/scripts/99-register.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -euo pipefail + +# Registration https://www.suse.com/support/kb/doc/?id=000018564 +if ! which SUSEConnect > /dev/null 2>&1; then + zypper --non-interactive install suseconnect-ng +fi +SUSEConnect --email "REPLACEME(scc-emai)" --url "https://scc.suse.com" --regcode "REPLACEME(scc-regcode)" diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/eib.yaml b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/eib.yaml new file mode 100644 index 0000000..c8b0a79 --- /dev/null +++ b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/eib.yaml @@ -0,0 +1,96 @@ +apiVersion: 1.1 +image: + imageType: raw + arch: x86_64 + baseImage: REPLACEME(slmicro-base-image-being-built-with-kiwi) + outputImageName: 331-mgmt-cluster.raw +operatingSystem: + rawConfiguration: + diskSize: 50G + packages: + packageList: + - jq + - open-iscsi + - qemu-guest-agent + - openssh-server-config-rootlogin + sccRegistrationCode: REPLACEME(scc-registration-code) + systemd: + disable: + - rebootmgr.service + - transactional-update.timer + - transactional-update-cleanup.timer + enable: + - qemu-guest-agent + users: + - username: root + createHomeDir: true + encryptedPassword: REPLACEME(root-encrypted-password) + sshKeys: + - ssh-rsa REPLACEME(sshkey) +kubernetes: + network: + apiVIP: 192.168.123.10 + apiHost: 192-168-123-10.sslip.io + nodes: + - hostname: vm1 + initializer: true + type: server + - hostname: vm2 + type: server + - hostname: vm3 + type: server + helm: + charts: + - name: metal3 + version: 303.0.7+up0.11.5 + repositoryName: suse-edge-charts + targetNamespace: metal3-system + createNamespace: true + installationNamespace: kube-system + valuesFile: metal3.yaml + - name: rancher-turtles + version: 303.0.4+up0.20.0 + repositoryName: suse-edge-charts + targetNamespace: rancher-turtles-system + createNamespace: true + installationNamespace: kube-system + - createNamespace: true + installationNamespace: kube-system + name: rancher + repositoryName: rancher-prime + targetNamespace: cattle-system + valuesFile: rancher.yaml + version: 2.11.2 + - createNamespace: true + installationNamespace: kube-system + name: cert-manager + repositoryName: jetstack + targetNamespace: cert-manager + valuesFile: certmanager.yaml + version: v1.18.1 + - name: longhorn-crd + version: 106.2.0+up1.8.1 + repositoryName: rancher + targetNamespace: longhorn-system + createNamespace: true + installationNamespace: kube-system + - name: longhorn + version: 106.2.0+up1.8.1 + repositoryName: rancher + targetNamespace: longhorn-system + createNamespace: true + installationNamespace: kube-system + repositories: + - name: rancher-prime + plainHTTP: false + skipTLSVerify: true + url: https://charts.rancher.com/server-charts/prime + - name: jetstack + plainHTTP: false + skipTLSVerify: true + url: https://charts.jetstack.io + - name: suse-edge-charts + url: oci://registry.suse.com/edge/charts + - name: rancher + url: https://charts.rancher.io + version: v1.32.4+rke2r1 diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/config/server.yaml b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/config/server.yaml new file mode 100644 index 0000000..6d57aa2 --- /dev/null +++ b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/config/server.yaml @@ -0,0 +1,5 @@ +cni: + - multus + - cilium +write-kubeconfig-mode: '0644' +selinux: true \ No newline at end of file diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/helm/values/certmanager.yaml b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/helm/values/certmanager.yaml new file mode 100644 index 0000000..1b4551c --- /dev/null +++ b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/helm/values/certmanager.yaml @@ -0,0 +1 @@ +installCRDs: true diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/helm/values/metal3.yaml b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/helm/values/metal3.yaml new file mode 100644 index 0000000..6f4954d --- /dev/null +++ b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/helm/values/metal3.yaml @@ -0,0 +1,10 @@ +global: + ironicIP: REPLACEME(ironic-ip) + enable_vmedia_tls: false + additionalTrustedCAs: false +metal3-ironic: + global: + predictableNicNames: "true" + persistence: + ironic: + size: "5Gi" diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/helm/values/rancher.yaml b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/helm/values/rancher.yaml new file mode 100644 index 0000000..746f233 --- /dev/null +++ b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/helm/values/rancher.yaml @@ -0,0 +1,3 @@ +hostname: rancher-REPLACEME(rancher-ip).sslip.io +bootstrapPassword: "REPLACEME(bootstrappassword)" +replicas: 1 diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/manifests/cluster.yaml b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/manifests/cluster.yaml new file mode 100644 index 0000000..6a122d5 --- /dev/null +++ b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/manifests/cluster.yaml @@ -0,0 +1,118 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: single-node-cluster + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/18 + services: + cidrBlocks: + - 10.96.0.0/12 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: RKE2ControlPlane + name: single-node-cluster + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3Cluster + name: single-node-cluster +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: Metal3Cluster +metadata: + name: single-node-cluster + namespace: default +spec: + controlPlaneEndpoint: + host: 192.168.123.125 + port: 6443 + noCloudProvider: true +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: RKE2ControlPlane +metadata: + name: single-node-cluster + namespace: default +spec: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: Metal3MachineTemplate + name: single-node-cluster-controlplane + replicas: 1 + version: v1.32.4+rke2r1 + rolloutStrategy: + type: "RollingUpdate" + rollingUpdate: + maxSurge: 0 + registrationMethod: "control-plane-endpoint" + serverConfig: + cni: calico + cniMultusEnable: true + agentConfig: + format: ignition + additionalUserData: + config: | + variant: fcos + version: 1.4.0 + systemd: + units: + - name: rke2-preinstall.service + enabled: true + contents: | + [Unit] + Description=rke2-preinstall + Wants=network-online.target + Before=rke2-install.service + ConditionPathExists=!/run/cluster-api/bootstrap-success.complete + [Service] + Type=oneshot + User=root + ExecStartPre=/bin/sh -c "mount -L config-2 /mnt" + ExecStart=/bin/sh -c "sed -i \"s/BAREMETALHOST_UUID/$(jq -r .uuid /mnt/openstack/latest/meta_data.json)/\" /etc/rancher/rke2/config.yaml" + ExecStart=/bin/sh -c "echo \"node-name: $(jq -r .name /mnt/openstack/latest/meta_data.json)\" >> /etc/rancher/rke2/config.yaml" + ExecStartPost=/bin/sh -c "umount /mnt" + [Install] + WantedBy=multi-user.target + kubelet: + extraArgs: + - provider-id=metal3://BAREMETALHOST_UUID + nodeName: "localhost.localdomain" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: Metal3MachineTemplate +metadata: + name: single-node-cluster-controlplane + namespace: default +spec: + template: + spec: + dataTemplate: + name: single-node-cluster-controlplane-template + hostSelector: + matchLabels: + cluster-role: control-plane + image: + checksum: http://REPLACEME(webserver-url)/331-downstream-cluster.raw.sha256 + checksumType: sha256 + format: raw + url: http://REPLACEME(webserver-url)/331-downstream-cluster.raw +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: Metal3DataTemplate +metadata: + name: single-node-cluster-controlplane-template + namespace: default +spec: + clusterName: single-node-cluster + metaData: + objectNames: + - key: name + object: machine + - key: local-hostname + object: machine + - key: local_hostname + object: machine diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/manifests/downstream-host.yaml b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/manifests/downstream-host.yaml new file mode 100644 index 0000000..6584e65 --- /dev/null +++ b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/manifests/downstream-host.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: vm1-downstream-credentials + namespace: default +type: Opaque +data: + username: REPLACEME(bmc-username) + password: REPLACEME(bmc-password) +--- +apiVersion: metal3.io/v1alpha1 +kind: BareMetalHost +metadata: + name: vm1-downstream + namespace: default + labels: + cluster-role: control-plane +spec: + online: true + bootMACAddress: 00:00:00:10:01:01 + bmc: + address: redfish-virtualmedia+https://REPLACEME(sushy-url)/redfish/v1/Systems/vm1-downstream + disableCertificateVerification: true + credentialsName: vm1-downstream-credentials + rootDeviceHints: + deviceName: /dev/vda diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/manifests/ingress-ippool.yaml b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/manifests/ingress-ippool.yaml new file mode 100644 index 0000000..2da6035 --- /dev/null +++ b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/manifests/ingress-ippool.yaml @@ -0,0 +1,13 @@ +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: ingress-ippool + namespace: metallb-system +spec: + addresses: + - 192.168.123.11/32 + serviceAllocation: + priority: 100 + serviceSelectors: + - matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [rke2-ingress-nginx]} diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/manifests/ingress-l2adv.yaml b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/manifests/ingress-l2adv.yaml new file mode 100644 index 0000000..e08c438 --- /dev/null +++ b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/kubernetes/manifests/ingress-l2adv.yaml @@ -0,0 +1,8 @@ +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: ingress-l2-adv + namespace: metallb-system +spec: + ipAddressPools: + - ingress-ippool \ No newline at end of file diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/os-files/var/lib/rancher/rke2/server/manifests/rke2-cilium-config.yaml b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/os-files/var/lib/rancher/rke2/server/manifests/rke2-cilium-config.yaml new file mode 100644 index 0000000..00e4949 --- /dev/null +++ b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/os-files/var/lib/rancher/rke2/server/manifests/rke2-cilium-config.yaml @@ -0,0 +1,9 @@ +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: rke2-cilium + namespace: kube-system +spec: + valuesContent: |- + cni: + exclusive: false \ No newline at end of file diff --git a/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/os-files/var/lib/rancher/rke2/server/manifests/rke2-ingress-config.yaml b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/os-files/var/lib/rancher/rke2/server/manifests/rke2-ingress-config.yaml new file mode 100644 index 0000000..6b69f8f --- /dev/null +++ b/slemicro/eib-examples/edge-331-mgmt-cluster-metal3/os-files/var/lib/rancher/rke2/server/manifests/rke2-ingress-config.yaml @@ -0,0 +1,17 @@ +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: rke2-ingress-nginx + namespace: kube-system +spec: + valuesContent: |- + controller: + config: + use-forwarded-headers: "true" + enable-real-ip: "true" + publishService: + enabled: true + service: + enabled: true + type: LoadBalancer + externalTrafficPolicy: Local \ No newline at end of file