Added some Edge 3.3.1 examples

slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/README.md

@@ -0,0 +1,43 @@
+# Example deployment of a SUSE Edge 3.3.1 downstream single-node cluster
+
+You need to have sushy-emulator running for BMC management via libvirt (see the [create_vm.sh](https://github.com/suse-edge/misc/blob/main/baremetal_vm/create_vm.sh) script for inspiration)
+
+- Find and replace the "REPLACEME" strings.
+- Follow the SUSE Edge documentation on how to [Build an updated SUSE Linux Micro image](https://documentation.suse.com/suse-edge/3.3/html/edge/guides-kiwi-builder-images.html). You can use the "Base" profile.
+- Drop the resulting image in the `base-images` folder.
+- Create the EIB image as:
+
+```
+./create_eib.sh -e eib-examples/edge-331-downstream-cluster-single-node-metal3/ -f vm1-downstream
+```
+
+- Copy the raw image to a webserver and generate the sha256sum:
+
+```
+cp eib-examples/edge-331-downstream-cluster-single-node-metal3/331-downstream-cluster.raw /path/to/my/webserver/files/
+pushd /path/to/my/webserver/files/
+sha256sum 331-downstream-cluster.raw > 331-downstream-cluster.raw.sha256
+popd
+```
+
+- Create an empty VM:
+
+```
+./create_empty_vm.sh -f vm1-downstream -s "40"
+```
+
+- The VM will be provisioned by the [management cluster](../edge-331-mgmt-cluster-metal3)
+
+The vm1-downstream file looks like:
+
+```
+VMFOLDER="/var/lib/libvirt/images/"
+VMNAME="vm1-downstream"
+CPUS="8"
+MEMORY="10240"
+MACADDRESS="00:00:00:10:01:01"
+LIBVIRT_DISK_SETTINGS="bus=virtio,cache=unsafe"
+EIB_IMAGE="registry.suse.com/edge/3.3/edge-image-builder:1.2.1"
+```
+
+Please adjust to your enviornment according to your needs.

slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/custom/scripts/01-growfs.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Bugzilla - https://bugzilla.suse.com/show_bug.cgi?id=1217430
+growfs() {
+  mnt="$1"
+  dev="$(findmnt --fstab --target ${mnt} --evaluate --real --output SOURCE --noheadings)"
+  # /dev/sda3 -> /dev/sda, /dev/nvme0n1p3 -> /dev/nvme0n1
+  parent_dev="/dev/$(lsblk --nodeps -rno PKNAME "${dev}")"
+  # Last number in the device name: /dev/nvme0n1p42 -> 42
+  partnum="$(echo "${dev}" | sed 's/^.*[^0-9]\([0-9]\+\)$/\1/')"
+  ret=0
+  growpart "$parent_dev" "$partnum" || ret=$?
+  [ $ret -eq 0 ] || [ $ret -eq 1 ] || exit 1
+  /usr/lib/systemd/systemd-growfs "$mnt"
+}
+growfs /

slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/eib.yaml

@@ -0,0 +1,33 @@
+apiVersion: 1.1
+image:
+  imageType: raw
+  arch: x86_64
+  baseImage: REPLACEME(slmicro-base-image-being-built-with-kiwi)
+  outputImageName: 331-downstream-cluster.raw
+operatingSystem:
+  time:
+    timezone: Europe/Madrid
+  kernelArgs:
+    - ignition.platform.id=openstack
+    - net.ifnames=1
+  rawConfiguration:
+    diskSize: 3G
+  packages:
+    packageList:
+      - jq
+      - qemu-guest-agent
+      - openssh-server-config-rootlogin
+    sccRegistrationCode: REPLACEME(scc-registration-code)
+  systemd:
+    disable:
+      - rebootmgr.service
+      - transactional-update.timer
+      - transactional-update-cleanup.timer
+    enable:
+      - qemu-guest-agent
+  users:
+    - username: root
+      createHomeDir: true
+      encryptedPassword: REPLACEME(root-encrypted-password)
+      sshKeys:
+        - ssh-rsa REPLACEME(sshkey)

slemicro/eib-examples/edge-331-downstream-cluster-single-node-metal3/network/configure-network.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -eux
+
+# Attempt to statically configure a nic in the case where we find a network_data.json
+# In a configuration drive
+
+CONFIG_DRIVE=$(blkid --label config-2 || true)
+if [ -z "${CONFIG_DRIVE}" ]; then
+  echo "No config-2 device found, skipping network configuration"
+  exit 0
+fi
+
+mount -o ro $CONFIG_DRIVE /mnt
+
+NETWORK_DATA_FILE="/mnt/openstack/latest/network_data.json"
+
+if [ ! -f "${NETWORK_DATA_FILE}" ]; then
+  umount /mnt
+  echo "No network_data.json found, skipping network configuration"
+  exit 0
+fi
+
+# FIXME: we can probably improve this, but there's no jq in the ramdisk
+DESIRED_HOSTNAME=$(cat /mnt/openstack/latest/meta_data.json | tr ',{}' '\n' | grep '\"metal3-name\"' | sed 's/.*\"metal3-name\": \"\(.*\)\"/\1/')
+
+mkdir -p /tmp/nmc/{desired,generated}
+cp ${NETWORK_DATA_FILE} /tmp/nmc/desired/${DESIRED_HOSTNAME}.yaml
+umount /mnt
+
+./nmc generate --config-dir /tmp/nmc/desired --output-dir /tmp/nmc/generated
+./nmc apply --config-dir /tmp/nmc/generated

slemicro/eib-examples/edge-331-mgmt-cluster-metal3/README.md

@@ -0,0 +1,28 @@
+# Example deployment of a SUSE Edge 3.3.1 management cluster
+
+- Find and replace the "REPLACEME" strings.
+- Follow the SUSE Edge documentation on how to [Build an updated SUSE Linux Micro image](https://documentation.suse.com/suse-edge/3.3/html/edge/guides-kiwi-builder-images.html). You can use the "Base" profile.
+- Drop the resulting image in the `base-images` folder.
+- Create the Management Cluster as:
+
+```
+./create_eib.sh -e eib-examples/edge-331-mgmt-cluster-metal3/ -f vm1
+for vm in vm1 vm2 vm3 ; do
+  ./create_vm_with_image.sh -i eib-examples/edge-331-mgmt-cluster-metal3/331-mgmt-cluster.raw -f ${vm}
+done
+```
+
+The vm files look like:
+
+```
+VMFOLDER="/var/lib/libvirt/images/"
+VMNAME="vm1"
+CPUS="10"
+MEMORY="10240"
+# +1 to the latest octet per VM
+MACADDRESS="00:00:00:00:01:01"
+LIBVIRT_DISK_SETTINGS="bus=virtio,cache=unsafe"
+EIB_IMAGE="registry.suse.com/edge/3.3/edge-image-builder:1.2.1"
+```
+
+Please adjust to your enviornment according to your needs.

slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/basic-setup.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Pre-requisites. Cluster already running
+export KUBECTL="/var/lib/rancher/rke2/bin/kubectl"
+export KUBECONFIG="/etc/rancher/rke2/rke2.yaml"
+
+##################
+# METAL3 DETAILS #
+##################
+export METAL3_CHART_TARGETNAMESPACE="metal3-system"
+
+###########
+# METALLB #
+###########
+export METALLBNAMESPACE="metallb-system"
+
+###########
+# RANCHER #
+###########
+export RANCHER_CHART_TARGETNAMESPACE="cattle-system"
+export RANCHER_FINALPASSWORD="adminadminadmin"
+
+die(){
+  echo ${1} 1>&2
+  exit ${2}
+}

slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/metal3.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+set -euo pipefail
+
+BASEDIR="$(dirname "$0")"
+source ${BASEDIR}/basic-setup.sh
+
+METAL3LOCKNAMESPACE="default"
+METAL3LOCKCMNAME="metal3-lock"
+
+trap 'catch $? $LINENO' EXIT
+
+catch() {
+  if [ "$1" != "0" ]; then
+    echo "Error $1 occurred on $2"
+    ${KUBECTL} delete configmap ${METAL3LOCKCMNAME} -n ${METAL3LOCKNAMESPACE}
+  fi
+}
+
+# Get or create the lock to run all those steps just in a single node
+# As the first node is created WAY before the others, this should be enough
+# TODO: Investigate if leases is better
+if [ $(${KUBECTL} get cm -n ${METAL3LOCKNAMESPACE} ${METAL3LOCKCMNAME} -o name | wc -l) -lt 1 ]; then
+  ${KUBECTL} create configmap ${METAL3LOCKCMNAME} -n ${METAL3LOCKNAMESPACE} --from-literal foo=bar
+else
+  exit 0
+fi
+
+# Wait for metal3
+while ! ${KUBECTL} wait --for condition=ready -n ${METAL3_CHART_TARGETNAMESPACE} $(${KUBECTL} get pods -n ${METAL3_CHART_TARGETNAMESPACE} -l app.kubernetes.io/name=metal3-ironic -o name) --timeout=10s; do sleep 2 ; done
+
+# Get the ironic IP
+IRONICIP=$(${KUBECTL} get cm -n ${METAL3_CHART_TARGETNAMESPACE} ironic-bmo -o jsonpath='{.data.IRONIC_IP}')
+
+# If LoadBalancer, use metallb, else it is NodePort
+if [ $(${KUBECTL} get svc -n ${METAL3_CHART_TARGETNAMESPACE} metal3-metal3-ironic -o jsonpath='{.spec.type}') == "LoadBalancer" ]; then
+  # Wait for metallb
+  while ! ${KUBECTL} wait --for condition=ready -n ${METALLBNAMESPACE} $(${KUBECTL} get pods -n ${METALLBNAMESPACE} -l app.kubernetes.io/component=controller -o name) --timeout=10s; do sleep 2 ; done
+
+  # Don't create the ippool if already created
+  ${KUBECTL} get ipaddresspool -n ${METALLBNAMESPACE} ironic-ip-pool -o name || cat <<-EOF | ${KUBECTL} apply -f -
+  apiVersion: metallb.io/v1beta1
+  kind: IPAddressPool
+  metadata:
+    name: ironic-ip-pool
+    namespace: ${METALLBNAMESPACE}
+  spec:
+    addresses:
+    - ${IRONICIP}/32
+    serviceAllocation:
+      priority: 100
+      serviceSelectors:
+      - matchExpressions:
+        - {key: app.kubernetes.io/name, operator: In, values: [metal3-ironic]}
+	EOF
+
+  # Same for L2 Advs
+  ${KUBECTL} get L2Advertisement -n ${METALLBNAMESPACE} ironic-ip-pool-l2-adv -o name || cat <<-EOF | ${KUBECTL} apply -f -
+  apiVersion: metallb.io/v1beta1
+  kind: L2Advertisement
+  metadata:
+    name: ironic-ip-pool-l2-adv
+    namespace: ${METALLBNAMESPACE}
+  spec:
+    ipAddressPools:
+    - ironic-ip-pool
+	EOF
+fi
+
+# If rancher is deployed
+if [ $(${KUBECTL} get pods -n ${RANCHER_CHART_TARGETNAMESPACE} -l app=rancher -o name | wc -l) -ge 1 ]; then
+  cat <<-EOF | ${KUBECTL} apply -f -
+	apiVersion: management.cattle.io/v3
+	kind: Feature
+	metadata:
+	  name: embedded-cluster-api
+	spec:
+	  value: false
+	EOF
+
+  # Disable Rancher webhooks for CAPI
+  ${KUBECTL} delete --ignore-not-found=true mutatingwebhookconfiguration.admissionregistration.k8s.io mutating-webhook-configuration
+  ${KUBECTL} delete --ignore-not-found=true validatingwebhookconfigurations.admissionregistration.k8s.io validating-webhook-configuration
+  ${KUBECTL} wait --for=delete namespace/cattle-provisioning-capi-system --timeout=300s
+fi
+
+# Clean up the lock cm
+
+${KUBECTL} delete configmap ${METAL3LOCKCMNAME} -n ${METAL3LOCKNAMESPACE}

slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/mgmt-stack-setup.service

@@ -0,0 +1,30 @@
+[Unit]
+Description=Setup Management stack components
+Wants=network-online.target
+# It requires rke2 or k3s running, but it won't fail if those services are not present
+After=network.target network-online.target rke2-server.service k3s.service
+# At least, the basic-setup.sh one needs to be present
+ConditionPathExists=/opt/mgmt/bin/basic-setup.sh
+
+[Service]
+User=root
+Type=forking
+# Metal3 can take A LOT to download the IPA image
+TimeoutStartSec=1800
+
+ExecStartPre=/bin/sh -c "echo 'Setting up Management components...'"
+# Scripts are executed in StartPre because Start can only run a single on
+ExecStartPre=/opt/mgmt/bin/rancher.sh
+ExecStartPre=/opt/mgmt/bin/metal3.sh
+ExecStart=/bin/sh -c "echo 'Finished setting up Management components'"
+RemainAfterExit=yes
+KillMode=process
+# Disable & delete everything
+ExecStartPost=rm -f /opt/mgmt/bin/rancher.sh
+ExecStartPost=rm -f /opt/mgmt/bin/metal3.sh
+ExecStartPost=rm -f /opt/mgmt/bin/basic-setup.sh
+ExecStartPost=/bin/sh -c "systemctl disable mgmt-stack-setup.service"
+ExecStartPost=rm -f /etc/systemd/system/mgmt-stack-setup.service
+
+[Install]
+WantedBy=multi-user.target

slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/files/rancher.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+set -euo pipefail
+
+BASEDIR="$(dirname "$0")"
+source ${BASEDIR}/basic-setup.sh
+
+RANCHERLOCKNAMESPACE="default"
+RANCHERLOCKCMNAME="rancher-lock"
+
+if [ -z "${RANCHER_FINALPASSWORD}" ]; then
+  # If there is no final password, then finish the setup right away
+  exit 0
+fi
+
+trap 'catch $? $LINENO' EXIT
+
+catch() {
+  if [ "$1" != "0" ]; then
+    echo "Error $1 occurred on $2"
+    ${KUBECTL} delete configmap ${RANCHERLOCKCMNAME} -n ${RANCHERLOCKNAMESPACE}
+  fi
+}
+
+# Get or create the lock to run all those steps just in a single node
+# As the first node is created WAY before the others, this should be enough
+# TODO: Investigate if leases is better
+if [ $(${KUBECTL} get cm -n ${RANCHERLOCKNAMESPACE} ${RANCHERLOCKCMNAME} -o name | wc -l) -lt 1 ]; then
+  ${KUBECTL} create configmap ${RANCHERLOCKCMNAME} -n ${RANCHERLOCKNAMESPACE} --from-literal foo=bar
+else
+  exit 0
+fi
+
+# Wait for rancher to be deployed
+while ! ${KUBECTL} wait --for condition=ready -n ${RANCHER_CHART_TARGETNAMESPACE} $(${KUBECTL} get pods -n ${RANCHER_CHART_TARGETNAMESPACE} -l app=rancher -o name) --timeout=10s; do sleep 2 ; done
+until ${KUBECTL} get ingress -n ${RANCHER_CHART_TARGETNAMESPACE} rancher > /dev/null 2>&1; do sleep 10; done
+
+RANCHERBOOTSTRAPPASSWORD=$(${KUBECTL} get secret -n ${RANCHER_CHART_TARGETNAMESPACE} bootstrap-secret -o jsonpath='{.data.bootstrapPassword}' | base64 -d)
+RANCHERHOSTNAME=$(${KUBECTL} get ingress -n ${RANCHER_CHART_TARGETNAMESPACE} rancher -o jsonpath='{.spec.rules[0].host}')
+
+# Skip the whole process if things have been set already
+if [ -z $(${KUBECTL} get settings.management.cattle.io first-login -ojsonpath='{.value}') ]; then
+  # Add the protocol
+  RANCHERHOSTNAME="https://${RANCHERHOSTNAME}"
+  TOKEN=""
+  while [ -z "${TOKEN}" ]; do
+    # Get token
+    sleep 2
+    TOKEN=$(curl -sk -X POST ${RANCHERHOSTNAME}/v3-public/localProviders/local?action=login -H 'content-type: application/json' -d "{\"username\":\"admin\",\"password\":\"${RANCHERBOOTSTRAPPASSWORD}\"}" | jq -r .token)
+  done
+
+  # Set password
+  #curl -sk ${RANCHERHOSTNAME}/v3/users?action=changepassword -H 'content-type: application/json' -H "Authorization: Bearer $TOKEN" -d "{\"currentPassword\":\"${RANCHERBOOTSTRAPPASSWORD}\",\"newPassword\":\"${RANCHER_FINALPASSWORD}\"}"
+
+  # Create a temporary API token (ttl=60 minutes)
+  APITOKEN=$(curl -sk ${RANCHERHOSTNAME}/v3/token -H 'content-type: application/json' -H "Authorization: Bearer ${TOKEN}" -d '{"type":"token","description":"automation","ttl":3600000}' | jq -r .token)
+
+  curl -sk ${RANCHERHOSTNAME}/v3/settings/server-url -H 'content-type: application/json' -H "Authorization: Bearer ${APITOKEN}" -X PUT -d "{\"name\":\"server-url\",\"value\":\"${RANCHERHOSTNAME}\"}"
+  curl -sk ${RANCHERHOSTNAME}/v3/settings/telemetry-opt -X PUT -H 'content-type: application/json' -H 'accept: application/json' -H "Authorization: Bearer ${APITOKEN}" -d '{"value":"out"}'
+fi
+
+# Clean up the lock cm
+${KUBECTL} delete configmap ${RANCHERLOCKCMNAME} -n ${RANCHERLOCKNAMESPACE}

slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/scripts/99-alias.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+echo "alias k=kubectl" >> /etc/profile.local
+echo "alias kubectl=/var/lib/rancher/rke2/bin/kubectl" >> /etc/profile.local
+echo "export KUBECONFIG=/etc/rancher/rke2/rke2.yaml" >> /etc/profile.local

slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/scripts/99-mgmt-setup.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Copy the scripts from combustion to the final location
+mkdir -p /opt/mgmt/bin/
+for script in basic-setup.sh rancher.sh metal3.sh; do
+	cp ${script} /opt/mgmt/bin/
+	chmod 744 ${script}
+done
+
+# Copy the systemd unit file and enable it at boot
+cp mgmt-stack-setup.service /etc/systemd/system/mgmt-stack-setup.service
+systemctl enable mgmt-stack-setup.service

slemicro/eib-examples/edge-331-mgmt-cluster-metal3/custom/scripts/99-register.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -euo pipefail
+
+# Registration https://www.suse.com/support/kb/doc/?id=000018564
+if ! which SUSEConnect > /dev/null 2>&1; then
+	zypper --non-interactive install suseconnect-ng
+fi
+SUSEConnect --email "REPLACEME(scc-emai)" --url "https://scc.suse.com" --regcode "REPLACEME(scc-regcode)"