diff --git a/packer/k3s.sh b/packer/k3s.sh index 64974d1..7589fe6 100755 --- a/packer/k3s.sh +++ b/packer/k3s.sh @@ -44,6 +44,10 @@ set -o noglob # Commit of k3s to download from temporary cloud storage. # * (for developer & QA use) # +# - INSTALL_K3S_PR +# PR build of k3s to download from Github Artifacts. +# * (for developer & QA use) +# # - INSTALL_K3S_BIN_DIR # Directory to install k3s binary, links, and uninstall script to, or use # /usr/local/bin as the default @@ -92,6 +96,7 @@ set -o noglob # Defaults to 'stable'. GITHUB_URL=https://github.com/k3s-io/k3s/releases +GITHUB_PR_URL="" STORAGE_URL=https://k3s-ci-builds.s3.amazonaws.com DOWNLOADER= @@ -275,11 +280,11 @@ can_skip_download_binary() { fi } -can_skip_download_selinux() { - if [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != true ] && [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != selinux ]; then - return 1 - fi -} +can_skip_download_selinux() { + if [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != true ] && [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != selinux ]; then + return 1 + fi +} # --- verify an executable k3s binary is installed --- verify_k3s_is_executable() { @@ -337,6 +342,7 @@ verify_downloader() { setup_tmp() { TMP_DIR=$(mktemp -d -t k3s-install.XXXXXXXXXX) TMP_HASH=${TMP_DIR}/k3s.hash + TMP_ZIP=${TMP_DIR}/k3s.zip TMP_BIN=${TMP_DIR}/k3s.bin cleanup() { code=$? @@ -350,7 +356,10 @@ setup_tmp() { # --- use desired k3s version if defined or find version from channel --- get_release_version() { - if [ -n "${INSTALL_K3S_COMMIT}" ]; then + if [ -n "${INSTALL_K3S_PR}" ]; then + VERSION_K3S="PR ${INSTALL_K3S_PR}" + get_pr_artifact_url + elif [ -n "${INSTALL_K3S_COMMIT}" ]; then VERSION_K3S="commit ${INSTALL_K3S_COMMIT}" elif [ -n "${INSTALL_K3S_VERSION}" ]; then VERSION_K3S=${INSTALL_K3S_VERSION} @@ -372,10 +381,49 @@ get_release_version() { info "Using ${VERSION_K3S} as release" } +# --- get k3s-selinux version --- +get_k3s_selinux_version() { + available_version="k3s-selinux-1.2-2.${rpm_target}.noarch.rpm" + info "Finding available k3s-selinux versions" + + # run verify_downloader in case it binary installation was skipped + verify_downloader curl || verify_downloader wget || fatal 'Can not find curl or wget for downloading files' + + case $DOWNLOADER in + curl) + DOWNLOADER_OPTS="-s" + ;; + wget) + DOWNLOADER_OPTS="-q -O -" + ;; + *) + fatal "Incorrect downloader executable '$DOWNLOADER'" + ;; + esac + for i in {1..3}; do + set +e + if [ "${rpm_channel}" = "testing" ]; then + version=$(timeout 5 ${DOWNLOADER} ${DOWNLOADER_OPTS} https://api.github.com/repos/k3s-io/k3s-selinux/releases | grep browser_download_url | awk '{ print $2 }' | grep -oE "[^\/]+${rpm_target}\.noarch\.rpm" | head -n 1) + else + version=$(timeout 5 ${DOWNLOADER} ${DOWNLOADER_OPTS} https://api.github.com/repos/k3s-io/k3s-selinux/releases/latest | grep browser_download_url | awk '{ print $2 }' | grep -oE "[^\/]+${rpm_target}\.noarch\.rpm") + fi + set -e + if [ "${version}" != "" ]; then + break + fi + sleep 1 + done + if [ "${version}" == "" ]; then + warn "Failed to get available versions of k3s-selinux..defaulting to ${available_version}" + return + fi + available_version=${version} +} + # --- download from github url --- download() { [ $# -eq 2 ] || fatal 'download needs exactly 2 arguments' - + set +e case $DOWNLOADER in curl) curl -o $1 -sfL $2 @@ -390,17 +438,24 @@ download() { # Abort if download command failed [ $? -eq 0 ] || fatal 'Download failed' + set -e } # --- download hash from github url --- download_hash() { - if [ -n "${INSTALL_K3S_COMMIT}" ]; then - HASH_URL=${STORAGE_URL}/k3s${SUFFIX}-${INSTALL_K3S_COMMIT}.sha256sum + if [ -n "${INSTALL_K3S_PR}" ]; then + info "Downloading hash ${GITHUB_PR_URL}" + curl -o ${TMP_ZIP} -H "Authorization: Bearer $GITHUB_TOKEN" -L ${GITHUB_PR_URL} + unzip -p ${TMP_ZIP} k3s.sha256sum > ${TMP_HASH} else - HASH_URL=${GITHUB_URL}/download/${VERSION_K3S}/sha256sum-${ARCH}.txt + if [ -n "${INSTALL_K3S_COMMIT}" ]; then + HASH_URL=${STORAGE_URL}/k3s${SUFFIX}-${INSTALL_K3S_COMMIT}.sha256sum + else + HASH_URL=${GITHUB_URL}/download/${VERSION_K3S}/sha256sum-${ARCH}.txt + fi + info "Downloading hash ${HASH_URL}" + download ${TMP_HASH} ${HASH_URL} fi - info "Downloading hash ${HASH_URL}" - download ${TMP_HASH} ${HASH_URL} HASH_EXPECTED=$(grep " k3s${SUFFIX}$" ${TMP_HASH}) HASH_EXPECTED=${HASH_EXPECTED%%[[:blank:]]*} } @@ -417,9 +472,48 @@ installed_hash_matches() { return 1 } +# Use the GitHub API to identify the artifact associated with a given PR +get_pr_artifact_url() { + GITHUB_API_URL=https://api.github.com/repos/k3s-io/k3s + + # Check if jq is installed + if ! [ -x "$(command -v jq)" ]; then + echo "jq is required to use INSTALL_K3S_PR. Please install jq and try again" + exit 1 + fi + + if [ -z "${GITHUB_TOKEN}" ]; then + fatal "Installing PR builds requires GITHUB_TOKEN with k3s-io/k3s repo authorization" + fi + + # GET request to the GitHub API to retrieve the latest commit SHA from the pull request + COMMIT_ID=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" "$GITHUB_API_URL/pulls/$INSTALL_K3S_PR" | jq -r '.head.sha') + + # GET request to the GitHub API to retrieve the Build workflow associated with the commit + wf_raw=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" "$GITHUB_API_URL/commits/$COMMIT_ID/check-runs") + build_workflow=$(printf "%s" "$wf_raw" | jq -r '.check_runs[] | select(.name == "build / Build")') + + # Extract the Run ID from the build workflow and lookup artifacts associated with the run + RUN_ID=$(echo "$build_workflow" | jq -r ' .details_url' | awk -F'/' '{print $(NF-2)}') + + # Extract the artifat ID for the "k3s" artifact + artifacts=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" "$GITHUB_API_URL/actions/runs/$RUN_ID/artifacts") + artifacts_url=$(echo "$artifacts" | jq -r '.artifacts[] | select(.name == "k3s") | .archive_download_url') + GITHUB_PR_URL=$artifacts_url +} + # --- download binary from github url --- download_binary() { - if [ -n "${INSTALL_K3S_COMMIT}" ]; then + if [ -n "${INSTALL_K3S_PR}" ]; then + # Since Binary and Hash are zipped together, check if TMP_ZIP already exists + if ! [ -f ${TMP_ZIP} ]; then + info "Downloading K3s artifact ${GITHUB_PR_URL}" + curl -o ${TMP_ZIP} -H "Authorization: Bearer $GITHUB_TOKEN" -L ${GITHUB_PR_URL} + fi + # extract k3s binary from zip + unzip -p ${TMP_ZIP} k3s > ${TMP_BIN} + return + elif [ -n "${INSTALL_K3S_COMMIT}" ]; then BIN_URL=${STORAGE_URL}/k3s${SUFFIX}-${INSTALL_K3S_COMMIT} else BIN_URL=${GITHUB_URL}/download/${VERSION_K3S}/k3s${SUFFIX} @@ -448,7 +542,7 @@ setup_binary() { # --- setup selinux policy --- setup_selinux() { - case ${INSTALL_K3S_CHANNEL} in + case ${INSTALL_K3S_CHANNEL} in *testing) rpm_channel=testing ;; @@ -466,18 +560,35 @@ setup_selinux() { fi [ -r /etc/os-release ] && . /etc/os-release - if [ "${ID_LIKE%%[ ]*}" = "suse" ]; then + if [ `expr "${ID_LIKE}" : ".*suse.*"` != 0 ]; then rpm_target=sle rpm_site_infix=microos package_installer=zypper + if [ "${ID_LIKE:-}" = suse ] && ( [ "${VARIANT_ID:-}" = sle-micro ] || [ "${ID:-}" = sle-micro ] ); then + rpm_target=sle + rpm_site_infix=slemicro + package_installer=zypper + fi + elif [ "${ID_LIKE:-}" = coreos ] || [ "${VARIANT_ID:-}" = coreos ]; then + rpm_target=coreos + rpm_site_infix=coreos + package_installer=rpm-ostree elif [ "${VERSION_ID%%.*}" = "7" ]; then rpm_target=el7 rpm_site_infix=centos/7 package_installer=yum - else + elif [ "${VERSION_ID%%.*}" = "8" ] || [ "${VERSION_ID%%.*}" -gt "36" ]; then rpm_target=el8 rpm_site_infix=centos/8 package_installer=yum + else + rpm_target=el9 + rpm_site_infix=centos/9 + package_installer=yum + fi + + if [ "${package_installer}" = "rpm-ostree" ] && [ -x /bin/yum ]; then + package_installer=yum fi if [ "${package_installer}" = "yum" ] && [ -x /usr/bin/dnf ]; then @@ -486,15 +597,17 @@ setup_selinux() { policy_hint="please install: ${package_installer} install -y container-selinux - ${package_installer} install -y https://${rpm_site}/k3s/${rpm_channel}/common/${rpm_site_infix}/noarch/k3s-selinux-1.2-2.${rpm_target}.noarch.rpm + ${package_installer} install -y https://${rpm_site}/k3s/${rpm_channel}/common/${rpm_site_infix}/noarch/${available_version} " if [ "$INSTALL_K3S_SKIP_SELINUX_RPM" = true ] || can_skip_download_selinux || [ ! -d /usr/share/selinux ]; then info "Skipping installation of SELinux RPM" - elif [ "${ID_LIKE:-}" != coreos ] && [ "${VARIANT_ID:-}" != coreos ]; then - install_selinux_rpm ${rpm_site} ${rpm_channel} ${rpm_target} ${rpm_site_infix} + return fi + get_k3s_selinux_version + install_selinux_rpm ${rpm_site} ${rpm_channel} ${rpm_target} ${rpm_site_infix} + policy_error=fatal if [ "$INSTALL_K3S_SELINUX_WARN" = true ] || [ "${ID_LIKE:-}" = coreos ] || [ "${VARIANT_ID:-}" = coreos ]; then policy_error=warn @@ -505,7 +618,7 @@ setup_selinux() { $policy_error "Failed to apply container_runtime_exec_t to ${BIN_DIR}/k3s, ${policy_hint}" fi elif [ ! -f /usr/share/selinux/packages/k3s.pp ]; then - if [ -x /usr/sbin/transactional-update ]; then + if [ -x /usr/sbin/transactional-update ] || [ "${ID_LIKE:-}" = coreos ] || [ "${VARIANT_ID:-}" = coreos ]; then warn "Please reboot your machine to activate the changes and avoid data loss." else $policy_error "Failed to find the k3s-selinux policy, ${policy_hint}" @@ -514,7 +627,7 @@ setup_selinux() { } install_selinux_rpm() { - if [ -r /etc/redhat-release ] || [ -r /etc/centos-release ] || [ -r /etc/oracle-release ] || [ "${ID_LIKE%%[ ]*}" = "suse" ]; then + if [ -r /etc/redhat-release ] || [ -r /etc/centos-release ] || [ -r /etc/oracle-release ] || [ -r /etc/fedora-release ] || [ "${ID_LIKE%%[ ]*}" = "suse" ]; then repodir=/etc/yum.repos.d if [ -d /etc/zypp/repos.d ]; then repodir=/etc/zypp/repos.d @@ -539,9 +652,17 @@ EOF sle) rpm_installer="zypper --gpg-auto-import-keys" if [ "${TRANSACTIONAL_UPDATE=false}" != "true" ] && [ -x /usr/sbin/transactional-update ]; then + transactional_update_run="transactional-update --no-selfupdate -d run" rpm_installer="transactional-update --no-selfupdate -d run ${rpm_installer}" : "${INSTALL_K3S_SKIP_START:=true}" fi + # create the /var/lib/rpm-state in SLE systems to fix the prein selinux macro + ${transactional_update_run} mkdir -p /var/lib/rpm-state + ;; + coreos) + rpm_installer="rpm-ostree --idempotent" + # rpm_install_extra_args="--apply-live" + : "${INSTALL_K3S_SKIP_START:=true}" ;; *) rpm_installer="yum" @@ -549,6 +670,15 @@ EOF esac if [ "${rpm_installer}" = "yum" ] && [ -x /usr/bin/dnf ]; then rpm_installer=dnf + fi + if rpm -q --quiet k3s-selinux; then + # remove k3s-selinux module before upgrade to allow container-selinux to upgrade safely + if check_available_upgrades container-selinux ${3} && check_available_upgrades k3s-selinux ${3}; then + MODULE_PRIORITY=$($SUDO semodule --list=full | grep k3s | cut -f1 -d" ") + if [ -n "${MODULE_PRIORITY}" ]; then + $SUDO semodule -X $MODULE_PRIORITY -r k3s || true + fi + fi fi # shellcheck disable=SC2086 $SUDO ${rpm_installer} install -y "k3s-selinux" @@ -556,6 +686,25 @@ EOF return } +check_available_upgrades() { + set +e + case ${2} in + sle) + available_upgrades=$($SUDO zypper -q -t -s 11 se -s -u --type package $1 | tail -n 1 | grep -v "No matching" | awk '{print $3}') + ;; + coreos) + # currently rpm-ostree does not support search functionality https://github.com/coreos/rpm-ostree/issues/1877 + ;; + *) + available_upgrades=$($SUDO yum -q --refresh list $1 --upgrades | tail -n 1 | awk '{print $2}') + ;; + esac + set -e + if [ -n "${available_upgrades}" ]; then + return 0 + fi + return 1 +} # --- download and verify k3s --- download_and_verify() { if can_skip_download_binary; then @@ -646,6 +795,27 @@ killtree() { ) 2>/dev/null } +remove_interfaces() { + # Delete network interface(s) that match 'master cni0' + ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do + iface=${iface%%@*} + [ -z "$iface" ] || ip link delete $iface + done + + # Delete cni related interfaces + ip link delete cni0 + ip link delete flannel.1 + ip link delete flannel-v6.1 + ip link delete kube-ipvs0 + ip link delete flannel-wg + ip link delete flannel-wg-v6 + + # Restart tailscale + if [ -n "$(command -v tailscale)" ]; then + tailscale set --advertise-routes= + fi +} + getshims() { ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1 } @@ -656,7 +826,7 @@ do_unmount_and_remove() { set +x while read -r _ path _; do case "$path" in $1*) echo "$path" ;; esac - done < /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount "$0" && rm -rf "$0"' + done < /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount -f "$0" && rm -rf "$0"' set -x } @@ -669,17 +839,8 @@ do_unmount_and_remove '/run/netns/cni-' # Remove CNI namespaces ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete -# Delete network interface(s) that match 'master cni0' -ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do - iface=${iface%%@*} - [ -z "$iface" ] || ip link delete $iface -done -ip link delete cni0 -ip link delete flannel.1 -ip link delete flannel-v6.1 -ip link delete kube-ipvs0 -ip link delete flannel-wg -ip link delete flannel-wg-v6 +remove_interfaces + rm -rf /var/lib/cni/ iptables-save | grep -v KUBE- | grep -v CNI- | grep -iv flannel | iptables-restore ip6tables-save | grep -v KUBE- | grep -v CNI- | grep -iv flannel | ip6tables-restore @@ -738,6 +899,9 @@ rm -f ${KILLALL_K3S_SH} if type yum >/dev/null 2>&1; then yum remove -y k3s-selinux rm -f /etc/yum.repos.d/rancher-k3s-common*.repo +elif type rpm-ostree >/dev/null 2>&1; then + rpm-ostree uninstall k3s-selinux + rm -f /etc/yum.repos.d/rancher-k3s-common*.repo elif type zypper >/dev/null 2>&1; then uninstall_cmd="zypper remove -y k3s-selinux" if [ "\${TRANSACTIONAL_UPDATE=false}" != "true" ] && [ -x /usr/sbin/transactional-update ]; then @@ -796,7 +960,7 @@ TasksMax=infinity TimeoutStartSec=0 Restart=always RestartSec=5s -ExecStartPre=/bin/sh -xc '! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service' +ExecStartPre=/bin/sh -xc '! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service 2>/dev/null' ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay ExecStart=${BIN_DIR}/k3s \\ @@ -836,8 +1000,8 @@ respawn_delay=5 respawn_max=0 set -o allexport -if [ -f /etc/environment ]; then sourcex /etc/environment; fi -if [ -f ${FILE_K3S_ENV} ]; then sourcex ${FILE_K3S_ENV}; fi +if [ -f /etc/environment ]; then . /etc/environment; fi +if [ -f ${FILE_K3S_ENV} ]; then . ${FILE_K3S_ENV}; fi set +o allexport EOF $SUDO chmod 0755 ${FILE_K3S_SERVICE} @@ -853,11 +1017,16 @@ EOF # --- write systemd or openrc service file --- create_service_file() { - [ "${HAS_SYSTEMD}" = true ] && create_systemd_service_file + [ "${HAS_SYSTEMD}" = true ] && create_systemd_service_file && restore_systemd_service_file_context [ "${HAS_OPENRC}" = true ] && create_openrc_service_file return 0 } +restore_systemd_service_file_context() { + $SUDO restorecon -R -i ${FILE_K3S_SERVICE} 2>/dev/null || true + $SUDO restorecon -R -i ${FILE_K3S_ENV} 2>/dev/null || true +} + # --- get hashes of the current k3s bin and service files get_installed_hashes() { $SUDO sha256sum ${BIN_DIR}/k3s ${FILE_K3S_SERVICE} ${FILE_K3S_ENV} 2>&1 || true @@ -886,6 +1055,19 @@ openrc_start() { $SUDO ${FILE_K3S_SERVICE} restart } +has_working_xtables() { + if $SUDO sh -c "command -v \"$1-save\"" 1> /dev/null && $SUDO sh -c "command -v \"$1-restore\"" 1> /dev/null; then + if $SUDO $1-save 2>/dev/null | grep -q '^-A CNI-HOSTPORT-MASQ -j MASQUERADE$'; then + warn "Host $1-save/$1-restore tools are incompatible with existing rules" + else + return 0 + fi + else + info "Host $1-save/$1-restore tools not found" + fi + return 1 +} + # --- startup systemd or openrc service --- service_enable_and_start() { if [ -f "/proc/cgroups" ] && [ "$(grep memory /proc/cgroups | while read -r n n n enabled; do echo $enabled; done)" -eq 0 ]; @@ -906,6 +1088,12 @@ service_enable_and_start() { return fi + for XTABLES in iptables ip6tables; do + if has_working_xtables ${XTABLES}; then + $SUDO ${XTABLES}-save 2>/dev/null | grep -v KUBE- | grep -iv flannel | $SUDO ${XTABLES}-restore + fi + done + [ "${HAS_SYSTEMD}" = true ] && systemd_start [ "${HAS_OPENRC}" = true ] && openrc_start return 0