From b5057d46da5ad8342599ed674ef524c16e362481 Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Fri, 26 May 2023 13:53:28 -0700
Subject: [PATCH] linuxulator & bhyve

- add debian bullseye
- centos: added
- made linux an include
- base: periodic tuning
---
 include/linux.sh          |  85 +++++++++++++++++++++
 mail-toaster.sh           |   1 +
 provision/base.sh         |   6 +-
 provision/bhyve-ubuntu.sh | 153 ++++++++++++++++++++++++++++++++++++++
 provision/centos.sh       |  35 +++++++++
 provision/host.sh         |   9 +++
 provision/ubuntu.sh       |  35 +++++++++
 7 files changed, 323 insertions(+), 1 deletion(-)
 create mode 100755 include/linux.sh
 create mode 100755 provision/bhyve-ubuntu.sh
 create mode 100755 provision/centos.sh
 create mode 100755 provision/ubuntu.sh

diff --git a/include/linux.sh b/include/linux.sh
new file mode 100755
index 00000000..2cf66d72
--- /dev/null
+++ b/include/linux.sh
@@ -0,0 +1,85 @@
+#!/bin/sh
+
+# see examples in provision/centos and provision/ubuntu
+
+configure_linuxulator()
+{
+	tell_status "enabling Linux emulation on Host (loads kernel modules)"
+	sysrc linux_enable=YES
+	sysrc linux_mounts_enable=NO
+	service linux start
+
+	tell_status "enabling Linux emulation in jail"
+	stage_sysrc linux_enable=YES
+	stage_sysrc linux_mounts_enable=NO
+	stage_exec service linux start
+}
+
+configure_apt_sources()
+{
+	case "$1" in
+		bionic|focal|jammy)
+			tell_status "restoring APT sources"
+			tee "$STAGE_MNT/compat/linux/etc/apt/sources.list" <<EO_UB_SOURCES
+deb http://archive.ubuntu.com/ubuntu $1 main universe restricted multiverse
+deb http://security.ubuntu.com/ubuntu/ $1-security universe multiverse restricted main
+deb http://archive.ubuntu.com/ubuntu $1-backports universe multiverse restricted main
+deb http://archive.ubuntu.com/ubuntu $1-updates universe multiverse restricted main
+EO_UB_SOURCES
+			;;
+		bullseye)
+			tell_status "adding APT sources"
+			tee "$STAGE_MNT/compat/linux/etc/apt/sources.list" <<EO_DEB_SOURCES
+deb http://deb.debian.org/debian $1 main contrib non-free
+deb http://deb.debian.org/debian-security/ $1-security main contrib non-free
+deb http://deb.debian.org/debian $1-updates main contrib non-free
+deb http://deb.debian.org/debian $1-backports main contrib non-free
+EO_DEB_SOURCES
+	esac
+}
+
+install_apt_updates()
+{
+	tell_status "updating apt"
+	stage_exec chroot /compat/linux apt update || exit 1
+
+	tell_status "updating installed apt packages"
+	stage_exec chroot /compat/linux apt upgrade -y || exit 1
+}
+
+install_linux()
+{
+	# tested with values of $1:
+	#   Ubuntu: bionic (18), focal (20) and jammy (22)
+	#   Debian: bullseye (11)
+	#   CentOS: centos (7)
+
+	configure_linuxulator
+
+	case "$1" in
+		centos)
+			tell_status "installing $1"
+			stage_pkg_install linux_base-c7 || exit 1
+		;;
+		bionic|bullseye|focal|jammy)
+			tell_status "installing (debian|ubuntu) $1"
+			stage_pkg_install debootstrap || exit 1
+			stage_exec debootstrap $1 /compat/linux
+			configure_apt_sources $1
+		;;
+	esac
+
+	case "$1" in
+		bionic) stage_exec chroot /compat/linux apt remove -y rsyslog ;;
+		jammy)  stage_exec mount -t devfs devfs /compat/linux/dev ;;
+	esac
+
+	case "$1" in
+		bionic|focal|jammy|bullseye) install_apt_updates ;;
+	esac
+
+	case "$1" in
+		jammy) stage_exec umount /compat/linux/dev ;;
+	esac
+}
+
diff --git a/mail-toaster.sh b/mail-toaster.sh
index 046fa5d3..1d248b6c 100755
--- a/mail-toaster.sh
+++ b/mail-toaster.sh
@@ -165,6 +165,7 @@ export JAIL_NET_INTERFACE=${JAIL_NET_INTERFACE:="lo1"}
 export JAIL_ORDERED_LIST="syslog base dns mysql clamav spamassassin dspam vpopmail haraka webmail munin haproxy rspamd avg dovecot redis geoip nginx mailtest apache postgres minecraft joomla php7 memcached sphinxsearch elasticsearch nictool sqwebmail dhcp letsencrypt tinydns roundcube squirrelmail rainloop rsnapshot mediawiki smf wordpress whmcs squirrelcart horde grafana unifi mongodb gitlab gitlab_runner dcc prometheus influxdb telegraf statsd mail_dmarc ghost jekyll borg nagios postfix puppeteer snappymail knot nsd bsd_cache"
 
 export ZFS_VOL=${ZFS_VOL:="zroot"}
+export ZFS_BHYVE_VOL="${ZFS_BHYVE_VOL:=$ZFS_VOL}"
 export ZFS_JAIL_MNT=${ZFS_JAIL_MNT:="/jails"}
 export ZFS_DATA_MNT=${ZFS_DATA_MNT:="/data"}
 export FBSD_MIRROR=${FBSD_MIRROR:="ftp://ftp.freebsd.org"}
diff --git a/provision/base.sh b/provision/base.sh
index 8a4fb4f1..67412b59 100755
--- a/provision/base.sh
+++ b/provision/base.sh
@@ -258,11 +258,14 @@ monthly_status_security_inline="NO"
 # These are redundant within a jail
 security_status_chkmounts_enable="NO"
 security_status_chksetuid_enable="NO"
+security_status_chkuid0_enable="NO"
 security_status_neggrpperm_enable="NO"
+security_status_ipfdenied_enable="NO"
 security_status_ipfwlimit_enable="NO"
 security_status_ipfwdenied_enable="NO"
-security_status_pfdenied_enable="NO"
 security_status_kernelmsg_enable="NO"
+security_status_pfdenied_enable="NO"
+security_status_tcpwrap_enable="NO"
 
 daily_accounting_enable="NO"
 daily_accounting_compress="YES"
@@ -291,6 +294,7 @@ daily_status_rwho_enable="NO"
 daily_submit_queuerun="NO"
 
 weekly_accounting_enable="NO"
+weekly_locate_enable="NO"
 weekly_show_success="NO"
 weekly_show_info="NO"
 weekly_show_badconfig="YES"
diff --git a/provision/bhyve-ubuntu.sh b/provision/bhyve-ubuntu.sh
new file mode 100755
index 00000000..9089f272
--- /dev/null
+++ b/provision/bhyve-ubuntu.sh
@@ -0,0 +1,153 @@
+#!/bin/sh
+
+. mail-toaster.sh || exit
+
+create_bridge()
+{
+	if ! grep -q "tap.up_on_open" /etc/sysctl.conf; then
+		tell_status "setting tap.up_on_open"
+		sysctl net.link.tap.up_on_open=1
+		echo "net.link.tap.up_on_open=1" >> /etc/sysctl.conf
+	fi
+
+	# create a named bridge for bhyve VMs
+	ifconfig bridge bridge-public 2>/dev/null || {
+		tell_status "creating bridge-public"
+		ifconfig bridge create name bridge-public
+		get_public_facing_nic
+		ifconfig bridge-public addm "$PUBLIC_NIC"
+		ifconfig bridge-public up
+	}
+
+	# create tap interface for VM
+	ifconfig tap-ubuntu 2>/dev/null || {
+		tell_status "creating VM tap interface"
+		ifconfig tap create name tap-ubuntu
+		ifconfig bridge-public addm tap-ubuntu
+	}
+
+	if ! grep -q if_bridge_load /boot/loader.conf; then
+		tell_status "enabling bridge & tap load at boot time"
+		sysrc -f /boot/loader.conf if_bridge_load=YES
+		sysrc -f /boot/loader.conf if_tap_load=YES
+	fi
+}
+
+configure_grub()
+{
+	tee -a device.map <<EO_DMAP
+(hd0) /dev/zvol/$ZFS_BHYVE_VOL/bhyve/ubuntu-guest
+(cd0) /$ZFS_BHYVE_VOL/ISO/ubuntu-22.04.2-live-server-amd64.iso
+EO_DMAP
+
+	tell_status "loading the Linux kernel"
+	grub-bhyve -m device.map -r cd0 -M 1024M ubuntu-guest
+
+	tee -a <<EO_GRUB
+	grub> ls
+(hd0) (cd0) (cd0,msdos1) (host)
+grub> ls (cd0)/isolinux
+boot.cat boot.msg grub.conf initrd.img isolinux.bin isolinux.cfg memtest
+splash.jpg TRANS.TBL vesamenu.c32 vmlinuz
+grub> linux (cd0)/isolinux/vmlinuz
+grub> initrd (cd0)/isolinux/initrd.img
+grub> boot
+EO_GRUB
+
+	# within Ubuntu VM
+	tee -a /etc/default/grub <<EO_DEFAULT_GRUB
+GRUB_CMDLINE_LINUX_DEFAULT=""
+GRUB_TERMINAL='serial console'
+GRUB_CMDLINE_LINUX="console=hvc0 console=ttyS0,115200n8"
+GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"
+EO_DEFAULT_GRUB
+
+	sudo update-grub
+}
+
+install_ubuntu_bhyve_zfs()
+{
+	if ! zfs_filesystem_exists "$ZFS_BHYVE_VOL/bhyve"; then
+		zfs create "$ZFS_BHYVE_VOL/bhyve"
+		zfs set recordsize=64K "$ZFS_BHYVE_VOL/bhyve"
+	fi
+
+	if ! zfs_filesystem_exists "$ZFS_BHYVE_VOL/bhyve/ubuntu-guest"; then
+		zfs create -V20G -o volmode=dev "$ZFS_BHYVE_VOL/bhyve/ubuntu-guest"
+	fi
+}
+
+install_ubuntu_bhyve()
+{
+	if ! grep -q vmm_load /boot/loader.conf; then
+		tell_status "loading kernel module: vmm"
+		kldstat vmm || kldload vmm || exit 1
+		sysrc -f /boot/loader.conf vmm_load=YES
+	fi
+
+	create_bridge
+	install_ubuntu_bhyve_zfs
+
+	tell_status "installing bhyve"
+	stage_pkg_install bhyve-firmware grub2-bhyve || exit
+	#configure_grub
+
+	bhyve \
+		-H -P -w \
+		-c 1 -m 1G \
+		-s 0:0,hostbridge \
+		-s 2:0,virtio-net,tap-ubuntu \
+		-s 3:0,ahci-cd,/$ZFS_BHYVE_VOL/ISO/ubuntu-22.04.2-live-server-amd64.iso \
+		-s 4:0,virtio-blk,/dev/zvol/$ZFS_BHYVE_VOL/bhyve/ubuntu-guest \
+		-s 29:0,fbuf,tcp=0.0.0.0:5900,w=800,h=600,wait \
+		-s 30:0,xhci,tablet \
+		-s 31:0,lpc \
+		-l bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI.fd \
+		-l com1,stdio \
+		ubuntu-guest
+
+	bhyvectl --destroy --vm=ubuntu-guest
+}
+
+install_ubuntu()
+{
+	# tell_status "installing ubuntu"
+	install_ubuntu_bhyve
+}
+
+configure_ubuntu()
+{
+	local _pdir="$STAGE_MNT/usr/local/etc/periodic"
+}
+
+start_ubuntu()
+{
+	tell_status "starting up VM"
+	bhyve -AHP \
+		-c 4 -m 1G \
+		-s 0:0,hostbridge \
+		-s 2:0,virtio-net,tap-ubuntu \
+		-s 4:0,virtio-blk,/dev/zvol/$ZFS_BHYVE_VOL/bhyve/ubuntu-guest \
+		-s 29:0,fbuf,tcp=0.0.0.0:5900,w=800,h=600 \
+		-s 30:0,xhci,tablet \
+		-s 31:0,lpc \
+		-l com1,stdio \
+		-l bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI.fd \
+		ubuntu-guest
+
+	bhyvectl --destroy --vm=ubuntu-guest
+}
+
+test_ubuntu()
+{
+	echo "hrmm, how to test?"
+}
+
+#base_snapshot_exists || exit
+#create_staged_fs ubuntu-guest
+#start_staged_jail ubuntu-guest
+install_ubuntu
+configure_ubuntu
+#start_ubuntu
+#test_ubuntu
+#promote_staged_jail ubuntu-guest
diff --git a/provision/centos.sh b/provision/centos.sh
new file mode 100755
index 00000000..ca272b53
--- /dev/null
+++ b/provision/centos.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+
+. mail-toaster.sh || exit
+
+mt6-include linux
+
+export JAIL_START_EXTRA="allow.mount
+		allow.mount.devfs
+		allow.mount.fdescfs
+		allow.mount.procfs
+		allow.mount.linprocfs
+		allow.mount.linsysfs
+		allow.mount.tmpfs
+		enforce_statfs=1
+"
+export JAIL_CONF_EXTRA='
+		allow.raw_sockets;
+		mount += "devfs     $path/compat/linux/dev     devfs     rw  0 0";
+		mount += "tmpfs     $path/compat/linux/dev/shm tmpfs     rw,size=1g,mode=1777  0 0";
+		mount += "fdescfs   $path/compat/linux/dev/fd  fdescfs   rw,linrdlnk 0 0";
+		mount += "linprocfs $path/compat/linux/proc    linprocfs rw  0 0";
+		mount += "linsysfs  $path/compat/linux/sys     linsysfs  rw  0 0";
+		#mount += "/tmp      $path/compat/linux/tmp     nullfs    rw  0 0";
+		#mount += "/home     $path/compat/linux/home    nullfs    rw  0 0";'
+
+install_centos()
+{
+	install_linux centos
+}
+
+base_snapshot_exists || exit
+create_staged_fs centos
+start_staged_jail centos
+install_centos
+promote_staged_jail centos
diff --git a/provision/host.sh b/provision/host.sh
index a2f0bd33..03dc71c8 100755
--- a/provision/host.sh
+++ b/provision/host.sh
@@ -541,8 +541,17 @@ $(get_jail_ip "$_j")		$_j"
 	echo "$_hosts" >> "/etc/hosts"
 }
 
+update_mt6()
+{
+	if [ -d ".git" ]; then
+		git remote update
+		git status -u no
+	fi
+}
+
 update_host() {
 	sysrc background_fsck=NO
+	update_mt6
 	update_freebsd
 	configure_pkg_latest ""
 	configure_ntp
diff --git a/provision/ubuntu.sh b/provision/ubuntu.sh
new file mode 100755
index 00000000..efaf623e
--- /dev/null
+++ b/provision/ubuntu.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+
+. mail-toaster.sh || exit
+
+mt6-include linux
+
+export JAIL_START_EXTRA="allow.mount
+		allow.mount.devfs
+		allow.mount.fdescfs
+		allow.mount.procfs
+		allow.mount.linprocfs
+		allow.mount.linsysfs
+		allow.mount.tmpfs
+		enforce_statfs=1
+"
+export JAIL_CONF_EXTRA='
+		allow.raw_sockets;
+		mount += "devfs     $path/compat/linux/dev     devfs     rw  0 0";
+		mount += "tmpfs     $path/compat/linux/dev/shm tmpfs     rw,size=1g,mode=1777  0 0";
+		mount += "fdescfs   $path/compat/linux/dev/fd  fdescfs   rw,linrdlnk 0 0";
+		mount += "linprocfs $path/compat/linux/proc    linprocfs rw  0 0";
+		mount += "linsysfs  $path/compat/linux/sys     linsysfs  rw  0 0";
+		#mount += "/tmp      $path/compat/linux/tmp     nullfs    rw  0 0";
+		#mount += "/home     $path/compat/linux/home    nullfs    rw  0 0";'
+
+install_ubuntu()
+{
+	install_linux jammy
+}
+
+base_snapshot_exists || exit
+create_staged_fs ubuntu
+start_staged_jail ubuntu
+install_ubuntu
+promote_staged_jail ubuntu