-
Notifications
You must be signed in to change notification settings - Fork 16
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Linux: add support for jailed Linuxulator & bhyve (#544)
- Loading branch information
Showing
6 changed files
with
324 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| #!/bin/sh | ||
|
|
||
| # see examples in provision/centos and provision/ubuntu | ||
|
|
||
| configure_linuxulator() | ||
| { | ||
| tell_status "enabling Linux emulation on Host (loads kernel modules)" | ||
| sysrc linux_enable=YES | ||
| sysrc linux_mounts_enable=NO | ||
| service linux start | ||
|
|
||
| tell_status "enabling Linux emulation in jail" | ||
| stage_sysrc linux_enable=YES | ||
| stage_sysrc linux_mounts_enable=NO | ||
| stage_exec service linux start | ||
| } | ||
|
|
||
| configure_apt_sources() | ||
| { | ||
| case "$1" in | ||
| bionic|focal|jammy) | ||
| tell_status "restoring APT sources" | ||
| tee "$STAGE_MNT/compat/linux/etc/apt/sources.list" <<EO_UB_SOURCES | ||
| deb http://archive.ubuntu.com/ubuntu $1 main universe restricted multiverse | ||
| deb http://security.ubuntu.com/ubuntu/ $1-security universe multiverse restricted main | ||
| deb http://archive.ubuntu.com/ubuntu $1-backports universe multiverse restricted main | ||
| deb http://archive.ubuntu.com/ubuntu $1-updates universe multiverse restricted main | ||
| EO_UB_SOURCES | ||
| ;; | ||
| bullseye) | ||
| tell_status "adding APT sources" | ||
| tee "$STAGE_MNT/compat/linux/etc/apt/sources.list" <<EO_DEB_SOURCES | ||
| deb http://deb.debian.org/debian $1 main contrib non-free | ||
| deb http://deb.debian.org/debian-security/ $1-security main contrib non-free | ||
| deb http://deb.debian.org/debian $1-updates main contrib non-free | ||
| deb http://deb.debian.org/debian $1-backports main contrib non-free | ||
| EO_DEB_SOURCES | ||
| esac | ||
| } | ||
|
|
||
| install_apt_updates() | ||
| { | ||
| tell_status "updating apt" | ||
| stage_exec chroot /compat/linux apt update || exit 1 | ||
|
|
||
| tell_status "updating installed apt packages" | ||
| stage_exec chroot /compat/linux apt upgrade -y || exit 1 | ||
| } | ||
|
|
||
| install_linux() | ||
| { | ||
| # tested with values of $1: | ||
| # Ubuntu: bionic (18), focal (20) and jammy (22) | ||
| # Debian: bullseye (11), bookwork (12) | ||
| # CentOS: centos (7) | ||
|
|
||
| configure_linuxulator | ||
|
|
||
| case "$1" in | ||
| centos) | ||
| tell_status "installing $1" | ||
| stage_pkg_install linux_base-c7 || exit 1 | ||
| ;; | ||
| bionic|bookworm|bullseye|focal|jammy) | ||
| tell_status "installing (debian|ubuntu) $1" | ||
| stage_pkg_install debootstrap || exit 1 | ||
| stage_exec debootstrap $1 /compat/linux | ||
| configure_apt_sources $1 | ||
| ;; | ||
| esac | ||
|
|
||
| case "$1" in | ||
| bionic) stage_exec chroot /compat/linux apt remove -y rsyslog ;; | ||
| jammy) stage_exec mount -t devfs devfs /compat/linux/dev ;; | ||
| esac | ||
|
|
||
| case "$1" in | ||
| bionic|focal|jammy|bullseye) install_apt_updates ;; | ||
| esac | ||
|
|
||
| case "$1" in | ||
| jammy) stage_exec umount /compat/linux/dev ;; | ||
| esac | ||
| } | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,144 @@ | ||
| #!/bin/sh | ||
|
|
||
| . mail-toaster.sh || exit | ||
|
|
||
| create_bridge() | ||
| { | ||
| if ! grep -q "tap.up_on_open" /etc/sysctl.conf; then | ||
| tell_status "setting tap.up_on_open" | ||
| sysctl net.link.tap.up_on_open=1 | ||
| echo "net.link.tap.up_on_open=1" >> /etc/sysctl.conf | ||
| fi | ||
|
|
||
| # create a named bridge for bhyve VMs | ||
| ifconfig bridge bridge-public 2>/dev/null || { | ||
| tell_status "creating bridge-public" | ||
| ifconfig bridge create name bridge-public | ||
| get_public_facing_nic | ||
| ifconfig bridge-public addm "$PUBLIC_NIC" | ||
| ifconfig bridge-public up | ||
| } | ||
|
|
||
| # create tap interface for VM | ||
| ifconfig tap-ubuntu 2>/dev/null || { | ||
| tell_status "creating VM tap interface" | ||
| ifconfig tap create name tap-ubuntu | ||
| ifconfig bridge-public addm tap-ubuntu | ||
| } | ||
|
|
||
| if ! grep -q if_bridge_load /boot/loader.conf; then | ||
| tell_status "enabling bridge & tap load at boot time" | ||
| sysrc -f /boot/loader.conf if_bridge_load=YES | ||
| sysrc -f /boot/loader.conf if_tap_load=YES | ||
| fi | ||
| } | ||
|
|
||
| configure_grub() | ||
| { | ||
| tee -a device.map <<EO_DMAP | ||
| (hd0) /dev/zvol/$ZFS_BHYVE_VOL/bhyve/ubuntu-guest | ||
| (cd0) /$ZFS_BHYVE_VOL/ISO/ubuntu-22.04.2-live-server-amd64.iso | ||
| EO_DMAP | ||
|
|
||
| tell_status "loading the Linux kernel" | ||
| grub-bhyve -m device.map -r cd0 -M 1024M ubuntu-guest | ||
|
|
||
| tee -a <<EO_GRUB | ||
| grub> ls | ||
| (hd0) (cd0) (cd0,msdos1) (host) | ||
| grub> ls (cd0)/isolinux | ||
| boot.cat boot.msg grub.conf initrd.img isolinux.bin isolinux.cfg memtest | ||
| splash.jpg TRANS.TBL vesamenu.c32 vmlinuz | ||
| grub> linux (cd0)/isolinux/vmlinuz | ||
| grub> initrd (cd0)/isolinux/initrd.img | ||
| grub> boot | ||
| EO_GRUB | ||
|
|
||
| # within Ubuntu VM | ||
| tee -a /etc/default/grub <<EO_DEFAULT_GRUB | ||
| GRUB_CMDLINE_LINUX_DEFAULT="" | ||
| GRUB_TERMINAL='serial console' | ||
| GRUB_CMDLINE_LINUX="console=hvc0 console=ttyS0,115200n8" | ||
| GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1" | ||
| EO_DEFAULT_GRUB | ||
|
|
||
| sudo update-grub | ||
| } | ||
|
|
||
| install_ubuntu_bhyve_zfs() | ||
| { | ||
| if ! zfs_filesystem_exists "$ZFS_BHYVE_VOL/bhyve"; then | ||
| zfs create "$ZFS_BHYVE_VOL/bhyve" | ||
| zfs set recordsize=64K "$ZFS_BHYVE_VOL/bhyve" | ||
| fi | ||
|
|
||
| if ! zfs_filesystem_exists "$ZFS_BHYVE_VOL/bhyve/ubuntu-guest"; then | ||
| zfs create -V20G -o volmode=dev "$ZFS_BHYVE_VOL/bhyve/ubuntu-guest" | ||
| fi | ||
| } | ||
|
|
||
| install_ubuntu_bhyve() | ||
| { | ||
| if ! grep -q vmm_load /boot/loader.conf; then | ||
| tell_status "loading kernel module: vmm" | ||
| kldstat vmm || kldload vmm || exit 1 | ||
| sysrc -f /boot/loader.conf vmm_load=YES | ||
| fi | ||
|
|
||
| create_bridge | ||
| install_ubuntu_bhyve_zfs | ||
|
|
||
| tell_status "installing bhyve" | ||
| stage_pkg_install bhyve-firmware grub2-bhyve || exit | ||
| #configure_grub | ||
|
|
||
| bhyve \ | ||
| -H -P -w \ | ||
| -c 1 -m 1G \ | ||
| -s 0:0,hostbridge \ | ||
| -s 2:0,virtio-net,tap-ubuntu \ | ||
| -s 3:0,ahci-cd,/$ZFS_BHYVE_VOL/ISO/ubuntu-22.04.2-live-server-amd64.iso \ | ||
| -s 4:0,virtio-blk,/dev/zvol/$ZFS_BHYVE_VOL/bhyve/ubuntu-guest \ | ||
| -s 29:0,fbuf,tcp=0.0.0.0:5900,w=800,h=600,wait \ | ||
| -s 30:0,xhci,tablet \ | ||
| -s 31:0,lpc \ | ||
| -l bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI.fd \ | ||
| -l com1,stdio \ | ||
| ubuntu-guest | ||
|
|
||
| bhyvectl --destroy --vm=ubuntu-guest | ||
| } | ||
|
|
||
|
|
||
| configure_ubuntu() | ||
| { | ||
| local _pdir="$STAGE_MNT/usr/local/etc/periodic" | ||
| } | ||
|
|
||
| start_ubuntu() | ||
| { | ||
| tell_status "starting up VM" | ||
| bhyve -AHP \ | ||
| -c 4 -m 1G \ | ||
| -s 0:0,hostbridge \ | ||
| -s 2:0,virtio-net,tap-ubuntu \ | ||
| -s 4:0,virtio-blk,/dev/zvol/$ZFS_BHYVE_VOL/bhyve/ubuntu-guest \ | ||
| -s 29:0,fbuf,tcp=0.0.0.0:5900,w=800,h=600 \ | ||
| -s 30:0,xhci,tablet \ | ||
| -s 31:0,lpc \ | ||
| -l com1,stdio \ | ||
| -l bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI.fd \ | ||
| ubuntu-guest | ||
|
|
||
| bhyvectl --destroy --vm=ubuntu-guest | ||
| } | ||
|
|
||
| test_ubuntu() | ||
| { | ||
| echo "hrmm, how to test?" | ||
| } | ||
|
|
||
| install_ubuntu_bhyve | ||
| #configure_ubuntu | ||
| #start_ubuntu | ||
| #test_ubuntu |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| #!/bin/sh | ||
|
|
||
| set -e -u | ||
|
|
||
| . mail-toaster.sh | ||
|
|
||
| mt6-include linux | ||
|
|
||
| export JAIL_START_EXTRA="allow.mount | ||
| allow.mount.devfs | ||
| allow.mount.fdescfs | ||
| allow.mount.procfs | ||
| allow.mount.linprocfs | ||
| allow.mount.linsysfs | ||
| allow.mount.tmpfs | ||
| enforce_statfs=1 | ||
| " | ||
| export JAIL_CONF_EXTRA=' | ||
| allow.raw_sockets;' | ||
| export JAIL_FSTAB=" | ||
| devfs $ZFS_JAIL_MNT/centos/compat/linux/dev devfs rw 0 0 | ||
| tmpfs $ZFS_JAIL_MNT/centos/compat/linux/dev/shm tmpfs rw,size=1g,mode=1777 0 0 | ||
| fdescfs $ZFS_JAIL_MNT/centos/compat/linux/dev/fd fdescfs rw,linrdlnk 0 0 | ||
| linprocfs $ZFS_JAIL_MNT/centos/compat/linux/proc linprocfs rw 0 0 | ||
| linsysfs $ZFS_JAIL_MNT/centos/compat/linux/sys linsysfs rw 0 0 | ||
| #/tmp $ZFS_JAIL_MNT/centos/compat/linux/tmp nullfs rw 0 0 | ||
| #/home $ZFS_JAIL_MNT/centos/compat/linux/home nullfs rw 0 0" | ||
|
|
||
| install_centos() | ||
| { | ||
| install_linux centos | ||
| } | ||
|
|
||
| base_snapshot_exists || exit 1 | ||
| create_staged_fs centos | ||
| for _fs in dev proc sys tmp home; do | ||
| mkdir -p "$ZFS_JAIL_MNT/stage/compat/linux/$_fs" | ||
| done | ||
| chmod 777 "$ZFS_JAIL_MNT/stage/compat/linux/tmp" | ||
| start_staged_jail centos | ||
| install_centos | ||
| promote_staged_jail centos |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| #!/bin/sh | ||
|
|
||
| set -e -u | ||
|
|
||
| . mail-toaster.sh | ||
|
|
||
| mt6-include linux | ||
|
|
||
| export JAIL_START_EXTRA="allow.mount | ||
| allow.mount.devfs | ||
| allow.mount.fdescfs | ||
| allow.mount.procfs | ||
| allow.mount.linprocfs | ||
| allow.mount.linsysfs | ||
| allow.mount.tmpfs | ||
| enforce_statfs=1 | ||
| " | ||
| export JAIL_CONF_EXTRA=' | ||
| allow.raw_sockets;' | ||
| export JAIL_FSTAB=" | ||
| devfs $ZFS_JAIL_MNT/ubuntu/compat/linux/dev devfs rw 0 0 | ||
| tmpfs $ZFS_JAIL_MNT/ubuntu/compat/linux/dev/shm tmpfs rw,size=1g,mode=1777 0 0 | ||
| fdescfs $ZFS_JAIL_MNT/ubuntu/compat/linux/dev/fd fdescfs rw,linrdlnk 0 0 | ||
| linprocfs $ZFS_JAIL_MNT/ubuntu/compat/linux/proc linprocfs rw 0 0 | ||
| linsysfs $ZFS_JAIL_MNT/ubuntu/compat/linux/sys linsysfs rw 0 0 | ||
| #/tmp $ZFS_JAIL_MNT/ubuntu/compat/linux/tmp nullfs rw 0 0 | ||
| #/home $ZFS_JAIL_MNT/ubuntu/compat/linux/home nullfs rw 0 0" | ||
|
|
||
| install_ubuntu() | ||
| { | ||
| install_linux jammy | ||
| } | ||
|
|
||
| base_snapshot_exists || exit 1 | ||
| create_staged_fs ubuntu | ||
| for _fs in dev proc sys tmp home; do | ||
| mkdir -p "$ZFS_JAIL_MNT/stage/compat/linux/$_fs" | ||
| done | ||
| chmod 777 "$ZFS_JAIL_MNT/stage/compat/linux/tmp" | ||
| start_staged_jail ubuntu | ||
| install_ubuntu | ||
| promote_staged_jail ubuntu |