diff --git a/.github/workflows/gcp-l1-dev.yml b/.github/workflows/gcp-l1-dev.yml index 408070d..c932683 100644 --- a/.github/workflows/gcp-l1-dev.yml +++ b/.github/workflows/gcp-l1-dev.yml @@ -1,5 +1,5 @@ -run-name: GCP - Dev -name: GCP - Dev +run-name: GCP - ${{ github.event.inputs.config }} +name: FXCI - GCP on: workflow_dispatch: @@ -8,8 +8,10 @@ on: type: choice description: Choose which pool to build options: - - ubuntu2404-amd64-alpha - gw-fxci-gcp-l1 + - gw-fxci-gcp-l1-gui + - gw-fxci-gcp-l1-arm64 + - gw-fxci-gcp-l1-arm64-gui permissions: id-token: write diff --git a/config/gw-fxci-gcp-l1-arm64-gui.yaml b/config/gw-fxci-gcp-l1-arm64-gui.yaml new file mode 100644 index 0000000..e69de29 diff --git a/config/gw-fxci-gcp-l1-arm64.yaml b/config/gw-fxci-gcp-l1-arm64.yaml new file mode 100644 index 0000000..e69de29 diff --git a/config/ubuntu2404-amd64-alpha.yaml b/config/gw-fxci-gcp-l1-gui.yaml similarity index 61% rename from config/ubuntu2404-amd64-alpha.yaml rename to config/gw-fxci-gcp-l1-gui.yaml index 22b6054..af309d4 100644 --- a/config/ubuntu2404-amd64-alpha.yaml +++ b/config/gw-fxci-gcp-l1-gui.yaml @@ -1,11 +1,11 @@ --- image: - source_image_family: ubuntu-2404-lts-amd64 + source_image_family: ubuntu-2204-lts project_id: "taskcluster-imaging" workload_identity_provider_project_number: 324168772199 - image_name: ubuntu2404-amd64-alpha + image_name: gw-fxci-gcp-l1-gui-googlecompute-alpha zone: us-west1-a vm: disk_size: 100 taskcluster_version: 70.0.0 - tc_arch: AMD64 + tc_arch: AMD64 \ No newline at end of file diff --git a/config/gw-fxci-gcp-l1.yaml b/config/gw-fxci-gcp-l1.yaml index e79a344..17f5812 100644 --- a/config/gw-fxci-gcp-l1.yaml +++ b/config/gw-fxci-gcp-l1.yaml @@ -3,7 +3,7 @@ image: source_image_family: ubuntu-2204-lts project_id: "taskcluster-imaging" workload_identity_provider_project_number: 324168772199 - image_name: gw-fxci-gcp-l1-gui-googlecompute-alpha + image_name: gw-fxci-gcp-l1-googlecompute-alpha zone: us-west1-a vm: disk_size: 100 diff --git a/gcp.pkr.hcl b/gcp.pkr.hcl index 6675a48..f8748b8 100644 --- a/gcp.pkr.hcl +++ b/gcp.pkr.hcl @@ -78,7 +78,7 @@ source "googlecompute" "gw-fxci-gcp-l1" { use_iap = true } -source "googlecompute" "ubuntu2204gw" { +source "googlecompute" "gw-fxci-gcp-l1-gui" { disk_size = var.disk_size image_licenses = ["projects/vm-options/global/licenses/enable-vmx"] image_name = var.image_name @@ -166,8 +166,9 @@ build { build { sources = [ - "source.googlecompute.ubuntu2204gw" + "source.googlecompute.gw-fxci-gcp-l1-gui" ] + ## Every image has tests, so create the tests directory provisioner "shell" { execute_command = "sudo -S sh -c '{{ .Vars }} {{ .Path }}'" @@ -183,54 +184,23 @@ build { destination = "/workerimages/tests/taskcluster.tests.ps1" } - ## Do we need these secrets? - // provisioner "shell" { - // execute_command = "sudo -S sh -c '{{ .Vars }} {{ .Path }}'" - // inline = [ - // "mkdir -p /etc/taskcluster/secrets", - // "touch /etc/taskcluster/secrets/worker_env_var_key", - // "touch /etc/taskcluster/secrets/worker_livelog_tls_cert", - // "touch /etc/taskcluster/secrets/worker_livelog_tls_key", - // "chmod +x /etc/taskcluster/secrets/worker_env_var_key", - // "chmod +x /etc/taskcluster/secrets/worker_livelog_tls_cert", - // "chmod +x /etc/taskcluster/secrets/worker_livelog_tls_key", - // ] - // } - provisioner "shell" { execute_command = "sudo -S bash -c '{{ .Vars }} {{ .Path }}'" environment_vars = [ "CLOUD=google", "TC_ARCH=${var.tc_arch}", "TASKCLUSTER_VERSION=${var.taskcluster_version}", + "NUM_LOOPBACK_AUDIO_DEVICES=8" ] + expect_disconnect = true scripts = [ - "${path.cwd}/scripts/linux/ubuntu-community-2404-bootstrap/bootstrap.sh" + "${path.cwd}/scripts/linux/ubuntu-jammy-from-community-gui/05-install.sh", + "${path.cwd}/scripts/linux/ubuntu-jammy-from-community-gui/50-wayland_errata.sh", + "${path.cwd}/scripts/linux/ubuntu-jammy-from-community-gui/60-reboot.sh", + "${path.cwd}/scripts/linux/ubuntu-jammy-from-community-gui/70-additional-talos-reqs.sh" ] } - # Do we need these secrets? - // provisioner "shell" { - // execute_command = "sudo -S bash -c '{{ .Vars }} {{ .Path }}'" - // environment_vars = [ - // "WORKER_ENV_VAR_KEY=${var.worker_env_var_key}", - // "TC_WORKER_CERT=${var.tc_worker_cert}", - // "TC_WORKER_KEY=${var.tc_worker_key}" - // ] - // scripts = [ - // "${path.cwd}/scripts/linux/taskcluster/tc.sh" - // ] - // } - - # Do we need these secrets? - // provisioner "shell" { - // execute_command = "sudo -S sh -c '{{ .Vars }} {{ .Path }}'" - // inline = [ - // "chown root:root -R /etc/taskcluster", - // "chmod 0400 -R /etc/taskcluster/secrets" - // ] - // } - provisioner "shell" { inline = ["/usr/bin/cloud-init status --wait"] } @@ -266,4 +236,5 @@ build { output = "packer-artifacts.json" strip_path = true } + } diff --git a/scripts/linux/ubuntu-jammy-from-community-gui/05-install.sh b/scripts/linux/ubuntu-jammy-from-community-gui/05-install.sh new file mode 100644 index 0000000..bfb6c7c --- /dev/null +++ b/scripts/linux/ubuntu-jammy-from-community-gui/05-install.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +set -exv + +# init helpers +function retry { + set +e + local n=0 + local max=10 + while true; do + "$@" && break || { + if [[ $n -lt $max ]]; then + ((n++)) + echo "Command failed" >&2 + sleep_time=$((2 ** n)) + echo "Sleeping $sleep_time seconds..." >&2 + sleep $sleep_time + echo "Attempt $n/$max:" >&2 + else + echo "Failed after $n attempts." >&2 + exit 1 + fi + } + done + set -e +} + +# taken from https://github.com/taskcluster/community-tc-config/blob/main/imagesets/generic-worker-ubuntu-22-04/bootstrap.sh + +# AJE added +export DEBIAN_FRONTEND=noninteractive + +# place a new worker unit file that is required by the graphical target +cat > /lib/systemd/system/worker.service << EOF +[Unit] +Description=Start TC worker + +[Service] +Type=simple +ExecStart=/usr/local/bin/start-worker /etc/start-worker.yml +# log to console to make output visible in cloud consoles, and syslog for ease of +# redirecting to external logging services +StandardOutput=syslog+console +StandardError=syslog+console +User=root + +[Install] +RequiredBy=graphical.target +EOF + +# podman installed in non-gui +retry apt-get install -y ubuntu-desktop ubuntu-gnome-desktop + +# Installs the v4l2loopback kernel module +# used for the video device, and vkms +# required by Wayland +retry apt-get install -y linux-modules-extra-$(uname -r) +# needed for mutter to work with DRM rather than falling back to X11 +grep -Fx vkms /etc/modules || echo vkms >> /etc/modules +# disable udev rule that tags platform-vkms with "mutter-device-ignore" +# ENV{ID_PATH}=="platform-vkms", TAG+="mutter-device-ignore" +sed '/platform-vkms/d' /lib/udev/rules.d/61-mutter.rules > /etc/udev/rules.d/61-mutter.rules + +# vnc configuration omitted +# - see https://github.com/taskcluster/community-tc-config/blob/5431d9f72f52eeb2bb232dcac55ad399f747ac6a/imagesets/generic-worker-ubuntu-22-04-staging/bootstrap.sh + +# use fc-cache:i386 to pre-build the font cache for i386 binaries +# i386 line: apt-get -q -y -f install fontconfig:i386 +# TODO: do we need to specify arch here? +apt-get -q -y -f install fontconfig diff --git a/scripts/linux/ubuntu-jammy-from-community-gui/50-wayland_errata.sh b/scripts/linux/ubuntu-jammy-from-community-gui/50-wayland_errata.sh new file mode 100644 index 0000000..3d2dff4 --- /dev/null +++ b/scripts/linux/ubuntu-jammy-from-community-gui/50-wayland_errata.sh @@ -0,0 +1,137 @@ +#!/bin/bash + +set -exv + +# init helpers +helpers_dir=${MONOPACKER_HELPERS_DIR:-"/etc/monopacker/scripts"} +for h in ${helpers_dir}/*.sh; do + . $h; +done + +# see https://github.com/mozilla-platform-ops/monopacker/issues/138 + + +# +# install tools +# + +# used to modify specific blocks in .conf files +apt install -y crudini + + +# +# dconf settings +# + +cat > /etc/dconf/profile/user << EOF +user-db:user +system-db:local +EOF + +mkdir /etc/dconf/db/local.d/ +# dconf user settings +cat > /etc/dconf/db/local.d/00-tc-gnome-settings << EOF +# /org/gnome/desktop/session/idle-delay +[org/gnome/desktop/session] +idle-delay=uint32 0 + +# /org/gnome/desktop/lockdown/disable-lock-screen +[org/gnome/desktop/lockdown] +disable-lock-screen=true +EOF + +# make dbus read the new configuration +sudo dconf update + +# test +ls -hal /etc/dconf/db/ + + +# +# gdm3 settings +# + +# in [daemon] block of /etc/gdm3/custom.conf we need: +# +# XorgEnable=false + +crudini --set /etc/gdm3/custom.conf daemon XorgEnable 'false' + +# verify/test +cat /etc/gdm3/custom.conf +echo "----" +grep 'XorgEnable' /etc/gdm3/custom.conf +grep 'XorgEnable' /etc/gdm3/custom.conf | grep false + + +# +# gdm wait service file +# + +# This hack is required because without we end up in a situation where the +# wayland seat is in a weird state and consequences are: +# - either x11 session +# - either xwayland fallback +# - either wayland but with missing keyboard capability that breaks +# things including copy/paste + +mkdir -p /etc/systemd/system/gdm.service.d/ +cat > /etc/systemd/system/gdm.service.d/gdm-wait.conf << EOF +[Unit] +Description=Extra 30s wait + +[Service] +ExecStartPre=/bin/sleep 30 +EOF + + +# +# extra packages +# + +# ttf-mscorefonts-installer is part of ubuntu-restricted-extras, accept license +echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections + +# install stuff +apt install -y \ + gnome-screenshot \ + ubuntu-restricted-extras + +# test that gnome-screenshot is present +ls -la "$(which gnome-screenshot)" +ls -la /usr/bin/gnome-screenshot + +# +# write mutter's monitors.xml +# + +cat > /etc/xdg/monitors.xml << EOF + + + + 0 + 0 + 1 + yes + + + Virtual-1 + unknown + unknown + unknown + + + 1920 + 1080 + 60.000 + + + + + +EOF + +# +# verify that the XDG_RUNTIME_DIR fix is in the g-w binary we're using +# +strings /usr/local/bin/generic-worker | grep XDG_RUNTIME_DIR diff --git a/scripts/linux/ubuntu-jammy-from-community-gui/60-reboot.sh b/scripts/linux/ubuntu-jammy-from-community-gui/60-reboot.sh new file mode 100644 index 0000000..e66d7cc --- /dev/null +++ b/scripts/linux/ubuntu-jammy-from-community-gui/60-reboot.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -exv + +# +# reboot to use the new kernel +# +shutdown -r now \ No newline at end of file diff --git a/scripts/linux/ubuntu-jammy-from-community-gui/70-additional-talos-reqs.sh b/scripts/linux/ubuntu-jammy-from-community-gui/70-additional-talos-reqs.sh new file mode 100644 index 0000000..9bcec0b --- /dev/null +++ b/scripts/linux/ubuntu-jammy-from-community-gui/70-additional-talos-reqs.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +set -exv + +# needed for fxci talos/raptor tests + +# no apt/deb prompts +export DEBIAN_FRONTEND=noninteractive + +# +# install kernel headers +# + +# problem: broken symlinks in directory below +# ls -la ls /usr/src/linux-headers-5.15.0-1030-gcp +# fix: +# sudo apt reinstall linux-gcp-headers-5.15.0-1030 +# +# ensure kernel headers are present so dkms works +# - had issue where there were broken symlinks + +version=`uname -r` +version_minus_dash_gcp=`uname -r | sed -r s/-gcp//` +short_version=`uname -r | cut -d "." -f1,2` +pkg_name="linux-gcp-${short_version}-headers-${version_minus_dash_gcp}" + +sudo apt-get update +sudo apt-get -y reinstall linux-headers-gcp linux-headers-`uname -r` ${pkg_name} + + +# +# apt packages +# + +# pre-reqs +apt-get install -y dkms kmod llvm sox libxcb1 nodejs xvfb apt-utils +# not working: linux-headers +# missing: lib32ncurses5 gstreamer + + +# +# install v4l2loopback +# +apt-get install -y v4l2loopback-dkms v4l2loopback-utils +# verify +dkms status + + +# +# enable v4loopback +# + +# required on 22.04? +# +# if [[ "$BUILD_V4L2LOOPBACK" ]]; then +# # This is for Ubuntu 18.04 in GCP. We have to build the module, otherwise it will not work. +# V4L2LOOPBACK_VERSION=${V4L2LOOPBACK_VERSION:-0.12.5} +# git clone -b v$V4L2LOOPBACK_VERSION https://github.com/umlaeute/v4l2loopback /usr/src/v4l2loopback-$V4L2LOOPBACK_VERSION +# # Edit the file `v4l2looback.c` and change the `MAX_DEVICES` definition to `100` +# # (NOTE: ignore the comments about overriding it in a `make` invocation; this isn't possible via dkms) +# sed -i -e "s/# *define MAX_DEVICES *[0-9]*/# define MAX_DEVICES $NUM_LOOPBACK_VIDEO_DEVICES/g" /usr/src/v4l2loopback-$V4L2LOOPBACK_VERSION/v4l2loopback.c +# dkms install -m v4l2loopback -v $V4L2LOOPBACK_VERSION +# fi + +# Required in GCP. +apt-get install linux-modules-extra-gcp -y + +# Configure video loopback devices +echo "options v4l2loopback devices=$NUM_LOOPBACK_VIDEO_DEVICES" > /etc/modprobe.d/v4l2loopback.conf +echo "videodev" | tee --append /etc/modules +echo "v4l2loopback" | tee --append /etc/modules + +# test the results + +modprobe videodev +lsmod | grep videodev + +modprobe v4l2loopback +lsmod | grep v4l2loopback +# currently failing... only 7 devices... /dev/video7 +test -e /dev/video$((NUM_LOOPBACK_VIDEO_DEVICES - 1)) + + +# +# configure audio loopback devices +# + +# Configure audio loopback devices, with options enable=1,1,1...,1 index = 0,1,...,N +i=0 +enable='' +index='' +while [ $i -lt ${NUM_LOOPBACK_AUDIO_DEVICES} ]; do + enable="$enable,1" + index="$index,$i" + i=$((i + 1)) +done +# slice off the leading `,` in each variable +enable=${enable:1} +index=${index:1} + +echo "options snd-aloop enable=$enable index=$index" > /etc/modprobe.d/snd-aloop.conf +echo "snd-aloop" | tee --append /etc/modules + +# test +modprobe snd-aloop +lsmod | grep snd_aloop +test -e /dev/snd/controlC$((NUM_LOOPBACK_AUDIO_DEVICES - 1)) + +# +# directories expected by talos +# +dirs="/builds /builds/slave /builds/slave/talos-data /builds/slave/talos-data/talos \ + /builds/git-shared /builds/hg-shared /builds/tooltool_cache" + +mkdir -p $dirs +# task user changes... set to root for now +chown -R root:root $dirs +chmod -R 0777 $dirs + +# test +ls -lad $dirs \ No newline at end of file