From b178a0367c774e90a73d0caef1084f440d860d94 Mon Sep 17 00:00:00 2001
From: Ondrej Mosnacek <omosnace@redhat.com>
Date: Thu, 29 Aug 2024 16:40:13 +0200
Subject: [PATCH] ci: use Testing Farm instead of GH Actions + MacOS + Vagrant

The current solution to run a VM on MacOS shared runners using Vagrant
is becoming very unreliable and almost always breaks. Replace it with
Testing Farm [1], utilizing the "Schedule tests on Testing Farm" GH
Action [2].

Advantages:
 - more reliable
 - allows us to test also on aarch64
 - currently no usage limits for public projects (will likely change)

Disadvantages:
 - requires an API key to be stored in the project's secrets (under the
   name TESTING_FARM_API_TOKEN)
 - GitHub won't allow the target project's secrets to be used when
   running a workflow on a pull request from a fork, therefore PRs won't
   be automatically tested unless the contributor sets their own API key
   in their fork
 - only people with a Fedora account in the fedora-contributor group can
   currently obtain an API key on their own (others would need to ask
   for it via email)
 - no real-time view of test progress (may become available in the
   future via the artifacts view)

The new CI runs the testsuite on a similar testing matrix as the old
one, although it only tests on the latest Fedora version and
additionally tests on the aarch64 architecture. It also runs the NFS
tests (./tools/nfs.sh), which the old one didn't.

[1] https://testing-farm.io/
[2] https://github.com/marketplace/actions/schedule-tests-on-testing-farm

Signed-off-by: Ondrej Mosnacek <omosnace@redhat.com>
---
 .fmf/version                 |   1 +
 .github/workflows/checks.yml |  47 +++--------
 Vagrantfile                  | 102 ------------------------
 tmt/README.md                |  19 +++++
 tmt/plans.fmf                |  14 ++++
 tmt/tests.fmf                | 150 +++++++++++++++++++++++++++++++++++
 6 files changed, 196 insertions(+), 137 deletions(-)
 create mode 100644 .fmf/version
 delete mode 100644 Vagrantfile
 create mode 100644 tmt/README.md
 create mode 100644 tmt/plans.fmf
 create mode 100644 tmt/tests.fmf

diff --git a/.fmf/version b/.fmf/version
new file mode 100644
index 00000000..d00491fd
--- /dev/null
+++ b/.fmf/version
@@ -0,0 +1 @@
+1
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 6794eb74..858273d7 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -1,5 +1,6 @@
 name: checks
 on: [push, pull_request]
+
 jobs:
   style-check:
     runs-on: ubuntu-latest
@@ -10,43 +11,19 @@ jobs:
       - uses: actions/checkout@v4
       - run: sudo chown $(id -u):$(id -g) .
       - run: tools/check-syntax -f && git diff --exit-code
-  fedora-test:
-    runs-on: macos-12
+  run:
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         domain: [unconfined_t, sysadm_t]
-        env:
-          - { version: 38, kernel: default }
-          - { version: 39, kernel: default }
-          - { version: 39, kernel: secnext }
-    env:
-      FEDORA_VERSION: ${{ matrix.env.version }}
-      KERNEL_TYPE: ${{ matrix.env.kernel }}
-      ROOT_DOMAIN: ${{ matrix.domain }}
+        arch: [x86_64, aarch64]
+        kernel: [latest, secnext]
     steps:
-      - name: Install GNU coreutils
-        run: brew install coreutils
-      - uses: actions/checkout@v4
-      # macOS sometimes allows symlinks to have permissions other than 777,
-      # so change all symlink perms to match the Linux convention. Otherwise
-      # the rsync run by Vagrant will complain that it can't copy over the
-      # perms.
-      - name: Fix symlink permissions
-        run: find . -type link -exec chmod -h 777 \{\} \;
-      - name: Treat compiler warnings as errors
-        run: sed -i '' 's/-Wall/-Wall -Werror/' tests/Makefile
-      - name: Create a Vagrant VM
-        run: vagrant up
-      - name: Wait for the machine to come up if rebooting (max 5m)
-        run: gtimeout 5m "$SHELL" -c 'while ! vagrant ssh -- true; do sleep 1; done'
-      - name: Show Vagrant VM details
-        run: |
-          vagrant ssh -- uname -a
-          vagrant ssh -- cat /proc/cmdline
-      - name: Run SELinux testsuite
-        run: vagrant ssh -- sudo make -C /root/testsuite test
-      - name: Check unwanted denials
-        run: vagrant ssh -- '! sudo ausearch -m avc -i </dev/null | grep ${{ matrix.domain }}'
-      - name: Check .gitignore coverage
-        run: test "$(vagrant ssh -- sudo git -C /root/testsuite ls-files -o --exclude-standard | wc -l)" -eq 0
+      - name: Schedule test on Testing Farm
+        uses: sclorg/testing-farm-as-github-action@main
+        with:
+          api_key: ${{ secrets.TESTING_FARM_API_TOKEN }}
+          arch: ${{ matrix.arch }}
+          variables: STS_ROOT_DOMAIN=${{ matrix.domain }}; STS_KERNEL=${{ matrix.kernel }}
+          tmt_plan_filter: tag:ci
diff --git a/Vagrantfile b/Vagrantfile
deleted file mode 100644
index 682b8055..00000000
--- a/Vagrantfile
+++ /dev/null
@@ -1,102 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-# Vagrant configuration file which creates a virtual machine that can run the
-# test suite.
-#
-# To create a new virtual machine:
-#
-#    FEDORA_VERSION=33 vagrant up
-#
-# To launch tests (for example after modifications have been made):
-#
-#    vagrant rsync && vagrant ssh -- sudo make -C /root/testsuite test
-#
-# To destroy the virtual machine (for example to start again from a clean environment):
-#
-#    vagrant destroy
-
-# All Vagrant configuration is done below. The "2" in Vagrant.configure
-# configures the configuration version (we support older styles for
-# backwards compatibility). Please don't change it unless you know what
-# you're doing.
-Vagrant.configure("2") do |config|
-  config.vm.box = "fedora/#{ENV['FEDORA_VERSION']}-cloud-base"
-  config.vm.synced_folder ".", "/vagrant", disabled: true
-  config.vm.synced_folder ".", "/root/testsuite", type: "rsync",
-    # need to disable '--copy-links', which is in rsync__args by default
-    rsync__args: ["-vzra", "--delete"]
-
-  config.vm.provider "virtualbox" do |v|
-    v.memory = 4096
-  end
-  config.vm.provider "libvirt" do |v|
-    v.memory = 4096
-  end
-
-  case ENV['KERNEL_TYPE']
-  when 'default'
-    dnf_opts = ''
-    kernel_pkgs = 'kernel-devel-"$(uname -r)" kernel-modules-"$(uname -r)"'
-  when 'latest'
-    dnf_opts = ''
-    kernel_pkgs = 'kernel-devel kernel-modules'
-  when 'secnext'
-    dnf_opts = '--nogpgcheck --releasever rawhide --repofrompath kernel-secnext,https://repo.paul-moore.com/rawhide/x86_64'
-    kernel_pkgs = 'kernel-devel kernel-modules'
-  else
-    print("Invalid KERNEL_TYPE '#{ENV['KERNEL_TYPE']}'")
-    abort
-  end
-
-  extra_commands = ''
-  case ENV['ROOT_DOMAIN']
-  when 'unconfined_t'
-  when 'sysadm_t'
-    extra_commands = <<EOF
-      semanage boolean --modify --on ssh_sysadm_login
-      semanage login --modify -s sysadm_u root
-      semanage login --add -s sysadm_u -r s0-s0:c0.c1023 vagrant
-EOF
-  else
-    print("Invalid ROOT_DOMAIN '#{ENV['ROOT_DOMAIN']}'")
-    abort
-  end
-
-  config.vm.provision :shell, inline: <<SCRIPT
-    dnf install -y #{dnf_opts} \
-      --allowerasing \
-      --skip-broken \
-      git-core \
-      make \
-      perl-Test \
-      perl-Test-Harness \
-      perl-Test-Simple \
-      perl-lib \
-      selinux-policy-devel \
-      gcc \
-      libselinux-devel \
-      net-tools \
-      netlabel_tools \
-      nftables \
-      iptables \
-      lksctp-tools-devel \
-      attr \
-      libbpf-devel \
-      keyutils-libs-devel \
-      quota \
-      xfsprogs-devel \
-      libuuid-devel \
-      e2fsprogs \
-      jfsutils \
-      dosfstools \
-      rdma-core-devel \
-      #{kernel_pkgs}
-
-    #{extra_commands}
-
-    # for secretmem test
-    grubby --update-kernel=ALL --args=secretmem.enable=1
-
-    reboot
-SCRIPT
-end
diff --git a/tmt/README.md b/tmt/README.md
new file mode 100644
index 00000000..b05cef06
--- /dev/null
+++ b/tmt/README.md
@@ -0,0 +1,19 @@
+# TMT test plans for selinux-testsuite
+
+This directory contains basic "test plans" for running the selinux-testsuite via the [TMT tool](https://tmt.readthedocs.io/en/stable/). They are primarily intended for the GitHub-Actions-driven CI, but they can be also used directly through `tmt`:
+
+```bash
+tmt run [-e STS_ROOT_DOMAIN=...] [-e STS_KERNEL=...] \
+    plans -f 'tag:-ci' --all provision -h ...
+```
+
+See `tmt run provision --help` for information about possible provisioning methods (most useful are `local`, `connect`, or `virtual.testcloud`).
+
+Possible values for the `STS_ROOT_DOMAIN` env parameter are:
+* `unconfined_t` - run the testsuite as an unconfined root.
+* `sysadm_t` - run the testsuite as a `sysadm_u:sysadm_r:sysadm_t:...` root.
+
+Possible values for the `STS_KERNEL` env parameter are:
+* `default` - try to use the kernel currently booted on the test machine.
+* `latest` - update to the latest kernel available in the repos and boot it.
+* `secnext` - install the "secnext" kernel from https://repo.paul-moore.com/ and boot it.
diff --git a/tmt/plans.fmf b/tmt/plans.fmf
new file mode 100644
index 00000000..52315f75
--- /dev/null
+++ b/tmt/plans.fmf
@@ -0,0 +1,14 @@
+/run:
+  summary: Run the testsuite
+  discover:
+    how: fmf
+    filter: tag:-ci
+  execute:
+    how: tmt
+/run-ci:
+  summary: Run the CI for the testsuite
+  tag: [ci]
+  discover:
+    how: fmf
+  execute:
+    how: tmt
diff --git a/tmt/tests.fmf b/tmt/tests.fmf
new file mode 100644
index 00000000..80f9e8f4
--- /dev/null
+++ b/tmt/tests.fmf
@@ -0,0 +1,150 @@
+# can't do this in the prepare step because of
+# https://github.com/teemtee/tmt/issues/902
+/prepare-system:
+  summary: Prepare the system for testing
+  order: 1
+  require: policycoreutils-python-utils
+  test: |
+    set -ex
+
+    if [ "$TMT_REBOOT_COUNT" -eq 0 ]; then
+      reboot=0
+
+      # Work around systemd/ssh bug
+      # https://github.com/fedora-selinux/selinux-policy/pull/2330
+      {
+        echo '(allow init_t sshd_t (fd (use)))'
+        echo '(allow systemd_logind_t sshd_t (fd (use)))'
+      } >ssh_workaround.cil
+      semodule -i ssh_workaround.cil
+      rm -f ssh_workaround.cil
+
+      case "$STS_ROOT_DOMAIN" in
+      unconfined_t|'')
+        ;;
+      sysadm_t)
+        semanage boolean --modify --on ssh_sysadm_login
+        semanage login --modify -s sysadm_u root
+
+        # Work around missing policy for sysadm_t
+        # https://github.com/fedora-selinux/selinux-policy/pull/2340
+        echo '(allow sysadm_t self (key_socket (create ioctl read getattr lock write setattr append bind connect getopt setopt shutdown)))' >sysadm_workaround.cil
+        semodule -i sysadm_workaround.cil
+        rm -f sysadm_workaround.cil
+
+        reboot=1
+        ;;
+      *)
+        echo "Invalid STS_ROOT_DOMAIN value!"
+        exit 1
+        ;;
+      esac
+
+      case "$STS_KERNEL" in
+      default|'')
+        dnf install -y kernel-modules-$(uname -r) kernel-devel-$(uname -r)
+        ;;
+      latest)
+        dnf install -y kernel-modules kernel-devel
+        ;;
+      secnext)
+        dnf install -y --disablerepo testing-farm-tag-repository \
+          --nogpgcheck --releasever rawhide \
+          --repofrompath 'kernel-secnext,https://repo.paul-moore.com/rawhide/$basearch' \
+          kernel-modules kernel-devel
+        ;;
+      *)
+        echo "Invalid STS_KERNEL value!"
+        exit 1
+        ;;
+      esac
+
+      if [ "$(grubby --default-kernel)" != "/boot/vmlinuz-$(uname -r)" ]; then
+        reboot=1
+      fi
+
+      if [ "$reboot" -ne 0 ]; then
+        tmt-reboot
+      fi
+    fi
+
+    # reset the audit log
+    :>/var/log/audit.log
+    rm -f /var/log/audit.log.*
+/machine-info:
+  summary: Show machine info
+  order: 2
+  test: |
+    set -x
+
+    uname -r
+    id -Z
+    rpm -q libselinux
+    rpm -q selinux-policy
+    lscpu
+/run:
+  summary: Run the testsuite
+  order: 3
+  duration: 20m
+  require:
+    - make
+    - perl-Test
+    - perl-Test-Harness
+    - perl-Test-Simple
+    - perl-lib
+    - selinux-policy-devel
+    - gcc
+    - libselinux-devel
+    - net-tools
+    - netlabel_tools
+    - iptables
+    - nftables
+    - lksctp-tools-devel
+    - attr
+    - libbpf-devel
+    - keyutils-libs-devel
+    - quota
+    - xfsprogs-devel
+    - libuuid-devel
+    - e2fsprogs
+    - jfsutils
+    - dosfstools
+    - rdma-core-devel
+  test: make -C .. test
+/run-nfs:
+  summary: Run the NFS tests
+  order: 3
+  duration: 25m
+  require: nfs-utils
+  test: env -C .. bash -x ./tools/nfs.sh
+/avc-check:
+  summary: Check unwanted denials
+  order: 4
+  tag: [ci]
+  test: '! ausearch -m avc -i </dev/null | grep "$STS_ROOT_DOMAIN"'
+/gitignore-coverage:
+  summary: Check .gitignore coverage
+  order: 4
+  tag: [ci]
+  require: git-core
+  test: |
+    # exclude tmt's litter
+    mkdir -p "$HOME/.config/git"
+    echo "tmt-*" >"$HOME/.config/git/ignore"
+
+    git -C .. ls-files -o --exclude-standard
+    test "$(git -C .. ls-files -o --exclude-standard | wc -l)" -eq 0
+/unprepare:
+  summary: Undo the preparation
+  order: 5
+  require: policycoreutils-python-utils
+  test: |
+    set -ex
+
+    make -C ../policy unload || true
+    if [ "$STS_ROOT_DOMAIN" = sysadm_t ]; then
+      semanage boolean --modify --off ssh_sysadm_login
+      semanage login --modify -s unconfined_u root
+      semodule -r sysadm_workaround
+    fi
+    semodule -r ssh_workaround