workflow: move build operations in a reusable workflow

.github/scripts/matrix.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+from copy import deepcopy
 from json import dumps
 from enum import Enum
 import os
@@ -62,6 +63,7 @@ def get_tests(config):
         "arch": Arch.X86_64.value,
         "toolchain": "gcc",
         "llvm-version": "16",
+        "run_veristat": True,
     },
     {
         "kernel": "LATEST",
@@ -104,38 +106,32 @@ def get_tests(config):
     else:
         matrix[idx]["toolchain_full"] = "llvm-" + matrix[idx]["llvm-version"]
 
+    # Set run_veristat to false by default.
+    matrix[idx]["run_veristat"] = matrix[idx].get("run_veristat", False)
+    # Feel in the tests
+    matrix[idx]["tests"] = {
+        "include": [generate_test_config(test) for test in get_tests(matrix[idx])]
+    }
+
 # Only a few repository within "kernel-patches" use self-hosted runners.
 if (
     os.environ["GITHUB_REPOSITORY_OWNER"] != "kernel-patches"
     or os.environ["GITHUB_REPOSITORY"] not in self_hosted_repos
 ):
-    # Outside of those repositories, we only run on x86_64 GH hosted runners (ubuntu-latest)
+    # Outside of those repositories, we only run on x86_64 GH hosted runners (ubuntu-20.04)
+    # We need to run on ubuntu 20.04 because our rootfs is based on debian buster and we
+    # otherwise get library versioning issue such as
+    # `./test_verifier: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found (required by ./test_verifier)`
     for idx in range(len(matrix) - 1, -1, -1):
         if matrix[idx]["arch"] != Arch.X86_64.value:
             del matrix[idx]
         else:
-            matrix[idx]["runs_on"] = ["ubuntu-latest"]
+            matrix[idx]["runs_on"] = ["ubuntu-20.04"]
 else:
     # Otherwise, run on (self-hosted, arch) runners
     for idx in range(len(matrix) - 1, -1, -1):
         matrix[idx]["runs_on"].extend(["self-hosted", matrix[idx]["arch"]])
 
 build_matrix = {"include": matrix}
-set_output("build_matrix", dumps(build_matrix))
 
-test_matrix = {
-    "include": [
-        {**config, **generate_test_config(test)}
-        for config in matrix
-        for test in get_tests(config)
-    ]
-}
-set_output("test_matrix", dumps(test_matrix))
-
-veristat_runs_on = next(
-    x["runs_on"]
-    for x in matrix
-    if x["arch"] == os.environ["veristat_arch"]
-    and x["toolchain"] == os.environ["veristat_toolchain"]
-)
-set_output("veristat_runs_on", veristat_runs_on)
+set_output("build_matrix", dumps(build_matrix))

.github/scripts/veristat-compare.py

@@ -52,17 +52,22 @@
 SUMMARY_HEADERS = ["File", "Program", "Verdict", "States Diff (%)"]
 
 # expected format: +0 (+0.00%) / -0 (-0.00%)
-TOTAL_STATES_DIFF_REGEX = \
+TOTAL_STATES_DIFF_REGEX = (
     r"(?P<absolute_diff>[+-]\d+) \((?P<percentage_diff>[+-]\d+\.\d+)\%\)"
+)
 
 
-TEXT_SUMMARY_TEMPLATE: Final[str] = """
+TEXT_SUMMARY_TEMPLATE: Final[
+    str
+] = """
 # {title}
 
 {table}
 """.strip()
 
-HTML_SUMMARY_TEMPLATE: Final[str] = """
+HTML_SUMMARY_TEMPLATE: Final[
+    str
+] = """
 # {title}
 
 <details>
@@ -135,21 +140,17 @@ def get_results_summary(self, markup: bool = False) -> str:
 
 
 def get_state_diff(value: str) -> float:
-    if value == 'N/A':
+    if value == "N/A":
         return 0.0
 
     matches = re.match(TOTAL_STATES_DIFF_REGEX, value)
     if not matches:
-        raise ValueError(
-            f"Failed to parse total states diff field value '{value}'"
-        )
+        raise ValueError(f"Failed to parse total states diff field value '{value}'")
 
     if percentage_diff := matches.group("percentage_diff"):
         return float(percentage_diff)
 
-    raise ValueError(
-        f"Invalid {VeristatFields.TOTAL_STATES_DIFF} field value: {value}"
-    )
+    raise ValueError(f"Invalid {VeristatFields.TOTAL_STATES_DIFF} field value: {value}")
 
 
 def parse_table(csv_file):
@@ -190,18 +191,16 @@ def parse_table(csv_file):
         if not add:
             continue
 
-        table.append([
-            record[VeristatFields.FILE_NAME],
-            record[VeristatFields.PROG_NAME],
-            verdict,
-            f"{diff:+.2f} %"
-        ])
-
-    return VeristatInfo(
-        table=table,
-        changes=changes,
-        new_failures=new_failures
-    )
+        table.append(
+            [
+                record[VeristatFields.FILE_NAME],
+                record[VeristatFields.PROG_NAME],
+                verdict,
+                f"{diff:+.2f} %",
+            ]
+        )
+
+    return VeristatInfo(table=table, changes=changes, new_failures=new_failures)
 
 
 def github_markup_decorate(input_str: str) -> str:
@@ -219,8 +218,7 @@ def format_table(headers: List[str], rows: List[List[str]]) -> str:
     # Row template string in the following format:
     # "{0:8}|{1:10}|{2:15}|{3:7}|{4:10}"
     row_template = "|".join(
-        f"{{{idx}:{width}}}"
-        for idx, width in enumerate(column_width)
+        f"{{{idx}:{width}}}" for idx, width in enumerate(column_width)
     )
     row_template_nl = f"|{row_template}|\n"
 
@@ -237,10 +235,7 @@ def format_table(headers: List[str], rows: List[List[str]]) -> str:
         return out.getvalue()
 
 
-def main(
-    compare_csv_filename: os.PathLike,
-    output_filename: os.PathLike
-) -> None:
+def main(compare_csv_filename: os.PathLike, output_filename: os.PathLike) -> None:
     with open(compare_csv_filename, newline="", encoding="utf-8") as csv_file:
         veristat_results = parse_table(csv_file)
 

.github/workflows/kernel-build-test.yml

@@ -0,0 +1,85 @@
+name: Reusable Build/Test/Veristat workflow
+
+on:
+  workflow_call:
+    inputs:
+      arch:
+        required: true
+        type: string
+        description: The architecture to build against, e.g x86_64, aarch64, s390x...
+      toolchain_full:
+        required: true
+        type: string
+        description: The toolchain and for llvm, its version, e.g gcc, llvm-15
+      toolchain:
+        required: true
+        type: string
+        description: The toolchain, e.g gcc, llvm
+      runs_on:
+        required: true
+        type: string
+        description: The runners to run the test on. This is a json string representing an array of labels.
+      llvm-version:
+        required: true
+        type: string
+        description: The version of LLVM used to build selftest.... for llvm toolchain, this should match the one from toolchain_full, for gcc it is an arbritrary version we decide to build selftests against.
+      kernel:
+        required: true
+        type: string
+        description: The kernel to run the test against. For KPD this is always LATEST, which runs against a newly built kernel.
+      tests:
+        required: true
+        type: string
+        description: A serialized json array with the tests to be running, it must follow the json-matrix format, https://www.jitsejan.com/use-github-actions-with-json-file-as-matrix
+      run_veristat:
+        required: true
+        type: boolean
+        description: Whether or not to run veristat
+    secrets:
+      AWS_ROLE_ARN:
+        required: true
+
+jobs:
+  # Build kernel and selftest
+  build:
+    uses: ./.github/workflows/kernel-build.yml
+    with:
+      arch: ${{ inputs.arch }}
+      toolchain_full: ${{ inputs.toolchain_full }}
+      toolchain: ${{ inputs.toolchain }}
+      runs_on: ${{ inputs.runs_on }}
+      llvm-version: ${{ inputs.llvm-version }}
+      kernel: ${{ inputs.kernel }}
+
+  test:
+    uses: ./.github/workflows/kernel-test.yml
+    # Setting name to test here to avoid lengthy autogenerated names due to matrix
+    # e.g build-and-test x86_64-gcc / test (test_progs_parallel, true, 30) / test_progs_parallel on x86_64 with gcc
+    name: "test"
+    needs: [build]
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(inputs.tests) }}
+    with:
+      arch: ${{ inputs.arch }}
+      toolchain_full: ${{ inputs.toolchain_full }}
+      runs_on: ${{ inputs.runs_on }}
+      kernel: ${{ inputs.kernel }}
+      test: ${{ matrix.test }}
+      continue_on_error: ${{ toJSON(matrix.continue_on_error) }}
+      timeout_minutes: ${{ matrix.timeout_minutes }}
+
+  veristat:
+    if: ${{ inputs.run_veristat }}
+    uses: ./.github/workflows/kernel-veristat.yml
+    needs: [build]
+    permissions:
+      id-token: write
+      contents: read
+    with:
+      arch: ${{ inputs.arch }}
+      toolchain: ${{ inputs.toolchain }}
+      aws_region: ${{ vars.AWS_REGION }}
+      runs_on: ${{ inputs.runs_on }}
+    secrets:
+      AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }}

.github/workflows/kernel-build.yml

@@ -0,0 +1,124 @@
+
+name: Reusable build workflow
+
+on:
+  workflow_call:
+    inputs:
+      arch:
+        required: true
+        type: string
+        description: The architecture to build against, e.g x86_64, aarch64, s390x...
+      toolchain_full:
+        required: true
+        type: string
+        description: The toolchain and for llvm, its version, e.g gcc, llvm-15
+      toolchain:
+        required: true
+        type: string
+        description: The toolchain, e.g gcc, llvm
+      runs_on:
+        required: true
+        type: string
+        description: The runners to run the test on. This is a json string representing an array of labels.
+      llvm-version:
+        required: true
+        type: string
+        description: The version of LLVM used to build selftest.... for llvm toolchain, this should match the one from toolchain_full, for gcc it is an arbritrary version we decide to build selftests against.
+      kernel:
+        required: true
+        type: string
+        description: The kernel to run the test against. For KPD this is always LATEST, which runs against a newly built kernel.
+
+
+jobs:
+  build:
+    name: build for ${{ inputs.arch }} with ${{ inputs.toolchain_full }}
+    runs-on: ${{ fromJSON(inputs.runs_on) }}
+    timeout-minutes: 100
+    env:
+        KERNEL: ${{ inputs.kernel }}
+        REPO_ROOT: ${{ github.workspace }}
+        REPO_PATH: ""
+        KBUILD_OUTPUT: kbuild-output/
+    steps:
+      - uses: actions/checkout@v3
+        # We fetch an actual bit of history here to facilitate incremental
+        # builds (which may check out some earlier upstream change).
+        with:
+          fetch-depth: 50
+      - if: ${{ github.repository == 'kernel-patches/vmtest' }}
+        name: Download bpf-next tree
+        uses: libbpf/ci/get-linux-source@main
+        with:
+          dest: '.kernel'
+      - if: ${{ github.repository == 'kernel-patches/vmtest' }}
+        name: Move linux source in place
+        shell: bash
+        run: |
+          rm -rf .kernel/.git
+          cp -rf .kernel/. .
+          rm -rf .kernel
+      - name: Get commit meta-data
+        id: get-commit-metadata
+        run: |
+          bash .github/scripts/get-commit-metadata.sh
+      - name: Pull recent KBUILD_OUTPUT contents
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.KBUILD_OUTPUT }}
+          key: kbuild-output-${{ inputs.arch }}-${{ inputs.toolchain_full }}-${{ steps.get-commit-metadata.outputs.branch }}-${{ steps.get-commit-metadata.outputs.timestamp }}-${{ steps.get-commit-metadata.outputs.commit }}
+          restore-keys: |
+            kbuild-output-${{ inputs.arch }}-${{ inputs.toolchain_full }}-${{ steps.get-commit-metadata.outputs.branch }}-${{ steps.get-commit-metadata.outputs.timestamp }}-
+            kbuild-output-${{ inputs.arch }}-${{ inputs.toolchain_full }}-${{ steps.get-commit-metadata.outputs.branch }}-
+            kbuild-output-${{ inputs.arch }}-${{ inputs.toolchain_full }}-
+      - name: Prepare incremental build
+        shell: bash
+        run: |
+          bash .github/scripts/prepare-incremental-builds.sh ${{ steps.get-commit-metadata.outputs.commit }}
+      - uses: libbpf/ci/patch-kernel@main
+        with:
+          patches-root: '${{ github.workspace }}/ci/diffs'
+          repo-root: '${{ github.workspace }}'
+      - name: Setup build environment
+        uses: libbpf/ci/setup-build-env@main
+        with:
+          llvm-version: ${{ inputs.llvm-version }}
+      - name: Build kernel image
+        uses: libbpf/ci/build-linux@main
+        with:
+          arch: ${{ inputs.arch }}
+          toolchain: ${{ inputs.toolchain }}
+          kbuild-output: ${{ env.KBUILD_OUTPUT }}
+          max-make-jobs: 32
+          llvm-version: ${{ inputs.llvm-version }}
+      - name: Build selftests
+        uses: libbpf/ci/build-selftests@main
+        with:
+          toolchain: ${{ inputs.toolchain }}
+          kbuild-output: ${{ env.KBUILD_OUTPUT }}
+          max-make-jobs: 32
+          llvm-version: ${{ inputs.llvm-version }}
+      - if: ${{ github.event_name != 'push' }}
+        name: Build samples
+        uses: libbpf/ci/build-samples@main
+        with:
+          toolchain: ${{ inputs.toolchain }}
+          kbuild-output: ${{ env.KBUILD_OUTPUT }}
+          max-make-jobs: 32
+          llvm-version: ${{ inputs.llvm-version }}
+      - name: Tar artifacts
+        run: |
+          bash .github/scripts/tar-artifact.sh ${{ inputs.arch }} ${{ inputs.toolchain_full }}
+      - if: ${{ github.event_name != 'push' }}
+        name: Remove KBUILD_OUTPUT content
+        shell: bash
+        run: |
+          # Remove $KBUILD_OUTPUT to prevent cache creation for pull requests.
+          # Only on pushed changes are build artifacts actually cached, because
+          # of github.com/actions/cache's cache isolation logic.
+          rm -rf "${KBUILD_OUTPUT}"
+      - uses: actions/upload-artifact@v3
+        with:
+          name: vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}
+          if-no-files-found: error
+          path: vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}.tar.zst