virtme-ng: use pre-compiled mainline kernels

Allow to download precompiled mainline kernels from the Ubuntu mainline
kernel repository (https://kernel.ubuntu.com/mainline).

Mainline builds are provided by Ubuntu for debugging purposes, they are
basically vanilla kernels (no additional patches applied), built with
the generic Ubuntu .config and packaged as deb.

These precompiled kernels can be used by virtme-ng to test specific
mainline tags, without having to rebuild the kernel from source.

To do so the option `--run` can now accept a Linux tag (i.e., v6.6-rc2).
When a tag is specified, virtme-ng will search in the Ubuntu mainline
repository and fetch the corresponding packages, if available.

Packages are then cached and extracted inside $HOME/.cache/virtme-ng, so
they just need to be downloaded the first time that a mainline tag is
requested.

Example usage:

 $ vng -r v6.6-rc2 -- uname -r
 6.6.0-060600rc2-generic

This feature allows to save even more time (and energy) when testing
mainline kernel versions, completely cutting the kernel rebuild time.

Signed-off-by: Andrea Righi <andrea.righi@canonical.com>

Author: Andrea Righi

bin/virtme-pull-kernel

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+#
+# TODO:
+#  - download packages in parallel
+#  - add a command to clean virtme-ng cache (e.g., `vng --clean-cache`)
+
+import os
+import re
+import sys
+import requests
+import subprocess
+from glob import glob
+from pathlib import Path
+from virtme_ng.utils import CACHE_DIR
+
+BASE_URL = "https://kernel.ubuntu.com/mainline"
+
+def download_file(url, destination):
+    response = requests.get(url, stream=True)
+    if response.status_code == 200:
+        with open(destination, 'wb') as file:
+            for chunk in response.iter_content(chunk_size=4096):
+                file.write(chunk)
+    else:
+        sys.stderr.write(f"failed to download {url}, error: {response.status_code}\n")
+        sys.exit(1)
+
+def extract_kernel():
+    deb_files = [f for f in os.listdir('.') if f.endswith('.deb')]
+    for deb_file in deb_files:
+        subprocess.run(['dpkg', '-x', deb_file, "."])
+
+def fetch_kernel(url):
+    response = requests.get(url)
+    if response.status_code == 200:
+        href_pattern = re.compile(r'href=["\']([^\s"\']+.deb)["\']')
+        matches = href_pattern.findall(response.text)
+        for match in matches:
+            # Skip headers packages
+            if 'headers' in match:
+                continue
+            # Skip if package is already downloaded
+            if os.path.exists(match):
+                continue
+            download_file(url + "/" + match, match)
+        if not glob("*.deb"):
+            sys.stderr.write(f"could not find kernel packages at {url}\n")
+            sys.exit(1)
+    else:
+        sys.stderr.write(f"failed to retrieve content, error: {response.status_code}\n")
+        sys.exit(1)
+
+def main():
+    version = sys.argv[1]
+    arch = sys.argv[2]
+
+    kernel_dir = f"{CACHE_DIR}/{version}/{arch}"
+    os.makedirs(kernel_dir, exist_ok=True)
+    os.chdir(kernel_dir)
+    fetch_kernel(BASE_URL + "/" + version + "/" + arch)
+    extract_kernel()
+
+if __name__ == '__main__':
+    main()

setup.py

@@ -153,6 +153,7 @@ def run(self):
     ],
     scripts=[
         "bin/virtme-prep-kdir-mods",
+        "bin/virtme-pull-kernel",
     ],
     include_package_data=True,
     classifiers=[

virtme_ng/run.py

@@ -18,6 +18,7 @@
     PIPE,
     CalledProcessError,
 )
+from glob import glob
 from select import select
 from pathlib import Path
 try:
@@ -28,7 +29,7 @@ def autocomplete(*args, **kwargs):
         pass
 
 from virtme.util import SilentError, uname, get_username
-from virtme_ng.utils import CONF_FILE
+from virtme_ng.utils import CONF_FILE, CACHE_DIR
 from virtme_ng.spinner import Spinner
 from virtme_ng.version import VERSION
 
@@ -776,7 +777,22 @@ def _get_virtme_overlay_rwdir(self, args):
 
     def _get_virtme_run(self, args):
         if args.run is not None:
-            self.virtme_param["kdir"] = "--kimg " + args.run
+            # If an upstream version is specified (using an upstream tag) fetch
+            # and run the corresponding kernel from the Ubuntu mainline
+            # repository.
+            if args.run.startswith('v'):
+                if args.arch is None:
+                    arch = 'amd64'
+                else:
+                    arch = args.arch
+                target = f"{CACHE_DIR}/{args.run}/{arch}/boot/vmlinuz-*"
+                if not glob(target):
+                    script = shutil.which('virtme-pull-kernel')
+                    print("downloading pre-compiled upstream kernel from Ubuntu mainline builds...")
+                    check_call_cmd([script, args.run, arch], quiet=not args.verbose, dry_run=args.dry_run)
+                self.virtme_param["kdir"] = "--kimg " + target
+            else:
+                self.virtme_param["kdir"] = "--kimg " + args.run
         else:
             self.virtme_param["kdir"] = "--kdir ./"
 

virtme_ng/utils.py

@@ -5,5 +5,6 @@
 
 from pathlib import Path
 
+CACHE_DIR = Path(Path.home(), ".cache", "virtme-ng")
 CONF_PATH = Path(Path.home(), ".config", "virtme-ng")
 CONF_FILE = Path(CONF_PATH, "virtme-ng.conf")