Add support for Ubuntu 24.04 and ARM64 (#202)

Adds support for:
- Ubuntu 24.04 (and thus Heroku-24 / `heroku/builder:24`)
- The ARM64 CPU architecture (for Ubuntu 24.04 only)

The ARM64 support is tested in CI via GitHub's ARM runners beta.

GUS-W-14667596.
GUS-W-15158299.

Author: edmorley

.github/workflows/ci.yml

@@ -41,34 +41,50 @@ jobs:
         run: cargo test --locked
 
   integration-test:
-    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        builder: ["builder:22", "builder:20"]
+        builder: ["builder:24", "builder:22", "builder:20"]
+        arch: ["amd64", "arm64"]
+        exclude:
+          - builder: "builder:22"
+            arch: "arm64"
+          - builder: "builder:20"
+            arch: "arm64"
+    runs-on: ${{ matrix.arch == 'arm64' && 'pub-hk-ubuntu-22.04-arm-large' || 'ubuntu-latest' }}
     env:
-      INTEGRATION_TEST_CNB_BUILDER: heroku/${{ matrix.builder }}
+      INTEGRATION_TEST_BUILDER: heroku/${{ matrix.builder }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      # The beta ARM64 runners don't yet ship with the normal installed tools.
+      - name: Install Docker, Rust and missing development libs (ARM64 only)
+        if: matrix.arch == 'arm64'
+        run: |
+          sudo apt-get update --error-on=any
+          sudo apt-get install -y --no-install-recommends acl docker.io docker-buildx libc6-dev
+          sudo usermod -aG docker $USER
+          sudo setfacl --modify user:$USER:rw /var/run/docker.sock
+          curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
+          echo "${HOME}/.cargo/bin" >> "${GITHUB_PATH}"
       - name: Install musl-tools
-        run: sudo apt-get install musl-tools --no-install-recommends
+        run: sudo apt-get install -y --no-install-recommends musl-tools
       - name: Update Rust toolchain
         run: rustup update
       - name: Install Rust linux-musl target
-        run: rustup target add x86_64-unknown-linux-musl
+        run: rustup target add ${{ matrix.arch == 'arm64' && 'aarch64-unknown-linux-musl' || 'x86_64-unknown-linux-musl' }}
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2.7.3
       - name: Install Pack CLI
         uses: buildpacks/github-actions/setup-pack@v5.6.0
       - name: Pull builder image
-        run: docker pull ${{ env.INTEGRATION_TEST_CNB_BUILDER }}
+        run: docker pull ${{ env.INTEGRATION_TEST_BUILDER }}
       # The integration tests are annotated with the `ignore` attribute, allowing us to run
       # only those and not the unit tests, via the `--ignored` option. On the latest stack
       # we run all integration tests, but on older stacks we only run stack-specific tests.
       - name: Run integration tests (all tests)
-        if: matrix.builder == 'builder:22'
+        if: matrix.builder == 'builder:24'
         run: cargo test --locked -- --ignored --test-threads 5
       - name: Run integration tests (stack-specific tests only)
-        if: matrix.builder != 'builder:22'
+        if: matrix.builder != 'builder:24'
         run: cargo test --locked -- --ignored --test-threads 5 'python_version_test::'

CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Added support for Ubuntu 24.04 (and thus Heroku-24 / `heroku/builder:24`). ([#202](https://github.com/heroku/buildpacks-python/pull/202))
+- Added support for the ARM64 CPU architecture (Ubuntu 24.04 only). ([#202](https://github.com/heroku/buildpacks-python/pull/202))
+
 ## [0.9.0] - 2024-05-03
 
 ### Changed

buildpack.toml

@@ -28,5 +28,17 @@ version = "20.04"
 name = "ubuntu"
 version = "22.04"
 
+[[targets.distros]]
+name = "ubuntu"
+version = "24.04"
+
+[[targets]]
+os = "linux"
+arch = "arm64"
+
+[[targets.distros]]
+name = "ubuntu"
+version = "24.04"
+
 [metadata.release]
 image = { repository = "docker.io/heroku/buildpack-python" }

tests/detect_test.rs

@@ -1,13 +1,12 @@
-use crate::tests::builder;
+use crate::tests::default_build_config;
 use indoc::indoc;
-use libcnb_test::{assert_contains, BuildConfig, PackResult, TestRunner};
+use libcnb_test::{assert_contains, PackResult, TestRunner};
 
 #[test]
 #[ignore = "integration test"]
 fn detect_rejects_non_python_projects() {
     TestRunner::default().build(
-        BuildConfig::new(builder(), "tests/fixtures/empty")
-            .expected_pack_result(PackResult::Failure),
+        default_build_config("tests/fixtures/empty").expected_pack_result(PackResult::Failure),
         |context| {
             assert_contains!(
                 context.pack_stdout,

tests/django_test.rs

@@ -1,14 +1,14 @@
-use crate::tests::builder;
+use crate::tests::default_build_config;
 use indoc::indoc;
-use libcnb_test::{assert_contains, assert_empty, BuildConfig, PackResult, TestRunner};
+use libcnb_test::{assert_contains, assert_empty, PackResult, TestRunner};
 
 // This test uses symlinks for requirements.txt and manage.py to confirm that it's possible to use
 // them when the Django app is nested inside a subdirectory (such as in backend+frontend monorepos).
 #[test]
 #[ignore = "integration test"]
 fn django_staticfiles_latest_django() {
     TestRunner::default().build(
-        BuildConfig::new(builder(), "tests/fixtures/django_staticfiles_latest_django")
+        default_build_config("tests/fixtures/django_staticfiles_latest_django")
             // Tests that env vars are passed to the 'manage.py' script invocations.
             .env("EXPECTED_ENV_VAR", "1"),
         |context| {
@@ -26,23 +26,20 @@ fn django_staticfiles_latest_django() {
     );
 }
 
-// This tests the oldest Django version that works on Python 3.9 (which is the
+// This tests the oldest Django version that works on Python 3.10 (which is the
 // oldest Python that is available on all of our supported builders).
 #[test]
 #[ignore = "integration test"]
 fn django_staticfiles_legacy_django() {
     TestRunner::default().build(
-        BuildConfig::new(builder(), "tests/fixtures/django_staticfiles_legacy_django"),
+        default_build_config("tests/fixtures/django_staticfiles_legacy_django"),
         |context| {
             assert_empty!(context.pack_stderr);
             assert_contains!(
                 context.pack_stdout,
                 indoc! {"
-                    Successfully installed Django-1.8.19
-                    
                     [Generating Django static files]
                     Running 'manage.py collectstatic'
-                    Linking '/workspace/testapp/static/robots.txt'
                     
                     1 static file symlinked to '/workspace/staticfiles'.
                 "}
@@ -55,7 +52,7 @@ fn django_staticfiles_legacy_django() {
 #[ignore = "integration test"]
 fn django_no_manage_py() {
     TestRunner::default().build(
-        BuildConfig::new(builder(), "tests/fixtures/django_no_manage_py"),
+        default_build_config("tests/fixtures/django_no_manage_py"),
         |context| {
             assert_empty!(context.pack_stderr);
             assert_contains!(
@@ -75,10 +72,7 @@ fn django_no_manage_py() {
 #[ignore = "integration test"]
 fn django_staticfiles_app_not_enabled() {
     TestRunner::default().build(
-        BuildConfig::new(
-            builder(),
-            "tests/fixtures/django_staticfiles_app_not_enabled",
-        ),
+        default_build_config("tests/fixtures/django_staticfiles_app_not_enabled"),
         |context| {
             assert_empty!(context.pack_stderr);
             assert_contains!(
@@ -97,7 +91,7 @@ fn django_staticfiles_app_not_enabled() {
 #[ignore = "integration test"]
 fn django_invalid_settings_module() {
     TestRunner::default().build(
-        BuildConfig::new(builder(), "tests/fixtures/django_invalid_settings_module")
+        default_build_config("tests/fixtures/django_invalid_settings_module")
             .expected_pack_result(PackResult::Failure),
         |context| {
             assert_contains!(
@@ -139,7 +133,7 @@ fn django_invalid_settings_module() {
 #[ignore = "integration test"]
 fn django_staticfiles_misconfigured() {
     TestRunner::default().build(
-        BuildConfig::new(builder(), "tests/fixtures/django_staticfiles_misconfigured")
+        default_build_config("tests/fixtures/django_staticfiles_misconfigured")
             .expected_pack_result(PackResult::Failure),
         |context| {
             assert_contains!(

tests/fixtures/django_staticfiles_legacy_django/requirements.txt

@@ -1,3 +1,3 @@
-# This is the oldest Django version that works on Python 3.9 (which is the
+# This is the oldest Django version that works on Python 3.10 (which is the
 # oldest Python that is available on all of our supported builders).
-Django==1.8.19
+Django==2.1.15

tests/fixtures/django_staticfiles_legacy_django/runtime.txt

@@ -1 +1 @@
-python-3.9.19
+python-3.10.14

tests/mod.rs

@@ -3,7 +3,9 @@
 //! These tests are not run via automatic integration test discovery, but instead are
 //! imported in main.rs so that they have access to private APIs (see comment in main.rs).
 
+use libcnb_test::BuildConfig;
 use std::env;
+use std::path::Path;
 
 mod detect_test;
 mod django_test;
@@ -19,8 +21,24 @@ const LATEST_PYTHON_3_11: &str = "3.11.9";
 const LATEST_PYTHON_3_12: &str = "3.12.3";
 const DEFAULT_PYTHON_VERSION: &str = LATEST_PYTHON_3_12;
 
-const DEFAULT_BUILDER: &str = "heroku/builder:22";
+const DEFAULT_BUILDER: &str = "heroku/builder:24";
+
+fn default_build_config(fixture_path: impl AsRef<Path>) -> BuildConfig {
+    let builder = builder();
+
+    // TODO: Once Pack build supports `--platform` and libcnb-test adjusted accordingly, change this
+    // to allow configuring the target arch independently of the builder name (eg via env var).
+    let target_triple = match builder.as_str() {
+        // Compile the buildpack for ARM64 iff the builder supports multi-arch and the host is ARM64.
+        "heroku/builder:24" if cfg!(target_arch = "aarch64") => "aarch64-unknown-linux-musl",
+        _ => "x86_64-unknown-linux-musl",
+    };
+
+    let mut config = BuildConfig::new(&builder, fixture_path);
+    config.target_triple(target_triple);
+    config
+}
 
 fn builder() -> String {
-    env::var("INTEGRATION_TEST_CNB_BUILDER").unwrap_or(DEFAULT_BUILDER.to_string())
+    env::var("INTEGRATION_TEST_BUILDER").unwrap_or(DEFAULT_BUILDER.to_string())
 }

tests/package_manager_test.rs

@@ -1,12 +1,12 @@
-use crate::tests::builder;
+use crate::tests::default_build_config;
 use indoc::indoc;
-use libcnb_test::{assert_contains, BuildConfig, PackResult, TestRunner};
+use libcnb_test::{assert_contains, PackResult, TestRunner};
 
 #[test]
 #[ignore = "integration test"]
 fn no_package_manager_detected() {
     TestRunner::default().build(
-        BuildConfig::new(builder(), "tests/fixtures/pyproject_toml_only")
+        default_build_config("tests/fixtures/pyproject_toml_only")
             .expected_pack_result(PackResult::Failure),
         |context| {
             assert_contains!(

tests/pip_test.rs

@@ -1,9 +1,7 @@
 use crate::packaging_tool_versions::PackagingToolVersions;
-use crate::tests::{builder, DEFAULT_PYTHON_VERSION};
+use crate::tests::{builder, default_build_config, DEFAULT_PYTHON_VERSION};
 use indoc::{formatdoc, indoc};
-use libcnb_test::{
-    assert_contains, assert_empty, BuildConfig, BuildpackReference, PackResult, TestRunner,
-};
+use libcnb_test::{assert_contains, assert_empty, BuildpackReference, PackResult, TestRunner};
 
 #[test]
 #[ignore = "integration test"]
@@ -14,7 +12,7 @@ fn pip_basic_install_and_cache_reuse() {
         wheel_version,
     } = PackagingToolVersions::default();
 
-    let config = BuildConfig::new(builder(), "tests/fixtures/pip_basic");
+    let config = default_build_config("tests/fixtures/pip_basic");
 
     TestRunner::default().build(&config, |context| {
         assert_empty!(context.pack_stderr);
@@ -97,8 +95,9 @@ fn pip_basic_install_and_cache_reuse() {
 #[test]
 #[ignore = "integration test"]
 fn pip_cache_invalidation_with_compatible_metadata() {
-    // TODO: Re-enable this test after the next buildpack release, at which point there will
-    // be a historic buildpack version with compatible cache metadata that we can use.
+    // TODO: Re-enable this test the next time the default-Python/pip/setuptools/wheel versions
+    // change, at which point there will be a historic buildpack version that has both compatible
+    // metadata, and that is also compatible with Ubuntu 24.04.
     #![allow(unreachable_code)]
     return;
 
@@ -108,7 +107,7 @@ fn pip_cache_invalidation_with_compatible_metadata() {
         wheel_version,
     } = PackagingToolVersions::default();
 
-    let config = BuildConfig::new(builder(), "tests/fixtures/pip_basic");
+    let config = default_build_config("tests/fixtures/pip_basic");
 
     TestRunner::default().build(
         config.clone().buildpacks([BuildpackReference::Other(
@@ -154,13 +153,19 @@ fn pip_cache_invalidation_with_compatible_metadata() {
 #[test]
 #[ignore = "integration test"]
 fn pip_cache_invalidation_with_incompatible_metadata() {
+    // TODO: Enable this test on Heroku-24 the next time there is an incompatible metadata change,
+    // meaning we can bump the historic buildpack version to one that also supports Ubuntu 24.04.
+    if builder() == "heroku/builder:24" {
+        return;
+    }
+
     let PackagingToolVersions {
         pip_version,
         setuptools_version,
         wheel_version,
     } = PackagingToolVersions::default();
 
-    let config = BuildConfig::new(builder(), "tests/fixtures/pip_basic");
+    let config = default_build_config("tests/fixtures/pip_basic");
 
     TestRunner::default().build(
         config.clone().buildpacks([BuildpackReference::Other(
@@ -206,7 +211,7 @@ fn pip_cache_invalidation_with_incompatible_metadata() {
 #[ignore = "integration test"]
 fn pip_editable_git_compiled() {
     TestRunner::default().build(
-        BuildConfig::new(builder(), "tests/fixtures/pip_editable_git_compiled")
+        default_build_config( "tests/fixtures/pip_editable_git_compiled")
             .env("WHEEL_PACKAGE_URL", "https://github.com/pypa/wheel"),
         |context| {
             assert_contains!(
@@ -221,7 +226,7 @@ fn pip_editable_git_compiled() {
 #[ignore = "integration test"]
 fn pip_install_error() {
     TestRunner::default().build(
-        BuildConfig::new(builder(), "tests/fixtures/pip_invalid_requirement")
+        default_build_config( "tests/fixtures/pip_invalid_requirement")
             .expected_pack_result(PackResult::Failure),
         |context| {
             // Ideally we could test a combined stdout/stderr, however libcnb-test doesn't support this: