From d48bff8c20c6fe7bfcf69b495ec567c331d81600 Mon Sep 17 00:00:00 2001 From: Niklas Dusenlund Date: Tue, 27 Aug 2024 10:43:36 +0200 Subject: [PATCH] Dockerfile: Maintenance Improve build/push/pull speed by reducing the number of layers and their sizes. Improve build speed by fetching pre-built binaries of cbindgen and bindgen. Silence curl progress output since it takes so many lines in CI. --- .containerversion | 2 +- Dockerfile | 124 ++++++++++++++++++++++++---------------------- 2 files changed, 67 insertions(+), 59 deletions(-) diff --git a/.containerversion b/.containerversion index 920a13966..c739b42c4 100644 --- a/.containerversion +++ b/.containerversion @@ -1 +1 @@ -43 +44 diff --git a/Dockerfile b/Dockerfile index 1269ee2e3..8b27a5494 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,20 +19,22 @@ # $ docker run --privileged --rm tonistiigi/binfmt --install arm64 FROM ubuntu:22.04 -ENV DEBIAN_FRONTEND noninteractive # These are automatically provided by docker (no need for --build-arg) ARG TARGETPLATFORM ARG TARGETARCH -RUN apt-get update && apt-get upgrade -y && apt-get install -y wget nano rsync curl gnupg2 jq unzip bzip2 xz-utils +RUN export DEBIAN_FRONTEND=noninteractive; \ + apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y nano rsync curl gnupg2 jq unzip bzip2 xz-utils && \ + rm -rf /var/lib/apt/lists/* + -# for clang-*-15, see https://apt.llvm.org/ RUN echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" >> /etc/apt/sources.list && \ echo "deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" >> /etc/apt/sources.list && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - + curl -sSL https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - -# Install gcc8-arm-none-eabi RUN if [ "${TARGETPLATFORM}" = "linux/arm64" ]; then \ GNU_TOOLCHAIN=https://developer.arm.com/-/media/Files/downloads/gnu/13.3.rel1/binrel/arm-gnu-toolchain-13.3.rel1-aarch64-arm-none-eabi.tar.xz \ GNU_TOOLCHAIN_HASH=c8824bffd057afce2259f7618254e840715f33523a3d4e4294f471208f976764 \ @@ -42,13 +44,14 @@ RUN if [ "${TARGETPLATFORM}" = "linux/arm64" ]; then \ GNU_TOOLCHAIN_HASH=fb31fbdfe08406ece43eef5df623c0b2deb8b53e405e2c878300f7a1f303ee52 \ GNU_TOOLCHAIN_FORMAT=bz2; \ fi; \ - wget -O gcc.tar.${GNU_TOOLCHAIN_FORMAT} ${GNU_TOOLCHAIN} &&\ - echo "$GNU_TOOLCHAIN_HASH gcc.tar.${GNU_TOOLCHAIN_FORMAT}" | sha256sum -c &&\ - tar -xvf gcc.tar.${GNU_TOOLCHAIN_FORMAT} -C /usr/local --strip-components=1 &&\ + curl -sSL -o gcc.tar.${GNU_TOOLCHAIN_FORMAT} ${GNU_TOOLCHAIN} && \ + echo "$GNU_TOOLCHAIN_HASH gcc.tar.${GNU_TOOLCHAIN_FORMAT}" | sha256sum -c && \ + tar -xf gcc.tar.${GNU_TOOLCHAIN_FORMAT} -C /usr/local --strip-components=1 && \ rm -f gcc.tar.${GNU_TOOLCHAIN_FORMAT} # Tools for building -RUN apt-get update && apt-get install -y \ +RUN export DEBIAN_FRONTEND=noninteractive; \ + apt-get update && apt-get install -y \ make \ llvm-18 \ gcc-10 \ @@ -64,98 +67,103 @@ RUN apt-get update && apt-get install -y \ libcmocka-dev \ libusb-1.0-0-dev \ libudev-dev \ - libhidapi-dev + libhidapi-dev \ + && rm -rf /var/lib/apt/lists/* -RUN apt-get update && apt-get install -y \ +RUN export DEBIAN_FRONTEND=noninteractive; \ + apt-get update && apt-get install -y \ doxygen \ - graphviz + graphviz \ + && rm -rf /var/lib/apt/lists/* # Set gcc-10 as the default gcc -RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 -RUN update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-10 100 +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 && \ + update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-10 100 && \ + rm /var/log/alternatives.log # Tools for CI -RUN apt-get update && apt-get install -y \ +RUN export DEBIAN_FRONTEND=noninteractive; \ + apt-get update && apt-get install -y \ python3 \ python3-pip \ clang-format-18 \ - clang-tidy-18 - -RUN python3 -m pip install --upgrade pip + clang-tidy-18 \ + && rm -rf /var/lib/apt/lists/* # Python modules -COPY py/bitbox02 /tmp/bitbox02 -RUN python3 -m pip install /tmp/bitbox02 -RUN rm -r /tmp/bitbox02 -COPY py/requirements.txt /tmp -RUN python3 -m pip install --upgrade --requirement /tmp/requirements.txt -RUN rm /tmp/requirements.txt - -# Python modules for CI -RUN python3 -m pip install --upgrade \ +RUN --mount=source=py,target=/mnt,rw \ + python3 -m pip install --no-compile --no-cache-dir /mnt/bitbox02 && \ + python3 -m pip install --no-compile --no-cache-dir --upgrade --requirement /mnt/requirements.txt && \ + python3 -m pip install --no-compile --no-cache-dir --upgrade \ pylint==2.13.9 \ pylint-protobuf==0.20.2 \ black==22.3.0 \ mypy==0.960 \ - mypy-protobuf==3.2.0 # Python modules for packaging -RUN python3 -m pip install --upgrade \ +RUN python3 -m pip install --no-compile --no-cache-dir --upgrade \ + mypy-protobuf==3.2.0 \ setuptools==41.2.0 \ wheel==0.33.6 \ twine==1.15.0 #Install protoc from release, because the version available on the repo is too old +ENV PATH /opt/protoc/bin:$PATH RUN if [ "${TARGETPLATFORM}" = "linux/arm64" ]; then \ PROTOC_URL=https://github.com/protocolbuffers/protobuf/releases/download/v21.2/protoc-21.2-linux-aarch_64.zip; \ else \ PROTOC_URL=https://github.com/protocolbuffers/protobuf/releases/download/v21.2/protoc-21.2-linux-x86_64.zip; \ fi; \ mkdir -p /opt/protoc && \ - curl -L0 ${PROTOC_URL} -o /tmp/protoc-21.2.zip && \ + curl -sSL0 ${PROTOC_URL} -o /tmp/protoc-21.2.zip && \ unzip /tmp/protoc-21.2.zip -d /opt/protoc && \ rm /tmp/protoc-21.2.zip -ENV PATH /opt/protoc/bin:$PATH # Developer tools -RUN apt-get update && apt-get install -y \ - bash-completion +RUN export DEBIAN_FRONTEND=noninteractive; \ + apt-get update && apt-get install -y \ + bash-completion \ + && rm -rf /var/lib/apt/lists/* # Install gcovr from PIP to get a newer version than in apt repositories -RUN python3 -m pip install gcovr +RUN python3 -m pip install --no-compile --no-cache-dir --upgrade \ + gcovr==7.2 # Install Go, used for the tools in tools/go and for test/gounittest -ENV GOPATH /opt/go -ENV GOROOT /opt/go_dist/go -ENV PATH $GOROOT/bin:$GOPATH/bin:$PATH +ENV PATH=/opt/go_dist/go/bin:/opt/go/bin:$PATH GOPATH=/opt/go GOROOT=/opt/go_dist/go RUN mkdir -p /opt/go_dist && \ - curl https://dl.google.com/go/go1.19.3.linux-${TARGETARCH}.tar.gz | tar -xz -C /opt/go_dist + curl -sSL https://dl.google.com/go/go1.19.3.linux-${TARGETARCH}.tar.gz | tar -xz -C /opt/go_dist # Install lcov from release (the one from the repos is too old). -RUN cd /opt && wget https://github.com/linux-test-project/lcov/releases/download/v1.14/lcov-1.14.tar.gz && tar -xf lcov-1.14.tar.gz -ENV PATH /opt/lcov-1.14/bin:$PATH +ENV PATH=/opt/lcov-1.14/bin:$PATH +RUN curl -sSL https://github.com/linux-test-project/lcov/releases/download/v1.14/lcov-1.14.tar.gz | tar -xz -C /opt # Install rust compiler -ENV PATH /opt/cargo/bin:$PATH -ENV RUSTUP_HOME=/opt/rustup -COPY src/rust/rust-toolchain.toml /tmp/rust-toolchain.toml -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | CARGO_HOME=/opt/cargo sh -s -- --default-toolchain $(grep -oP '(?<=channel = ")[^"]+' /tmp/rust-toolchain.toml) -y -RUN rustup target add thumbv7em-none-eabi -RUN rustup component add rustfmt -RUN rustup component add clippy -RUN rustup component add rust-src -RUN CARGO_HOME=/opt/cargo cargo install cbindgen --version 0.26.0 --locked -RUN CARGO_HOME=/opt/cargo cargo install bindgen-cli --version 0.69.4 --locked +# Since bindgen embeds information about its target directory, use a deterministic path for it. +ARG BINDGEN_VERSION=0.69.4 +ARG CBINDGEN_VERSION=0.26.0 +ENV PATH=/opt/cargo/bin:$PATH RUSTUP_HOME=/opt/rustup +RUN --mount=source=tools/prost-build-proto,target=/mnt/prost-build-proto,rw \ + --mount=source=src/rust/rust-toolchain.toml,target=/mnt/rust-toolchain.toml \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + CARGO_HOME=/opt/cargo sh -s -- --default-toolchain $(grep -oP '(?<=channel = ")[^"]+' /mnt/rust-toolchain.toml) -y && \ + rustup target add thumbv7em-none-eabi && \ + rustup component add rustfmt && \ + rustup component add clippy && \ + rustup component add rust-src && \ + if [ "${TARGETPLATFORM}" = "linux/arm64" ]; then \ + CARGO_HOME=/opt/cargo cargo install cbindgen --version ${CBINDGEN_VERSION} --locked && \ + CARGO_HOME=/opt/cargo cargo install bindgen-cli --version ${BINDGEN_VERSION} --locked --target-dir=/tmp/bindgen-target && rm -r /tmp/bindgen-target; \ + else \ + curl -sSL https://github.com/rust-lang/rust-bindgen/releases/download/v${BINDGEN_VERSION}/bindgen-cli-x86_64-unknown-linux-gnu.tar.xz | tar -xJ --strip-components=1 -C /opt/cargo/bin bindgen-cli-x86_64-unknown-linux-gnu/bindgen && \ + curl -sSL https://github.com/mozilla/cbindgen/releases/download/${CBINDGEN_VERSION}/cbindgen -o /opt/cargo/bin/cbindgen && chmod +x /opt/cargo/bin/cbindgen; \ + fi && \ + CARGO_HOME=/opt/cargo cargo install --path /mnt/prost-build-proto --locked && \ + rm -r /opt/cargo/registry/index /opt/cargo/.global-cache # Until cargo vendor supports vendoring dependencies of the rust std libs we # need a copy of this file next to the toml file. It also has to be world # writable so that invocations of `cargo vendor` can update it. Below is the # tracking issue for `cargo vendor` to support rust std libs. # https://github.com/rust-lang/wg-cargo-std-aware/issues/23 -RUN cp "$(rustc --print=sysroot)/lib/rustlib/src/rust/Cargo.lock" "$(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/" -RUN chmod 777 $(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/Cargo.lock - -COPY tools/prost-build-proto prost-build-proto -RUN CARGO_HOME=/opt/cargo cargo install --path prost-build-proto --locked - -# Clean temporary files to reduce image size -RUN rm -rf /var/lib/apt/lists/* +RUN cp "$(rustc --print=sysroot)/lib/rustlib/src/rust/Cargo.lock" "$(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/" && \ + chmod 777 $(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/Cargo.lock