Skip to content

Commit

Permalink
Dockerfile: Maintenance
Browse files Browse the repository at this point in the history 
Improve build/push/pull speed by reducing the number of layers and their
sizes.

Improve build speed by fetching pre-built binaries of cbindgen and
bindgen.

Silence curl progress output since it takes so many lines in CI.
  • Loading branch information
@NickeZ
NickeZ committed Aug 27, 2024
1 parent fc0515b commit d48bff8
Show file tree
Hide file tree
Showing 2 changed files with 67 additions and 59 deletions.
2 changes: 1 addition & 1 deletion .containerversion
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
43
44
124 changes: 66 additions & 58 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,20 +19,22 @@
# $ docker run --privileged --rm tonistiigi/binfmt --install arm64

FROM ubuntu:22.04
ENV DEBIAN_FRONTEND noninteractive

# These are automatically provided by docker (no need for --build-arg)
ARG TARGETPLATFORM
ARG TARGETARCH

RUN apt-get update && apt-get upgrade -y && apt-get install -y wget nano rsync curl gnupg2 jq unzip bzip2 xz-utils
RUN export DEBIAN_FRONTEND=noninteractive; \
apt-get update && \
apt-get upgrade -y && \
apt-get install -y nano rsync curl gnupg2 jq unzip bzip2 xz-utils && \
rm -rf /var/lib/apt/lists/*


# for clang-*-15, see https://apt.llvm.org/
RUN echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" >> /etc/apt/sources.list && \
echo "deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" >> /etc/apt/sources.list && \
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
curl -sSL https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -

# Install gcc8-arm-none-eabi
RUN if [ "${TARGETPLATFORM}" = "linux/arm64" ]; then \
GNU_TOOLCHAIN=https://developer.arm.com/-/media/Files/downloads/gnu/13.3.rel1/binrel/arm-gnu-toolchain-13.3.rel1-aarch64-arm-none-eabi.tar.xz \
GNU_TOOLCHAIN_HASH=c8824bffd057afce2259f7618254e840715f33523a3d4e4294f471208f976764 \
Expand All @@ -42,13 +44,14 @@ RUN if [ "${TARGETPLATFORM}" = "linux/arm64" ]; then \
GNU_TOOLCHAIN_HASH=fb31fbdfe08406ece43eef5df623c0b2deb8b53e405e2c878300f7a1f303ee52 \
GNU_TOOLCHAIN_FORMAT=bz2; \
fi; \
wget -O gcc.tar.${GNU_TOOLCHAIN_FORMAT} ${GNU_TOOLCHAIN} &&\
echo "$GNU_TOOLCHAIN_HASH gcc.tar.${GNU_TOOLCHAIN_FORMAT}" | sha256sum -c &&\
tar -xvf gcc.tar.${GNU_TOOLCHAIN_FORMAT} -C /usr/local --strip-components=1 &&\
curl -sSL -o gcc.tar.${GNU_TOOLCHAIN_FORMAT} ${GNU_TOOLCHAIN} && \
echo "$GNU_TOOLCHAIN_HASH gcc.tar.${GNU_TOOLCHAIN_FORMAT}" | sha256sum -c && \
tar -xf gcc.tar.${GNU_TOOLCHAIN_FORMAT} -C /usr/local --strip-components=1 && \
rm -f gcc.tar.${GNU_TOOLCHAIN_FORMAT}

# Tools for building
RUN apt-get update && apt-get install -y \
RUN export DEBIAN_FRONTEND=noninteractive; \
apt-get update && apt-get install -y \
make \
llvm-18 \
gcc-10 \
Expand All @@ -64,98 +67,103 @@ RUN apt-get update && apt-get install -y \
libcmocka-dev \
libusb-1.0-0-dev \
libudev-dev \
libhidapi-dev
libhidapi-dev \
&& rm -rf /var/lib/apt/lists/*

RUN apt-get update && apt-get install -y \
RUN export DEBIAN_FRONTEND=noninteractive; \
apt-get update && apt-get install -y \
doxygen \
graphviz
graphviz \
&& rm -rf /var/lib/apt/lists/*

# Set gcc-10 as the default gcc
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100
RUN update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-10 100
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 && \
update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-10 100 && \
rm /var/log/alternatives.log

# Tools for CI
RUN apt-get update && apt-get install -y \
RUN export DEBIAN_FRONTEND=noninteractive; \
apt-get update && apt-get install -y \
python3 \
python3-pip \
clang-format-18 \
clang-tidy-18

RUN python3 -m pip install --upgrade pip
clang-tidy-18 \
&& rm -rf /var/lib/apt/lists/*

# Python modules
COPY py/bitbox02 /tmp/bitbox02
RUN python3 -m pip install /tmp/bitbox02
RUN rm -r /tmp/bitbox02
COPY py/requirements.txt /tmp
RUN python3 -m pip install --upgrade --requirement /tmp/requirements.txt
RUN rm /tmp/requirements.txt

# Python modules for CI
RUN python3 -m pip install --upgrade \
RUN --mount=source=py,target=/mnt,rw \
python3 -m pip install --no-compile --no-cache-dir /mnt/bitbox02 && \
python3 -m pip install --no-compile --no-cache-dir --upgrade --requirement /mnt/requirements.txt && \
python3 -m pip install --no-compile --no-cache-dir --upgrade \
pylint==2.13.9 \
pylint-protobuf==0.20.2 \
black==22.3.0 \
mypy==0.960 \
mypy-protobuf==3.2.0

# Python modules for packaging
RUN python3 -m pip install --upgrade \
RUN python3 -m pip install --no-compile --no-cache-dir --upgrade \
mypy-protobuf==3.2.0 \
setuptools==41.2.0 \
wheel==0.33.6 \
twine==1.15.0

#Install protoc from release, because the version available on the repo is too old
ENV PATH /opt/protoc/bin:$PATH
RUN if [ "${TARGETPLATFORM}" = "linux/arm64" ]; then \
PROTOC_URL=https://github.com/protocolbuffers/protobuf/releases/download/v21.2/protoc-21.2-linux-aarch_64.zip; \
else \
PROTOC_URL=https://github.com/protocolbuffers/protobuf/releases/download/v21.2/protoc-21.2-linux-x86_64.zip; \
fi; \
mkdir -p /opt/protoc && \
curl -L0 ${PROTOC_URL} -o /tmp/protoc-21.2.zip && \
curl -sSL0 ${PROTOC_URL} -o /tmp/protoc-21.2.zip && \
unzip /tmp/protoc-21.2.zip -d /opt/protoc && \
rm /tmp/protoc-21.2.zip
ENV PATH /opt/protoc/bin:$PATH

# Developer tools
RUN apt-get update && apt-get install -y \
bash-completion
RUN export DEBIAN_FRONTEND=noninteractive; \
apt-get update && apt-get install -y \
bash-completion \
&& rm -rf /var/lib/apt/lists/*
# Install gcovr from PIP to get a newer version than in apt repositories
RUN python3 -m pip install gcovr
RUN python3 -m pip install --no-compile --no-cache-dir --upgrade \
gcovr==7.2

# Install Go, used for the tools in tools/go and for test/gounittest
ENV GOPATH /opt/go
ENV GOROOT /opt/go_dist/go
ENV PATH $GOROOT/bin:$GOPATH/bin:$PATH
ENV PATH=/opt/go_dist/go/bin:/opt/go/bin:$PATH GOPATH=/opt/go GOROOT=/opt/go_dist/go
RUN mkdir -p /opt/go_dist && \
curl https://dl.google.com/go/go1.19.3.linux-${TARGETARCH}.tar.gz | tar -xz -C /opt/go_dist
curl -sSL https://dl.google.com/go/go1.19.3.linux-${TARGETARCH}.tar.gz | tar -xz -C /opt/go_dist

# Install lcov from release (the one from the repos is too old).
RUN cd /opt && wget https://github.com/linux-test-project/lcov/releases/download/v1.14/lcov-1.14.tar.gz && tar -xf lcov-1.14.tar.gz
ENV PATH /opt/lcov-1.14/bin:$PATH
ENV PATH=/opt/lcov-1.14/bin:$PATH
RUN curl -sSL https://github.com/linux-test-project/lcov/releases/download/v1.14/lcov-1.14.tar.gz | tar -xz -C /opt

# Install rust compiler
ENV PATH /opt/cargo/bin:$PATH
ENV RUSTUP_HOME=/opt/rustup
COPY src/rust/rust-toolchain.toml /tmp/rust-toolchain.toml
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | CARGO_HOME=/opt/cargo sh -s -- --default-toolchain $(grep -oP '(?<=channel = ")[^"]+' /tmp/rust-toolchain.toml) -y
RUN rustup target add thumbv7em-none-eabi
RUN rustup component add rustfmt
RUN rustup component add clippy
RUN rustup component add rust-src
RUN CARGO_HOME=/opt/cargo cargo install cbindgen --version 0.26.0 --locked
RUN CARGO_HOME=/opt/cargo cargo install bindgen-cli --version 0.69.4 --locked
# Since bindgen embeds information about its target directory, use a deterministic path for it.
ARG BINDGEN_VERSION=0.69.4
ARG CBINDGEN_VERSION=0.26.0
ENV PATH=/opt/cargo/bin:$PATH RUSTUP_HOME=/opt/rustup
RUN --mount=source=tools/prost-build-proto,target=/mnt/prost-build-proto,rw \
--mount=source=src/rust/rust-toolchain.toml,target=/mnt/rust-toolchain.toml \
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
CARGO_HOME=/opt/cargo sh -s -- --default-toolchain $(grep -oP '(?<=channel = ")[^"]+' /mnt/rust-toolchain.toml) -y && \
rustup target add thumbv7em-none-eabi && \
rustup component add rustfmt && \
rustup component add clippy && \
rustup component add rust-src && \
if [ "${TARGETPLATFORM}" = "linux/arm64" ]; then \
CARGO_HOME=/opt/cargo cargo install cbindgen --version ${CBINDGEN_VERSION} --locked && \
CARGO_HOME=/opt/cargo cargo install bindgen-cli --version ${BINDGEN_VERSION} --locked --target-dir=/tmp/bindgen-target && rm -r /tmp/bindgen-target; \
else \
curl -sSL https://github.com/rust-lang/rust-bindgen/releases/download/v${BINDGEN_VERSION}/bindgen-cli-x86_64-unknown-linux-gnu.tar.xz | tar -xJ --strip-components=1 -C /opt/cargo/bin bindgen-cli-x86_64-unknown-linux-gnu/bindgen && \
curl -sSL https://github.com/mozilla/cbindgen/releases/download/${CBINDGEN_VERSION}/cbindgen -o /opt/cargo/bin/cbindgen && chmod +x /opt/cargo/bin/cbindgen; \
fi && \
CARGO_HOME=/opt/cargo cargo install --path /mnt/prost-build-proto --locked && \
rm -r /opt/cargo/registry/index /opt/cargo/.global-cache

# Until cargo vendor supports vendoring dependencies of the rust std libs we
# need a copy of this file next to the toml file. It also has to be world
# writable so that invocations of `cargo vendor` can update it. Below is the
# tracking issue for `cargo vendor` to support rust std libs.
# https://github.com/rust-lang/wg-cargo-std-aware/issues/23
RUN cp "$(rustc --print=sysroot)/lib/rustlib/src/rust/Cargo.lock" "$(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/"
RUN chmod 777 $(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/Cargo.lock

COPY tools/prost-build-proto prost-build-proto
RUN CARGO_HOME=/opt/cargo cargo install --path prost-build-proto --locked

# Clean temporary files to reduce image size
RUN rm -rf /var/lib/apt/lists/*
RUN cp "$(rustc --print=sysroot)/lib/rustlib/src/rust/Cargo.lock" "$(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/" && \
chmod 777 $(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/Cargo.lock

0 comments on commit d48bff8

Please sign in to comment.