Make docker image easier to reproduce

.ci/build-base-container

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker build --no-cache -f Dockerfile.ubuntu .

.ci/build-dev-container

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+docker build --pull --no-cache -t shiftcrypto/firmware_v2 .
+docker pull shiftcrypto/firmware_v2:41
+
+# Print out the differences but ignore them. We don't require reproducible builds right now.
+ocidiff shiftcrypto/firmware_v2:latest shiftcrypto/firmware_v2:41 || true

.ci/publish-container

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+CONTAINER_REPO=shiftcrypto/firmware_v2
+CONTAINER_VERSION=$(grep ^CONTAINER_VERSION Makefile | cut -d ' ' -f3)
+
+if docker manifest inspect $CONTAINER_REPO:$CONTAINER_VERSION; then
+	echo $CONTAINER_VERSION already exists, will not publish
+	exit 1
+fi
+
+docker build --no-cache -t $CONTAINER_REPO:latest -t $CONTAINER_REPO:$CONTAINER_VERSION .
+docker push $CONTAINER_REPO:latest $CONTAINER_REPO:$CONTAINER_VERSION

.ci/run-container-ci

@@ -25,7 +25,7 @@
 set -e
 set -x
 
-CONTAINER=shiftcrypto/firmware_v2:41
+CONTAINER=shiftcrypto/firmware_v2:42
 
 if [ "$1" == "pull" ] ; then
 	docker pull "$CONTAINER"

.github/workflows/container-images.yml

@@ -0,0 +1,26 @@
+name: Container nightly check
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 2 * * *'
+  env:
+    DIFFOCI_URL: https://github.com/reproducible-containers/diffoci/releases/download/v0.1.4/diffoci-v0.1.4.linux-amd64
+
+jobs:
+  check-base-image:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Fetch diffoci
+        run: curl -L $DIFFOCI_URL -o /usr/local/bin/diffoci
+      - name: Check that base container builds
+        run: ./.ci/build-base-container
+
+  check-dev-image:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Fetch diffoci
+        run: curl -L $DIFFOCI_URL -o /usr/local/bin/diffoci
+      - name: Check that dev container builds
+        run: ./.ci/build-dev-container

.github/workflows/publish-container-image.yml

@@ -0,0 +1,14 @@
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+    paths:
+      - Dockerfile
+jobs:
+  check-base-image:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Publish new container version
+        run: ./.ci/publish-container

Dockerfile

@@ -13,136 +13,78 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Latest Ubuntu LTS
-FROM ubuntu:22.04
-ENV DEBIAN_FRONTEND noninteractive
-
-RUN apt-get update && apt-get upgrade -y && apt-get install -y wget nano rsync curl gnupg2 jq unzip bzip2
-
-# for clang-*-15, see https://apt.llvm.org/
-RUN echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" >> /etc/apt/sources.list && \
-    echo "deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" >> /etc/apt/sources.list && \
-    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
+FROM shiftcrypto/ubuntu:1
 
 # Install gcc8-arm-none-eabi
-RUN mkdir ~/Downloads &&\
-    cd ~/Downloads &&\
-    wget -O gcc.tar.bz2 https://developer.arm.com/-/media/Files/downloads/gnu-rm/8-2018q4/gcc-arm-none-eabi-8-2018-q4-major-linux.tar.bz2?revision=d830f9dd-cd4f-406d-8672-cca9210dd220?product=GNU%20Arm%20Embedded%20Toolchain,64-bit,,Linux,8-2018-q4-major &&\
+RUN wget -O gcc.tar.bz2 https://developer.arm.com/-/media/Files/downloads/gnu-rm/8-2018q4/gcc-arm-none-eabi-8-2018-q4-major-linux.tar.bz2?revision=d830f9dd-cd4f-406d-8672-cca9210dd220?product=GNU%20Arm%20Embedded%20Toolchain,64-bit,,Linux,8-2018-q4-major &&\
     echo "fb31fbdfe08406ece43eef5df623c0b2deb8b53e405e2c878300f7a1f303ee52 gcc.tar.bz2" | sha256sum -c &&\
-    cd ~/Downloads &&\
-    tar -xjvf gcc.tar.bz2 &&\
-    rm -f gcc.tar.bz2 &&\
-    cd ~/Downloads && rsync -a gcc-arm-none-eabi-8-2018-q4-major/ /usr/local/
-
-# Tools for building
-RUN apt-get update && apt-get install -y \
-    build-essential \
-    llvm-18 \
-    gcc-10 \
-    binutils \
-    valgrind \
-    cmake \
-    git \
-    autotools-dev \
-    automake \
-    autoconf \
-    libtool \
-    pkg-config \
-    libcmocka-dev \
-    libc6-i386 \
-    lib32stdc++6 \
-    lib32z1 \
-    libusb-1.0-0-dev \
-    libudev-dev \
-    libhidapi-dev
-
-RUN apt-get update && apt-get install -y \
-    doxygen \
-    graphviz
+    tar -xjf gcc.tar.bz2 -C /usr/local --strip-components=1 &&\
+    rm gcc.tar.bz2 /root/.wget-hsts
 
 # Set gcc-10 as the default gcc
-RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100
-RUN update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-10 100
-
-# Tools for CI
-RUN apt-get update && apt-get install -y \
-    python3 \
-    python3-pip \
-    clang-format-18 \
-    clang-tidy-18
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 && rm /var/log/alternatives.log
+RUN update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-10 100 && rm /var/log/alternatives.log
 
-RUN python3 -m pip install --upgrade pip
+# Make Python3 the default
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && rm /var/log/alternatives.log
 
 # Python modules
-COPY py/bitbox02 /tmp/bitbox02
-RUN python3 -m pip install /tmp/bitbox02
-RUN rm -r /tmp/bitbox02
-COPY py/requirements.txt /tmp
-RUN python3 -m pip install --upgrade --requirement /tmp/requirements.txt
-RUN rm /tmp/requirements.txt
+RUN --mount=source=py/bitbox02,target=/mnt/bitbox02,rw \
+    pip install --no-compile --no-cache-dir /mnt/bitbox02
+RUN --mount=source=py/requirements.txt,target=/mnt/requirements.txt \
+    pip install --no-compile --no-cache-dir --upgrade --requirement /mnt/requirements.txt
 
 # Python modules for CI
-RUN python3 -m pip install --upgrade \
+RUN pip install --no-compile --no-cache-dir --upgrade \
     pylint==2.13.9 \
     pylint-protobuf==0.20.2 \
     black==22.3.0 \
     mypy==0.960 \
-    mypy-protobuf==3.2.0
-
-# Python modules for packaging
-RUN python3 -m pip install --upgrade \
+    mypy-protobuf==3.2.0 \
     setuptools==41.2.0 \
     wheel==0.33.6 \
-    twine==1.15.0
+    twine==1.15.0 \
+    gcovr==7.2
 
 #Install protoc from release, because the version available on the repo is too old
 RUN mkdir -p /opt/protoc && \
     curl -L0 https://github.com/protocolbuffers/protobuf/releases/download/v21.2/protoc-21.2-linux-x86_64.zip -o /tmp/protoc-21.2-linux-x86_64.zip && \
-    unzip /tmp/protoc-21.2-linux-x86_64.zip -d /opt/protoc
-ENV PATH /opt/protoc/bin:$PATH
-
-# Make Python3 the default
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
-
-# Developer tools
-RUN apt-get update && apt-get install -y \
-    bash-completion
-# Install gcovr from PIP to get a newer version than in apt repositories
-RUN python3 -m pip install gcovr
+    unzip /tmp/protoc-21.2-linux-x86_64.zip -d /opt/protoc &&\
+    rm /tmp/protoc-21.2-linux-x86_64.zip
+ENV PATH=/opt/protoc/bin:$PATH
 
 # Install Go, used for the tools in tools/go and for test/gounittest
-ENV GOPATH /opt/go
-ENV GOROOT /opt/go_dist/go
-ENV PATH $GOROOT/bin:$GOPATH/bin:$PATH
+ENV GOPATH=/opt/go
+ENV GOROOT=/opt/go_dist/go
+ENV PATH=$GOROOT/bin:$GOPATH/bin:$PATH
 RUN mkdir -p /opt/go_dist && \
     curl https://dl.google.com/go/go1.19.3.linux-amd64.tar.gz | tar -xz -C /opt/go_dist
 
 # Install lcov from release (the one from the repos is too old).
-RUN cd /opt && wget https://github.com/linux-test-project/lcov/releases/download/v1.14/lcov-1.14.tar.gz && tar -xf lcov-1.14.tar.gz
-ENV PATH /opt/lcov-1.14/bin:$PATH
+RUN curl -L https://github.com/linux-test-project/lcov/releases/download/v1.14/lcov-1.14.tar.gz | tar -xz -C /opt
+ENV PATH=/opt/lcov-1.14/bin:$PATH
 
 # Install rust compiler
-ENV PATH /opt/cargo/bin:$PATH
+# Since bindgen embeds information about its target directory, use a deterministic path for it.
+ENV PATH=/opt/cargo/bin:$PATH
 ENV RUSTUP_HOME=/opt/rustup
-COPY src/rust/rust-toolchain.toml /tmp/rust-toolchain.toml
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | CARGO_HOME=/opt/cargo sh -s -- --default-toolchain $(grep -oP '(?<=channel = ")[^"]+' /tmp/rust-toolchain.toml) -y
-RUN rustup target add thumbv7em-none-eabi
-RUN rustup component add rustfmt
-RUN rustup component add clippy
-RUN rustup component add rust-src
-RUN CARGO_HOME=/opt/cargo cargo install cbindgen --version 0.26.0 --locked
-RUN CARGO_HOME=/opt/cargo cargo install bindgen-cli --version 0.69.4 --locked
+RUN --mount=source=tools/prost-build-proto,target=/mnt/prost-build-proto,rw \
+    --mount=source=src/rust/rust-toolchain.toml,target=/mnt/rust-toolchain.toml \
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs |\
+    CARGO_HOME=/opt/cargo sh -s -- --default-toolchain $(grep -oP '(?<=channel = ")[^"]+' /mnt/rust-toolchain.toml) -y &&\
+    rustup target add thumbv7em-none-eabi &&\
+    rustup component add rustfmt &&\
+    rustup component add clippy &&\
+    rustup component add rust-src &&\
+    CARGO_HOME=/opt/cargo cargo install cbindgen --version 0.26.0 --locked &&\
+    CARGO_HOME=/opt/cargo cargo install bindgen-cli --version 0.69.4 --locked --target-dir=/tmp/bindgen-target &&\
+    CARGO_HOME=/opt/cargo cargo install --path /mnt/prost-build-proto --locked &&\
+    rm -r /tmp/bindgen-target /opt/cargo/registry/index /opt/cargo/.global-cache
 
 # Until cargo vendor supports vendoring dependencies of the rust std libs we
 # need a copy of this file next to the toml file. It also has to be world
 # writable so that invocations of `cargo vendor` can update it. Below is the
 # tracking issue for `cargo vendor` to support rust std libs.
 # https://github.com/rust-lang/wg-cargo-std-aware/issues/23
-RUN cp "$(rustc --print=sysroot)/lib/rustlib/src/rust/Cargo.lock" "$(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/"
-RUN chmod 777 $(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/Cargo.lock
-
-COPY tools/prost-build-proto prost-build-proto
-RUN CARGO_HOME=/opt/cargo cargo install --path prost-build-proto --locked
-
-# Clean temporary files to reduce image size
-RUN rm -rf /var/lib/apt/lists/*
+RUN cp "$(rustc --print=sysroot)/lib/rustlib/src/rust/Cargo.lock" "$(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/" &&\
+    chmod 777 $(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/Cargo.lock

Dockerfile.ubuntu

@@ -0,0 +1,62 @@
+# Copyright 2019 Shift Cryptosecurity AG
+# Copyright 2020 Shift Crypto AG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Latest Ubuntu LTS
+FROM ubuntu:22.04@sha256:0eb0f877e1c869a300c442c41120e778db7161419244ee5cbc6fa5f134e74736
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y \
+    wget \
+    curl \
+    gnupg2 \
+    && rm -r /var/lib/apt/lists/*
+
+# for clang-*-18, see https://apt.llvm.org/
+RUN echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" >> /etc/apt/sources.list && \
+    echo "deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" >> /etc/apt/sources.list && \
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+    rm /root/.wget-hsts
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y \
+    make \
+    git \
+    pkg-config \
+    nano \
+    jq \
+    unzip \
+    python3 \
+    python3-pip \
+    binutils \
+    valgrind \
+    cmake \
+    autotools-dev \
+    automake \
+    autoconf \
+    libtool \
+    libcmocka-dev \
+    bzip2 \
+    llvm-18 \
+    gcc-10 \
+    libc6-i386 \
+    lib32stdc++6 \
+    lib32z1 \
+    libusb-1.0-0-dev \
+    libudev-dev \
+    libhidapi-dev \
+    doxygen \
+    graphviz \
+    clang-format-18 \
+    clang-tidy-18 \
+    bash-completion \
+    && rm -r /var/lib/apt/lists/*

Makefile

@@ -14,6 +14,10 @@
 
 # This makefile is used as a command runner and not for tracking dependencies between recipies
 
+# This is the current version of the development container. If you make changes to the Dockerfile,
+# bump this number to release a new version.
+CONTAINER_VERSION := 41
+
 .DEFAULT_GOAL := firmware
 SANITIZE ?= ON
 simulator: SANITIZE = OFF
@@ -129,15 +133,15 @@ jlink-flash-factory-setup: | build
 jlink-flash-firmware-semihosting: | build-semihosting
 	JLinkExe -if SWD -device ATSAMD51J20 -speed 4000 -autoconnect 1 -CommanderScript ./build-semihosting/scripts/firmware.jlink
 dockerinit:
-	./scripts/container.sh build --pull --platform linux/amd64 --force-rm --no-cache -t shiftcrypto/firmware_v2 .
+	./scripts/container.sh pull shiftcrypto/firmware_v2:${CONTAINER_VERSION}
 dockerdev:
-	./scripts/dockerenv.sh
+	./scripts/dockerenv.sh ${CONTAINER_VERSION}
 dockerrel:
-	./scripts/dockerenv.sh release
+	./scripts/dockerenv.sh release ${CONTAINER_VERSION}
 generate-atecc608-config:
 	cd tools/atecc608 && go run main.go
 ci:
-	./.ci/ci
+	./.ci/ci ${CONTAINER_VERSION}
 prepare-tidy: | build build-build
 	make -C build rust-cbindgen
 	make -C build-build rust-cbindgen