Skip to content

Use Docker BuildKit secrets for Conan authentication #239

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jfpanisset merged 1 commit into from
Apr 22, 2025
Merged

Use Docker BuildKit secrets for Conan authentication #239

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 2 additions & 6 deletions packages/common/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,12 +162,6 @@ ENV GCC_INSTALL_PREFIX=/opt/rh/${ASWF_DTS_PREFIX}-${ASWF_DTS_VERSION}/root/usr
# With "conan create --profile" we no longer need to set a default profile
ENV CONAN_HOME=${CONAN_USER_HOME}/.conan2

# Allow "conan upload" to authenticate without persistent storage
ARG CONAN_LOGIN_USERNAME
ENV CONAN_LOGIN_USERNAME=${CONAN_LOGIN_USERNAME}
ARG CONAN_PASSWORD
ENV CONAN_PASSWORD=${CONAN_PASSWORD}

RUN --mount=type=cache,target=${CONAN_USER_HOME}/d \
--mount=type=cache,target=${CCACHE_DIR} \
--mount=type=bind,rw,target=${CONAN_USER_HOME}/.conan2,source=packages/conan/settings \
Expand All @@ -188,6 +182,8 @@ RUN --mount=type=cache,target=${CONAN_USER_HOME}/d \
--mount=type=cache,target=${CCACHE_DIR} \
--mount=type=bind,rw,target=${CONAN_USER_HOME}/.conan2,source=packages/conan/settings \
--mount=type=bind,rw,target=${CONAN_USER_HOME}/recipes,source=packages/conan/recipes \
--mount=type=secret,id=conan_login_username,env=CONAN_LOGIN_USERNAME \
--mount=type=secret,id=conan_password,env=CONAN_PASSWORD \
if [ -n "${ASWF_CONAN_PUSH}" ] ; then \
conan upload --remote ${ASWF_PKG_ORG} ${ASWF_PKG_NAME}/${ASWF_PKG_VERSION}@${ASWF_PKG_ORG}/${ASWF_CONAN_CHANNEL} ;\
else \
Expand Down
41 changes: 23 additions & 18 deletions python/aswfdocker/builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,10 @@ def make_bake_dict(
},
"tags": tags,
"output": ["type=registry,push=true" if self.push else "type=docker"],
"secrets": [
"id=conan_login_username,env=CONAN_LOGIN_USERNAME",
"id=conan_password,env=CONAN_PASSWORD",
],
}
if self.group_info.type == constants.ImageType.PACKAGE:
if use_conan:
Expand Down Expand Up @@ -236,24 +240,25 @@ def _build_conan_package(
):
# pylint: disable=consider-using-f-string
major_version = utils.get_major_version(version)
version_info = self.index.version_info(major_version)
base_cmd = self._get_conan_base_cmd(version_info)
if conan_login:
# We keep this as a separate step: the end result is to store credentials in
# packages/conan/.conan/.conan.db which is not thread safe: once we are able
# to run Conan builds from a single "docker buildx bake" invocation, we will
# want to keep the login step separate.
self._run_in_docker(
base_cmd,
[
"conan",
"remote",
"auth",
self.build_info.docker_org,
],
dry_run,
)
#
# version_info = self.index.version_info(major_version)
# base_cmd = self._get_conan_base_cmd(version_info)
# if conan_login:
# # "conan remote auth" stores credentials in
# # ${CONAN_HOME]/.conan2/credentials.json but we don't have a simple way to persist
# # this file between build steps, since instead we will use the secrets mechanism
# # in the buildx bake file to pass the CONNA_LOGIN_USERNAME and CONAN_PASSWORD
# # values as environment variables to allow `conan upload" to authenticate on the fly.
# self._run_in_docker(
# base_cmd,
# [
# "conan",
# "remote",
# "auth",
# self.build_info.docker_org,
# ],
# dry_run,
# )

# These are kept for reference, they now live in
# packages/common/Dockerfile
#
Expand Down
21 changes: 15 additions & 6 deletions python/aswfdocker/tests/test_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,10 @@ def test_image_base_2019clang_dict(self):
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-openvdb:{openvdb_version}",
],
"output": ["type=docker"],
"secrets": [
"id=conan_login_username,env=CONAN_LOGIN_USERNAME",
"id=conan_password,env=CONAN_PASSWORD",
],
}
},
},
Expand Down Expand Up @@ -281,6 +285,10 @@ def test_image_base_2019_2020_dict(self):
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:{base_versions[1]}",
],
"output": ["type=docker"],
"secrets": [
"id=conan_login_username,env=CONAN_LOGIN_USERNAME",
"id=conan_password,env=CONAN_PASSWORD",
],
},
"ci-base-2019": {
"context": ".",
Expand Down Expand Up @@ -346,6 +354,10 @@ def test_image_base_2019_2020_dict(self):
f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:{base_versions[0]}",
],
"output": ["type=docker"],
"secrets": [
"id=conan_login_username,env=CONAN_LOGIN_USERNAME",
"id=conan_password,env=CONAN_PASSWORD",
],
},
},
},
Expand Down Expand Up @@ -503,25 +515,22 @@ def test_builderlist_cli_conan(self):
tempfile.gettempdir(), "docker-bake-PACKAGE-vfx1-2-2019-2020.json"
)
cmds = result.output.strip().splitlines()
# We expect 5 steps
# We expect 3 steps
# 1 - docker buildx to build the non-Conan packages
# 2 - docker run to login to repository (2x for each image)
# 3 - docker buildx to build and upload (2x for each openexr package)
self.assertEqual(len(cmds), 5)
# 2 - docker buildx to build and upload (2x for each openexr package)
self.assertEqual(len(cmds), 3)
self.assertEqual(
cmds[self._i],
f"INFO:aswfdocker.builder:Would run: 'docker buildx bake -f {bake_path} --progress auto'",
)
self._i += 1
self._assertEndsWith(cmds, "conan remote auth aswftesting'")
self.assertEqual(
cmds[self._i],
f"INFO:aswfdocker.builder:Would run: 'docker buildx bake -f {bake_path} "
+ "--set=*.output=type=cacheonly --set=*.target.target=ci-conan-package-builder "
+ "--progress auto ci-package-openexr-2019'",
)
self._i += 1
self._assertEndsWith(cmds, "conan remote auth aswftesting'")
self.assertEqual(
cmds[self._i],
f"INFO:aswfdocker.builder:Would run: 'docker buildx bake -f {bake_path} "
Expand Down
Loading