From 87bfe1c467c7cccbc03fa0d9e00e2e22fc9b92ba Mon Sep 17 00:00:00 2001 From: Jean-Francois Panisset Date: Mon, 21 Apr 2025 22:30:31 -0700 Subject: [PATCH] Use Docker BuildKit secrets for Conan authentication We don't have a simple way to persist Conan authentication against a remote repository, instead use the BuildKit secrets mechanism to pass CONAN_LOGIN_USERNAME and CONAN_PASSWORD secrets env vars to the build container. Signed-off-by: Jean-Francois Panisset --- packages/common/Dockerfile | 8 ++--- python/aswfdocker/builder.py | 41 ++++++++++++++----------- python/aswfdocker/tests/test_builder.py | 21 +++++++++---- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/packages/common/Dockerfile b/packages/common/Dockerfile index a3fd6213..14d81c12 100644 --- a/packages/common/Dockerfile +++ b/packages/common/Dockerfile @@ -162,12 +162,6 @@ ENV GCC_INSTALL_PREFIX=/opt/rh/${ASWF_DTS_PREFIX}-${ASWF_DTS_VERSION}/root/usr # With "conan create --profile" we no longer need to set a default profile ENV CONAN_HOME=${CONAN_USER_HOME}/.conan2 -# Allow "conan upload" to authenticate without persistent storage -ARG CONAN_LOGIN_USERNAME -ENV CONAN_LOGIN_USERNAME=${CONAN_LOGIN_USERNAME} -ARG CONAN_PASSWORD -ENV CONAN_PASSWORD=${CONAN_PASSWORD} - RUN --mount=type=cache,target=${CONAN_USER_HOME}/d \ --mount=type=cache,target=${CCACHE_DIR} \ --mount=type=bind,rw,target=${CONAN_USER_HOME}/.conan2,source=packages/conan/settings \ @@ -188,6 +182,8 @@ RUN --mount=type=cache,target=${CONAN_USER_HOME}/d \ --mount=type=cache,target=${CCACHE_DIR} \ --mount=type=bind,rw,target=${CONAN_USER_HOME}/.conan2,source=packages/conan/settings \ --mount=type=bind,rw,target=${CONAN_USER_HOME}/recipes,source=packages/conan/recipes \ + --mount=type=secret,id=conan_login_username,env=CONAN_LOGIN_USERNAME \ + --mount=type=secret,id=conan_password,env=CONAN_PASSWORD \ if [ -n "${ASWF_CONAN_PUSH}" ] ; then \ conan upload --remote ${ASWF_PKG_ORG} ${ASWF_PKG_NAME}/${ASWF_PKG_VERSION}@${ASWF_PKG_ORG}/${ASWF_CONAN_CHANNEL} ;\ else \ diff --git a/python/aswfdocker/builder.py b/python/aswfdocker/builder.py index 9bdd6c80..60ddd80c 100644 --- a/python/aswfdocker/builder.py +++ b/python/aswfdocker/builder.py @@ -131,6 +131,10 @@ def make_bake_dict( }, "tags": tags, "output": ["type=registry,push=true" if self.push else "type=docker"], + "secrets": [ + "id=conan_login_username,env=CONAN_LOGIN_USERNAME", + "id=conan_password,env=CONAN_PASSWORD", + ], } if self.group_info.type == constants.ImageType.PACKAGE: if use_conan: @@ -236,24 +240,25 @@ def _build_conan_package( ): # pylint: disable=consider-using-f-string major_version = utils.get_major_version(version) - version_info = self.index.version_info(major_version) - base_cmd = self._get_conan_base_cmd(version_info) - if conan_login: - # We keep this as a separate step: the end result is to store credentials in - # packages/conan/.conan/.conan.db which is not thread safe: once we are able - # to run Conan builds from a single "docker buildx bake" invocation, we will - # want to keep the login step separate. - self._run_in_docker( - base_cmd, - [ - "conan", - "remote", - "auth", - self.build_info.docker_org, - ], - dry_run, - ) - # + # version_info = self.index.version_info(major_version) + # base_cmd = self._get_conan_base_cmd(version_info) + # if conan_login: + # # "conan remote auth" stores credentials in + # # ${CONAN_HOME]/.conan2/credentials.json but we don't have a simple way to persist + # # this file between build steps, since instead we will use the secrets mechanism + # # in the buildx bake file to pass the CONNA_LOGIN_USERNAME and CONAN_PASSWORD + # # values as environment variables to allow `conan upload" to authenticate on the fly. + # self._run_in_docker( + # base_cmd, + # [ + # "conan", + # "remote", + # "auth", + # self.build_info.docker_org, + # ], + # dry_run, + # ) + # These are kept for reference, they now live in # packages/common/Dockerfile # diff --git a/python/aswfdocker/tests/test_builder.py b/python/aswfdocker/tests/test_builder.py index b86b9748..9e59319c 100644 --- a/python/aswfdocker/tests/test_builder.py +++ b/python/aswfdocker/tests/test_builder.py @@ -194,6 +194,10 @@ def test_image_base_2019clang_dict(self): f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-openvdb:{openvdb_version}", ], "output": ["type=docker"], + "secrets": [ + "id=conan_login_username,env=CONAN_LOGIN_USERNAME", + "id=conan_password,env=CONAN_PASSWORD", + ], } }, }, @@ -281,6 +285,10 @@ def test_image_base_2019_2020_dict(self): f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:{base_versions[1]}", ], "output": ["type=docker"], + "secrets": [ + "id=conan_login_username,env=CONAN_LOGIN_USERNAME", + "id=conan_password,env=CONAN_PASSWORD", + ], }, "ci-base-2019": { "context": ".", @@ -346,6 +354,10 @@ def test_image_base_2019_2020_dict(self): f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-base:{base_versions[0]}", ], "output": ["type=docker"], + "secrets": [ + "id=conan_login_username,env=CONAN_LOGIN_USERNAME", + "id=conan_password,env=CONAN_PASSWORD", + ], }, }, }, @@ -503,17 +515,15 @@ def test_builderlist_cli_conan(self): tempfile.gettempdir(), "docker-bake-PACKAGE-vfx1-2-2019-2020.json" ) cmds = result.output.strip().splitlines() - # We expect 5 steps + # We expect 3 steps # 1 - docker buildx to build the non-Conan packages - # 2 - docker run to login to repository (2x for each image) - # 3 - docker buildx to build and upload (2x for each openexr package) - self.assertEqual(len(cmds), 5) + # 2 - docker buildx to build and upload (2x for each openexr package) + self.assertEqual(len(cmds), 3) self.assertEqual( cmds[self._i], f"INFO:aswfdocker.builder:Would run: 'docker buildx bake -f {bake_path} --progress auto'", ) self._i += 1 - self._assertEndsWith(cmds, "conan remote auth aswftesting'") self.assertEqual( cmds[self._i], f"INFO:aswfdocker.builder:Would run: 'docker buildx bake -f {bake_path} " @@ -521,7 +531,6 @@ def test_builderlist_cli_conan(self): + "--progress auto ci-package-openexr-2019'", ) self._i += 1 - self._assertEndsWith(cmds, "conan remote auth aswftesting'") self.assertEqual( cmds[self._i], f"INFO:aswfdocker.builder:Would run: 'docker buildx bake -f {bake_path} "