diff --git a/.test/example-commands.sh b/.test/example-commands.sh index f165244..4bcbaf8 100644 --- a/.test/example-commands.sh +++ b/.test/example-commands.sh @@ -3,15 +3,49 @@ # # -SOURCE_DATE_EPOCH=1700741054 docker buildx build --progress=plain --provenance=mode=max --sbom=generator="$BASHBREW_BUILDKIT_SBOM_GENERATOR" --output '"type=oci","dest=temp.tar","annotation.org.opencontainers.image.source=https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli","annotation-manifest-descriptor.org.opencontainers.image.source=https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli","annotation.org.opencontainers.image.revision=6d541d27b5dd12639e5a33a675ebca04d3837d74","annotation-manifest-descriptor.org.opencontainers.image.revision=6d541d27b5dd12639e5a33a675ebca04d3837d74","annotation.org.opencontainers.image.version=24.0.7-cli","annotation-manifest-descriptor.org.opencontainers.image.version=24.0.7-cli","annotation.org.opencontainers.image.url=https://hub.docker.com/_/docker","annotation-manifest-descriptor.org.opencontainers.image.url=https://hub.docker.com/_/docker"' --tag 'docker:24.0.7-cli' --tag 'docker:24.0-cli' --tag 'docker:24-cli' --tag 'docker:cli' --tag 'docker:24.0.7-cli-alpine3.18' --tag 'oisupport/staging-amd64:4b199ac326c74b3058a147e14f553af9e8e1659abc29bd3e82c9c9807b66ee43' --platform 'linux/amd64' --build-context 'alpine:3.18=docker-image://alpine:3.18@sha256:d695c3de6fcd8cfe3a6222b0358425d40adfd129a8a47c3416faff1a8aece389' --build-arg BUILDKIT_SYNTAX="$BASHBREW_BUILDKIT_SYNTAX" --file 'Dockerfile' 'https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli' -# -# +SOURCE_DATE_EPOCH=1700741054 \ + docker buildx build --progress=plain \ + --provenance=mode=max \ + --sbom=generator="$BASHBREW_BUILDKIT_SBOM_GENERATOR" \ + --output '"type=oci","dest=temp.tar"' \ + --annotation 'org.opencontainers.image.source=https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli' \ + --annotation 'org.opencontainers.image.revision=6d541d27b5dd12639e5a33a675ebca04d3837d74' \ + --annotation 'org.opencontainers.image.created=2023-11-23T12:04:14Z' \ + --annotation 'org.opencontainers.image.version=24.0.7-cli' \ + --annotation 'org.opencontainers.image.url=https://hub.docker.com/_/docker' \ + --annotation 'manifest-descriptor:org.opencontainers.image.source=https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli' \ + --annotation 'manifest-descriptor:org.opencontainers.image.revision=6d541d27b5dd12639e5a33a675ebca04d3837d74' \ + --annotation 'manifest-descriptor:org.opencontainers.image.created=1970-01-01T00:00:00Z' \ + --annotation 'manifest-descriptor:org.opencontainers.image.version=24.0.7-cli' \ + --annotation 'manifest-descriptor:org.opencontainers.image.url=https://hub.docker.com/_/docker' \ + --tag 'docker:24.0.7-cli' \ + --tag 'docker:24.0-cli' \ + --tag 'docker:24-cli' \ + --tag 'docker:cli' \ + --tag 'docker:24.0.7-cli-alpine3.18' \ + --tag 'oisupport/staging-amd64:4b199ac326c74b3058a147e14f553af9e8e1659abc29bd3e82c9c9807b66ee43' \ + --platform 'linux/amd64' \ + --build-context 'alpine:3.18=docker-image://alpine:3.18@sha256:d695c3de6fcd8cfe3a6222b0358425d40adfd129a8a47c3416faff1a8aece389' \ + --build-arg BUILDKIT_SYNTAX="$BASHBREW_BUILDKIT_SYNTAX" \ + --file 'Dockerfile' \ + 'https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli' mkdir temp tar -xvf temp.tar -C temp -jq '.manifests |= (del(.[].annotations) | unique)' temp/index.json > temp/index.json.new +rm temp.tar +jq ' + .manifests |= ( + del(.[].annotations) + | unique + | if length != 1 then + error("unexpected number of manifests: " + length) + else . end + ) +' temp/index.json > temp/index.json.new mv temp/index.json.new temp/index.json +# +# crane push temp 'oisupport/staging-amd64:4b199ac326c74b3058a147e14f553af9e8e1659abc29bd3e82c9c9807b66ee43' -rm -rf temp temp.tar +rm -rf temp # # docker:24.0.7-windowsservercore-ltsc2022 [windows-amd64] @@ -20,7 +54,21 @@ docker pull 'mcr.microsoft.com/windows/servercore:ltsc2022@sha256:d4ab2dd7d3d0fc docker tag 'mcr.microsoft.com/windows/servercore:ltsc2022@sha256:d4ab2dd7d3d0fce6edc5df459565a4c96bbb1d0148065b215ab5ddcab1e42eb4' 'mcr.microsoft.com/windows/servercore:ltsc2022' # # -SOURCE_DATE_EPOCH=1700741054 DOCKER_BUILDKIT=0 docker build --tag 'docker:24.0.7-windowsservercore-ltsc2022' --tag 'docker:24.0-windowsservercore-ltsc2022' --tag 'docker:24-windowsservercore-ltsc2022' --tag 'docker:windowsservercore-ltsc2022' --tag 'docker:24.0.7-windowsservercore' --tag 'docker:24.0-windowsservercore' --tag 'docker:24-windowsservercore' --tag 'docker:windowsservercore' --tag 'oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1fee7582174de82ae861613ae3072e' --platform 'windows/amd64' --file 'Dockerfile' 'https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/windows/windowsservercore-ltsc2022' +SOURCE_DATE_EPOCH=1700741054 \ + DOCKER_BUILDKIT=0 \ + docker build \ + --tag 'docker:24.0.7-windowsservercore-ltsc2022' \ + --tag 'docker:24.0-windowsservercore-ltsc2022' \ + --tag 'docker:24-windowsservercore-ltsc2022' \ + --tag 'docker:windowsservercore-ltsc2022' \ + --tag 'docker:24.0.7-windowsservercore' \ + --tag 'docker:24.0-windowsservercore' \ + --tag 'docker:24-windowsservercore' \ + --tag 'docker:windowsservercore' \ + --tag 'oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1fee7582174de82ae861613ae3072e' \ + --platform 'windows/amd64' \ + --file 'Dockerfile' \ + 'https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/windows/windowsservercore-ltsc2022' # # docker push 'oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1fee7582174de82ae861613ae3072e' diff --git a/.test/test.sh b/.test/test.sh index 73a3f66..3d5cff5 100755 --- a/.test/test.sh +++ b/.test/test.sh @@ -19,7 +19,7 @@ time "$dir/../sources.sh" "$@" > "$dir/sources.json" time "$dir/../builds.sh" --cache "$dir/cache-builds.json" "$dir/sources.json" > "$dir/builds.json" # generate an "example commands" file so that changes to generated commands are easier to review -jq -r -L "$dir/.." ' +SOURCE_DATE_EPOCH=0 jq -r -L "$dir/.." ' include "meta"; [ first(.[] | select(normalized_builder == "buildkit")), diff --git a/meta.jq b/meta.jq index 002b816..69ddb6c 100644 --- a/meta.jq +++ b/meta.jq @@ -17,19 +17,26 @@ def normalized_builder: end else . end ; +def docker_uses_containerd_storage: + # TODO somehow detect docker-with-containerd-storage + false +; # input: "build" object (with "buildId" top level key) # output: boolean def should_use_docker_buildx_driver: normalized_builder == "buildkit" and ( - .build.arch as $arch - # bashbrew remote arches --json tianon/buildkit:0.12 | jq '.arches | keys_unsorted' -c - | ["amd64","arm32v5","arm32v6","arm32v7","arm64v8","i386","mips64le","ppc64le","riscv64","s390x"] - # TODO this needs to be based on the *host* architecture, not the *target* architecture (amd64 vs i386) - | index($arch) - | not + docker_uses_containerd_storage + or ( + .build.arch as $arch + # bashbrew remote arches --json tianon/buildkit:0.12 | jq '.arches | keys_unsorted' -c + | ["amd64","arm32v5","arm32v6","arm32v7","arm64v8","i386","mips64le","ppc64le","riscv64","s390x"] + # TODO this needs to be based on the *host* architecture, not the *target* architecture (amd64 vs i386) + | index($arch) + | not + # TODO "failed to read dockerfile: failed to load cache key: subdir not supported yet" asdflkjalksdjfklasdjfklajsdklfjasdklgfnlkasdfgbhnkljasdhgouiahsdoifjnask,.dfgnklasdbngoikasdhfoiasjdklfjasdlkfjalksdjfkladshjflikashdbgiohasdfgiohnaskldfjhnlkasdhfnklasdhglkahsdlfkjasdlkfjadsklfjsdl (hence "tianon/buildkit" instead of "moby/buildkit"; need *all* the arches we care about/support for consistent support) + ) ) - # TODO "failed to read dockerfile: failed to load cache key: subdir not supported yet" asdflkjalksdjfklasdjfklajsdklfjasdklgfnlkasdfgbhnkljasdhgouiahsdoifjnask,.dfgnklasdbngoikasdhfoiasjdklfjasdlkfjalksdjfkladshjflikashdbgiohasdfgiohnaskldfjhnlkasdhfnklasdhglkahsdlfkjasdlkfjadsklfjsdl (hence "tianon/buildkit" instead of "moby/buildkit") ; # input: "build" object (with "buildId" top level key) # output: string "pull command" ("docker pull ..."), may be multiple lines, expects to run in Bash with "set -Eeuo pipefail", might be empty @@ -71,66 +78,110 @@ def git_build_url: ) + "#" + .GitCommit + ":" + .Directory ; # input: "build" object (with "buildId" top level key) +# output: map of annotations to set +def build_annotations($buildUrl): + { + # https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/annotations.md#pre-defined-annotation-keys + "org.opencontainers.image.source": $buildUrl, + "org.opencontainers.image.revision": .source.entry.GitCommit, + "org.opencontainers.image.created": (.source.entry.SOURCE_DATE_EPOCH | strftime("%FT%TZ")), # see notes below about image index vs image manifest + + # TODO come up with less assuming values here? (Docker Hub assumption, tag ordering assumption) + "org.opencontainers.image.version": ( # value of the first image tag + first(.source.allTags[] | select(contains(":"))) + | sub("^.*:"; "") + # TODO maybe we should do the first, longest, non-latest tag instead of just the first tag? + ), + "org.opencontainers.image.url": ( # URL to Docker Hub + first(.source.allTags[] | select(contains(":"))) + | sub(":.*$"; "") + | if contains("/") then + "r/" + . + else + "_/" + . + end + | "https://hub.docker.com/" + . + ), + + # TODO org.opencontainers.image.vendor ? (feels leaky to put "Docker Official Images" here when this is all otherwise mostly generic) + } + | with_entries(select(.value)) # strip off anything missing a value (possibly "source", "url", "version", etc) +; +def build_annotations: + build_annotations(git_build_url) +; +# input: multi-line string with indentation and comments +# output: multi-line string with less indentation and no comments +def unindent_and_decomment_jq($indents): + # trim out comment lines and unnecessary indentation + gsub("(?m)^(\t+#[^\n]*\n?|\t{\($indents)}(?.*)$)"; "\(.extra // "")") + # trim out empty lines + | gsub("\n\n+"; "\n") +; +# input: "build" object (with "buildId" top level key) # output: string "build command" ("docker buildx build ..."), may be multiple lines, expects to run in Bash with "set -Eeuo pipefail" def build_command: normalized_builder as $builder - | git_build_url as $buildUrl | if $builder == "buildkit" then - [ + git_build_url as $buildUrl + | ( + (should_use_docker_buildx_driver | not) + or docker_uses_containerd_storage + ) as $supportsAnnotationsAndAttestsations + | [ ( [ @sh "SOURCE_DATE_EPOCH=\(.source.entry.SOURCE_DATE_EPOCH)", # TODO EXPERIMENTAL_BUILDKIT_SOURCE_POLICY=<(jq ...) "docker buildx build --progress=plain", - if should_use_docker_buildx_driver then "--load" else # TODO if we get containerd integration and thus use "--load" unconditionally again, we should update this to still set annotations! (and still gate SBOMs on appropriate scanner-supported architectures) + if $supportsAnnotationsAndAttestsations then "--provenance=mode=max", # see "bashbrew remote arches docker/scout-sbom-indexer:1" (we need the SBOM scanner to be runnable on the host architecture) # bashbrew remote arches --json docker/scout-sbom-indexer:1 | jq '.arches | keys_unsorted' -c if .build.arch as $arch | ["amd64","arm32v5","arm32v7","arm64v8","i386","ppc64le","riscv64","s390x"] | index($arch) then # TODO this needs to be based on the *host* architecture, not the *target* architecture (amd64 vs i386) "--sbom=generator=\"$BASHBREW_BUILDKIT_SBOM_GENERATOR\"" + # TODO this should also be totally optional -- for example, Tianon doesn't want SBOMs on his personal images else empty end, - ( - "--output " + ( - [ - "type=oci", # TODO find a better way to build/tag with a full list of tags but only actually *push* to one of them so we don't have to round-trip through containerd - "dest=temp.tar", # TODO choose/find a good "safe" place to put this (temporarily) - ( - { - # https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/annotations.md#pre-defined-annotation-keys - "org.opencontainers.image.source": $buildUrl, - "org.opencontainers.image.revision": .source.entry.GitCommit, - - # TODO come up with less assuming values here? (Docker Hub assumption, tag ordering assumption) - "org.opencontainers.image.version": ( # value of the first image tag - first(.source.allTags[] | select(contains(":"))) - | sub("^.*:"; "") - # TODO maybe we should do the first, longest, non-latest tag instead of just the first tag? - ), - "org.opencontainers.image.url": ( # URL to Docker Hub - first(.source.allTags[] | select(contains(":"))) - | sub(":.*$"; "") - | if contains("/") then - "r/" + . - else - "_/" + . - end - | "https://hub.docker.com/" + . - ), - # TODO org.opencontainers.image.vendor ? (feels leaky to put "Docker Official Images" here when this is all otherwise mostly generic) - } - | to_entries[] | select(.value != null) | - "annotation." + .key + "=" + .value, - "annotation-manifest-descriptor." + .key + "=" + .value - ), - empty - ] - | @csv - | @sh - ) - ), empty - end, + else empty end, + "--output " + ( + [ + if should_use_docker_buildx_driver then + "type=docker" + else + "type=oci", + "dest=temp.tar", # TODO choose/find a good "safe" place to put this (temporarily) + empty + end, + empty + ] + | @csv + | @sh + ), + ( + if $supportsAnnotationsAndAttestsations then + build_annotations($buildUrl) + | to_entries + # separate loops so that "image manifest" annotations are grouped separate from the index/descriptor annotations (easier to read) + | ( + .[] + | @sh "--annotation \(.key + "=" + .value)" + ), + ( + .[] + | @sh "--annotation \( + "manifest-descriptor:" + .key + "=" + + if .key == "org.opencontainers.image.created" then + # the "current" time breaks reproducibility (for the purposes of build verification), so we put "now" in the image index but "SOURCE_DATE_EPOCH" in the image manifest (which is the thing we'd ideally like to have reproducible, eventually) + (env.SOURCE_DATE_EPOCH // now) | tonumber | strftime("%FT%TZ") + # (this assumes the actual build is going to happen shortly after generating the command) + else .value end + )", + empty + ) + else empty end + ), ( .source.arches[].tags[], .source.arches[].archTags[], @@ -148,8 +199,26 @@ def build_command: @sh "--file \(.source.entry.File)", ($buildUrl | @sh), empty - ] | join(" ") + ] | join(" \\\n\t") ), + if should_use_docker_buildx_driver then empty else + # munge the tarball into a suitable "oci layout" directory (ready for "crane push") + "mkdir temp", + "tar -xvf temp.tar -C temp", + "rm temp.tar", + # munge the index to what crane wants ("Error: layout contains 5 entries, consider --index") + @sh "jq \(" + .manifests |= ( + del(.[].annotations) + | unique + | if length != 1 then + error(\"unexpected number of manifests: \" + length) + else . end + ) + " | unindent_and_decomment_jq(4)) temp/index.json > temp/index.json.new", + "mv temp/index.json.new temp/index.json", + empty + end, # possible improvements in buildkit/buildx that could help us: # - allowing OCI output directly to a directory instead of a tar (thus getting symmetry with the oci-layout:// inputs it can take) # - allowing tag as one thing and push as something else, potentially mutually exclusive @@ -158,7 +227,8 @@ def build_command: empty ] | join("\n") elif $builder == "classic" then - [ + git_build_url as $buildUrl + | [ ( [ @sh "SOURCE_DATE_EPOCH=\(.source.entry.SOURCE_DATE_EPOCH)", @@ -175,7 +245,7 @@ def build_command: ($buildUrl | @sh), empty ] - | join(" ") + | join(" \\\n\t") ), empty ] | join("\n") @@ -199,14 +269,9 @@ def push_command: @sh "docker push \(.build.img)" elif $builder == "buildkit" then [ - # extract to a directory and "crane push" (easier to get correct than "ctr image import" + "ctr image push", especially with authentication) - "mkdir temp", - "tar -xvf temp.tar -C temp", - # munge the index to what crane wants ("Error: layout contains 5 entries, consider --index") - @sh "jq \(".manifests |= (del(.[].annotations) | unique)") temp/index.json > temp/index.json.new", - "mv temp/index.json.new temp/index.json", + # "crane push" is easier to get correct than "ctr image import" + "ctr image push", especially with authentication @sh "crane push temp \(.build.img)", - "rm -rf temp temp.tar", + "rm -rf temp", empty ] | join("\n") elif $builder == "oci-import" then