Skip to content

Commit

Permalink
Refactor meta.jq to prepare for signing and oci-import implementa…
Browse files Browse the repository at this point in the history 
…tion

- explicit "containerd image storage in dockerd" placeholder
- explicit "annotations" helper/generator (so they can be added in the `oci-import` builder later)
- setting the `org.opencontainers.image.created` annoation inside the image index to the actual build date (otherwise `SOURCE_DATE_EPOCH` makes it hard to tell an image is actually freshly built)
- more whitespace in generated commands for better readability
- extract OCI tar in the build step (so the "output" of the build is an OCI layout directory vs tarball) instead of the pull step (so we can more ergonomically add pre-push signing)
- use buildx's `--annotation` flag for making annotations easier to read
  • Loading branch information
@tianon
tianon committed Jan 25, 2024
1 parent 7d4bdda commit af7b858
Show file tree
Hide file tree
Showing 3 changed files with 180 additions and 67 deletions.
60 changes: 54 additions & 6 deletions .test/example-commands.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,49 @@

# </pull>
# <build>
SOURCE_DATE_EPOCH=1700741054 docker buildx build --progress=plain --provenance=mode=max --sbom=generator="$BASHBREW_BUILDKIT_SBOM_GENERATOR" --output '"type=oci","dest=temp.tar","annotation.org.opencontainers.image.source=https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli","annotation-manifest-descriptor.org.opencontainers.image.source=https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli","annotation.org.opencontainers.image.revision=6d541d27b5dd12639e5a33a675ebca04d3837d74","annotation-manifest-descriptor.org.opencontainers.image.revision=6d541d27b5dd12639e5a33a675ebca04d3837d74","annotation.org.opencontainers.image.version=24.0.7-cli","annotation-manifest-descriptor.org.opencontainers.image.version=24.0.7-cli","annotation.org.opencontainers.image.url=https://hub.docker.com/_/docker","annotation-manifest-descriptor.org.opencontainers.image.url=https://hub.docker.com/_/docker"' --tag 'docker:24.0.7-cli' --tag 'docker:24.0-cli' --tag 'docker:24-cli' --tag 'docker:cli' --tag 'docker:24.0.7-cli-alpine3.18' --tag 'oisupport/staging-amd64:4b199ac326c74b3058a147e14f553af9e8e1659abc29bd3e82c9c9807b66ee43' --platform 'linux/amd64' --build-context 'alpine:3.18=docker-image://alpine:3.18@sha256:d695c3de6fcd8cfe3a6222b0358425d40adfd129a8a47c3416faff1a8aece389' --build-arg BUILDKIT_SYNTAX="$BASHBREW_BUILDKIT_SYNTAX" --file 'Dockerfile' 'https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli'
# </build>
# <push>
SOURCE_DATE_EPOCH=1700741054 \
docker buildx build --progress=plain \
--provenance=mode=max \
--sbom=generator="$BASHBREW_BUILDKIT_SBOM_GENERATOR" \
--output '"type=oci","dest=temp.tar"' \
--annotation 'org.opencontainers.image.source=https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli' \
--annotation 'org.opencontainers.image.revision=6d541d27b5dd12639e5a33a675ebca04d3837d74' \
--annotation 'org.opencontainers.image.created=2023-11-23T12:04:14Z' \
--annotation 'org.opencontainers.image.version=24.0.7-cli' \
--annotation 'org.opencontainers.image.url=https://hub.docker.com/_/docker' \
--annotation 'manifest-descriptor:org.opencontainers.image.source=https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli' \
--annotation 'manifest-descriptor:org.opencontainers.image.revision=6d541d27b5dd12639e5a33a675ebca04d3837d74' \
--annotation 'manifest-descriptor:org.opencontainers.image.created=1970-01-01T00:00:00Z' \
--annotation 'manifest-descriptor:org.opencontainers.image.version=24.0.7-cli' \
--annotation 'manifest-descriptor:org.opencontainers.image.url=https://hub.docker.com/_/docker' \
--tag 'docker:24.0.7-cli' \
--tag 'docker:24.0-cli' \
--tag 'docker:24-cli' \
--tag 'docker:cli' \
--tag 'docker:24.0.7-cli-alpine3.18' \
--tag 'oisupport/staging-amd64:4b199ac326c74b3058a147e14f553af9e8e1659abc29bd3e82c9c9807b66ee43' \
--platform 'linux/amd64' \
--build-context 'alpine:3.18=docker-image://alpine:3.18@sha256:d695c3de6fcd8cfe3a6222b0358425d40adfd129a8a47c3416faff1a8aece389' \
--build-arg BUILDKIT_SYNTAX="$BASHBREW_BUILDKIT_SYNTAX" \
--file 'Dockerfile' \
'https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli'
mkdir temp
tar -xvf temp.tar -C temp
jq '.manifests |= (del(.[].annotations) | unique)' temp/index.json > temp/index.json.new
rm temp.tar
jq '
.manifests |= (
del(.[].annotations)
| unique
| if length != 1 then
error("unexpected number of manifests: " + length)
else . end
)
' temp/index.json > temp/index.json.new
mv temp/index.json.new temp/index.json
# </build>
# <push>
crane push temp 'oisupport/staging-amd64:4b199ac326c74b3058a147e14f553af9e8e1659abc29bd3e82c9c9807b66ee43'
rm -rf temp temp.tar
rm -rf temp
# </push>

# docker:24.0.7-windowsservercore-ltsc2022 [windows-amd64]
Expand All @@ -20,7 +54,21 @@ docker pull 'mcr.microsoft.com/windows/servercore:ltsc2022@sha256:d4ab2dd7d3d0fc
docker tag 'mcr.microsoft.com/windows/servercore:ltsc2022@sha256:d4ab2dd7d3d0fce6edc5df459565a4c96bbb1d0148065b215ab5ddcab1e42eb4' 'mcr.microsoft.com/windows/servercore:ltsc2022'
# </pull>
# <build>
SOURCE_DATE_EPOCH=1700741054 DOCKER_BUILDKIT=0 docker build --tag 'docker:24.0.7-windowsservercore-ltsc2022' --tag 'docker:24.0-windowsservercore-ltsc2022' --tag 'docker:24-windowsservercore-ltsc2022' --tag 'docker:windowsservercore-ltsc2022' --tag 'docker:24.0.7-windowsservercore' --tag 'docker:24.0-windowsservercore' --tag 'docker:24-windowsservercore' --tag 'docker:windowsservercore' --tag 'oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1fee7582174de82ae861613ae3072e' --platform 'windows/amd64' --file 'Dockerfile' 'https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/windows/windowsservercore-ltsc2022'
SOURCE_DATE_EPOCH=1700741054 \
DOCKER_BUILDKIT=0 \
docker build \
--tag 'docker:24.0.7-windowsservercore-ltsc2022' \
--tag 'docker:24.0-windowsservercore-ltsc2022' \
--tag 'docker:24-windowsservercore-ltsc2022' \
--tag 'docker:windowsservercore-ltsc2022' \
--tag 'docker:24.0.7-windowsservercore' \
--tag 'docker:24.0-windowsservercore' \
--tag 'docker:24-windowsservercore' \
--tag 'docker:windowsservercore' \
--tag 'oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1fee7582174de82ae861613ae3072e' \
--platform 'windows/amd64' \
--file 'Dockerfile' \
'https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/windows/windowsservercore-ltsc2022'
# </build>
# <push>
docker push 'oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1fee7582174de82ae861613ae3072e'
Expand Down
2 changes: 1 addition & 1 deletion .test/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ time "$dir/../sources.sh" "$@" > "$dir/sources.json"
time "$dir/../builds.sh" --cache "$dir/cache-builds.json" "$dir/sources.json" > "$dir/builds.json"

# generate an "example commands" file so that changes to generated commands are easier to review
jq -r -L "$dir/.." '
SOURCE_DATE_EPOCH=0 jq -r -L "$dir/.." '
include "meta";
[
first(.[] | select(normalized_builder == "buildkit")),
Expand Down
185 changes: 125 additions & 60 deletions meta.jq
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,26 @@ def normalized_builder:
end
else . end
;
def docker_uses_containerd_storage:
# TODO somehow detect docker-with-containerd-storage
false
;
# input: "build" object (with "buildId" top level key)
# output: boolean
def should_use_docker_buildx_driver:
normalized_builder == "buildkit"
and (
.build.arch as $arch
# bashbrew remote arches --json tianon/buildkit:0.12 | jq '.arches | keys_unsorted' -c
| ["amd64","arm32v5","arm32v6","arm32v7","arm64v8","i386","mips64le","ppc64le","riscv64","s390x"]
# TODO this needs to be based on the *host* architecture, not the *target* architecture (amd64 vs i386)
| index($arch)
| not
docker_uses_containerd_storage
or (
.build.arch as $arch
# bashbrew remote arches --json tianon/buildkit:0.12 | jq '.arches | keys_unsorted' -c
| ["amd64","arm32v5","arm32v6","arm32v7","arm64v8","i386","mips64le","ppc64le","riscv64","s390x"]
# TODO this needs to be based on the *host* architecture, not the *target* architecture (amd64 vs i386)
| index($arch)
| not
# TODO "failed to read dockerfile: failed to load cache key: subdir not supported yet" asdflkjalksdjfklasdjfklajsdklfjasdklgfnlkasdfgbhnkljasdhgouiahsdoifjnask,.dfgnklasdbngoikasdhfoiasjdklfjasdlkfjalksdjfkladshjflikashdbgiohasdfgiohnaskldfjhnlkasdhfnklasdhglkahsdlfkjasdlkfjadsklfjsdl (hence "tianon/buildkit" instead of "moby/buildkit"; need *all* the arches we care about/support for consistent support)
)
)
# TODO "failed to read dockerfile: failed to load cache key: subdir not supported yet" asdflkjalksdjfklasdjfklajsdklfjasdklgfnlkasdfgbhnkljasdhgouiahsdoifjnask,.dfgnklasdbngoikasdhfoiasjdklfjasdlkfjalksdjfkladshjflikashdbgiohasdfgiohnaskldfjhnlkasdhfnklasdhglkahsdlfkjasdlkfjadsklfjsdl (hence "tianon/buildkit" instead of "moby/buildkit")
;
# input: "build" object (with "buildId" top level key)
# output: string "pull command" ("docker pull ..."), may be multiple lines, expects to run in Bash with "set -Eeuo pipefail", might be empty
Expand Down Expand Up @@ -71,66 +78,110 @@ def git_build_url:
) + "#" + .GitCommit + ":" + .Directory
;
# input: "build" object (with "buildId" top level key)
# output: map of annotations to set
def build_annotations($buildUrl):
{
# https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/annotations.md#pre-defined-annotation-keys
"org.opencontainers.image.source": $buildUrl,
"org.opencontainers.image.revision": .source.entry.GitCommit,
"org.opencontainers.image.created": (.source.entry.SOURCE_DATE_EPOCH | strftime("%FT%TZ")), # see notes below about image index vs image manifest

# TODO come up with less assuming values here? (Docker Hub assumption, tag ordering assumption)
"org.opencontainers.image.version": ( # value of the first image tag
first(.source.allTags[] | select(contains(":")))
| sub("^.*:"; "")
# TODO maybe we should do the first, longest, non-latest tag instead of just the first tag?
),
"org.opencontainers.image.url": ( # URL to Docker Hub
first(.source.allTags[] | select(contains(":")))
| sub(":.*$"; "")
| if contains("/") then
"r/" + .
else
"_/" + .
end
| "https://hub.docker.com/" + .
),

# TODO org.opencontainers.image.vendor ? (feels leaky to put "Docker Official Images" here when this is all otherwise mostly generic)
}
| with_entries(select(.value)) # strip off anything missing a value (possibly "source", "url", "version", etc)
;
def build_annotations:
build_annotations(git_build_url)
;
# input: multi-line string with indentation and comments
# output: multi-line string with less indentation and no comments
def unindent_and_decomment_jq($indents):
# trim out comment lines and unnecessary indentation
gsub("(?m)^(\t+#[^\n]*\n?|\t{\($indents)}(?<extra>.*)$)"; "\(.extra // "")")
# trim out empty lines
| gsub("\n\n+"; "\n")
;
# input: "build" object (with "buildId" top level key)
# output: string "build command" ("docker buildx build ..."), may be multiple lines, expects to run in Bash with "set -Eeuo pipefail"
def build_command:
normalized_builder as $builder
| git_build_url as $buildUrl
| if $builder == "buildkit" then
[
git_build_url as $buildUrl
| (
(should_use_docker_buildx_driver | not)
or docker_uses_containerd_storage
) as $supportsAnnotationsAndAttestsations
| [
(
[
@sh "SOURCE_DATE_EPOCH=\(.source.entry.SOURCE_DATE_EPOCH)",
# TODO EXPERIMENTAL_BUILDKIT_SOURCE_POLICY=<(jq ...)
"docker buildx build --progress=plain",
if should_use_docker_buildx_driver then "--load" else # TODO if we get containerd integration and thus use "--load" unconditionally again, we should update this to still set annotations! (and still gate SBOMs on appropriate scanner-supported architectures)
if $supportsAnnotationsAndAttestsations then
"--provenance=mode=max",
# see "bashbrew remote arches docker/scout-sbom-indexer:1" (we need the SBOM scanner to be runnable on the host architecture)
# bashbrew remote arches --json docker/scout-sbom-indexer:1 | jq '.arches | keys_unsorted' -c
if .build.arch as $arch | ["amd64","arm32v5","arm32v7","arm64v8","i386","ppc64le","riscv64","s390x"] | index($arch) then
# TODO this needs to be based on the *host* architecture, not the *target* architecture (amd64 vs i386)
"--sbom=generator=\"$BASHBREW_BUILDKIT_SBOM_GENERATOR\""
# TODO this should also be totally optional -- for example, Tianon doesn't want SBOMs on his personal images
else empty end,
(
"--output " + (
[
"type=oci", # TODO find a better way to build/tag with a full list of tags but only actually *push* to one of them so we don't have to round-trip through containerd
"dest=temp.tar", # TODO choose/find a good "safe" place to put this (temporarily)
(
{
# https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/annotations.md#pre-defined-annotation-keys
"org.opencontainers.image.source": $buildUrl,
"org.opencontainers.image.revision": .source.entry.GitCommit,

# TODO come up with less assuming values here? (Docker Hub assumption, tag ordering assumption)
"org.opencontainers.image.version": ( # value of the first image tag
first(.source.allTags[] | select(contains(":")))
| sub("^.*:"; "")
# TODO maybe we should do the first, longest, non-latest tag instead of just the first tag?
),
"org.opencontainers.image.url": ( # URL to Docker Hub
first(.source.allTags[] | select(contains(":")))
| sub(":.*$"; "")
| if contains("/") then
"r/" + .
else
"_/" + .
end
| "https://hub.docker.com/" + .
),
# TODO org.opencontainers.image.vendor ? (feels leaky to put "Docker Official Images" here when this is all otherwise mostly generic)
}
| to_entries[] | select(.value != null) |
"annotation." + .key + "=" + .value,
"annotation-manifest-descriptor." + .key + "=" + .value
),
empty
]
| @csv
| @sh
)
),
empty
end,
else empty end,
"--output " + (
[
if should_use_docker_buildx_driver then
"type=docker"
else
"type=oci",
"dest=temp.tar", # TODO choose/find a good "safe" place to put this (temporarily)
empty
end,
empty
]
| @csv
| @sh
),
(
if $supportsAnnotationsAndAttestsations then
build_annotations($buildUrl)
| to_entries
# separate loops so that "image manifest" annotations are grouped separate from the index/descriptor annotations (easier to read)
| (
.[]
| @sh "--annotation \(.key + "=" + .value)"
),
(
.[]
| @sh "--annotation \(
"manifest-descriptor:" + .key + "="
+ if .key == "org.opencontainers.image.created" then
# the "current" time breaks reproducibility (for the purposes of build verification), so we put "now" in the image index but "SOURCE_DATE_EPOCH" in the image manifest (which is the thing we'd ideally like to have reproducible, eventually)
(env.SOURCE_DATE_EPOCH // now) | tonumber | strftime("%FT%TZ")
# (this assumes the actual build is going to happen shortly after generating the command)
else .value end
)",
empty
)
else empty end
),
(
.source.arches[].tags[],
.source.arches[].archTags[],
Expand All @@ -148,8 +199,26 @@ def build_command:
@sh "--file \(.source.entry.File)",
($buildUrl | @sh),
empty
] | join(" ")
] | join(" \\\n\t")
),
if should_use_docker_buildx_driver then empty else
# munge the tarball into a suitable "oci layout" directory (ready for "crane push")
"mkdir temp",
"tar -xvf temp.tar -C temp",
"rm temp.tar",
# munge the index to what crane wants ("Error: layout contains 5 entries, consider --index")
@sh "jq \("
.manifests |= (
del(.[].annotations)
| unique
| if length != 1 then
error(\"unexpected number of manifests: \" + length)
else . end
)
" | unindent_and_decomment_jq(4)) temp/index.json > temp/index.json.new",
"mv temp/index.json.new temp/index.json",
empty
end,
# possible improvements in buildkit/buildx that could help us:
# - allowing OCI output directly to a directory instead of a tar (thus getting symmetry with the oci-layout:// inputs it can take)
# - allowing tag as one thing and push as something else, potentially mutually exclusive
Expand All @@ -158,7 +227,8 @@ def build_command:
empty
] | join("\n")
elif $builder == "classic" then
[
git_build_url as $buildUrl
| [
(
[
@sh "SOURCE_DATE_EPOCH=\(.source.entry.SOURCE_DATE_EPOCH)",
Expand All @@ -175,7 +245,7 @@ def build_command:
($buildUrl | @sh),
empty
]
| join(" ")
| join(" \\\n\t")
),
empty
] | join("\n")
Expand All @@ -199,14 +269,9 @@ def push_command:
@sh "docker push \(.build.img)"
elif $builder == "buildkit" then
[
# extract to a directory and "crane push" (easier to get correct than "ctr image import" + "ctr image push", especially with authentication)
"mkdir temp",
"tar -xvf temp.tar -C temp",
# munge the index to what crane wants ("Error: layout contains 5 entries, consider --index")
@sh "jq \(".manifests |= (del(.[].annotations) | unique)") temp/index.json > temp/index.json.new",
"mv temp/index.json.new temp/index.json",
# "crane push" is easier to get correct than "ctr image import" + "ctr image push", especially with authentication
@sh "crane push temp \(.build.img)",
"rm -rf temp temp.tar",
"rm -rf temp",
empty
] | join("\n")
elif $builder == "oci-import" then
Expand Down

0 comments on commit af7b858

Please sign in to comment.