Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

build: provide a flag to disable publishing in dockerx build #2259

Merged
merged 1 commit into from
Feb 3, 2025
Merged

build: provide a flag to disable publishing in dockerx build #2259

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
156 changes: 30 additions & 126 deletions build/ci.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,6 @@ import (
"regexp"
"runtime"
"slices"
"strconv"
"strings"
"time"

Expand Down Expand Up @@ -519,9 +518,9 @@ func maybeSkipArchive(env build.Environment) {
// Builds the docker images and optionally uploads them to Docker Hub.
func doDocker(cmdline []string) {
var (
image = flag.Bool("image", false, `Whether to build and push an arch specific docker image`)
manifest = flag.String("manifest", "", `Push a multi-arch docker image for the specified architectures (usually "amd64,arm64")`)
upload = flag.String("upload", "", `Where to upload the docker image (usually "cortex/client-go")`)
platform = flag.String("platform", "", `Push a multi-arch docker image for the specified architectures (usually "linux/amd64,linux/arm64")`)
hubImage = flag.String("hub", "ethereum/client-go", `Where to upload the docker image`)
upload = flag.Bool("upload", false, `Whether to trigger upload`)
)
flag.CommandLine.Parse(cmdline)

Expand Down Expand Up @@ -555,129 +554,34 @@ func doDocker(cmdline []string) {
case strings.HasPrefix(env.Tag, "v1."):
tags = []string{"stable", fmt.Sprintf("release-%v", version.Family), "v" + version.Semantic}
}
// If architecture specific image builds are requested, build and push them
if *image {
build.MustRunCommand("docker", "build", "--build-arg", "COMMIT="+env.Commit, "--build-arg", "VERSION="+version.WithMeta, "--build-arg", "BUILDNUM="+env.Buildnum, "--tag", fmt.Sprintf("%s:TAG", *upload), ".")
build.MustRunCommand("docker", "build", "--build-arg", "COMMIT="+env.Commit, "--build-arg", "VERSION="+version.WithMeta, "--build-arg", "BUILDNUM="+env.Buildnum, "--tag", fmt.Sprintf("%s:alltools-TAG", *upload), "-f", "Dockerfile.alltools", ".")

// Tag and upload the images to Docker Hub
for _, tag := range tags {
cortexImage := fmt.Sprintf("%s:%s-%s", *upload, tag, runtime.GOARCH)
toolImage := fmt.Sprintf("%s:alltools-%s-%s", *upload, tag, runtime.GOARCH)

// If the image already exists (non version tag), check the build
// number to prevent overwriting a newer commit if concurrent builds
// are running. This is still a tiny bit racey if two published are
// done at the same time, but that's extremely unlikely even on the
// master branch.
for _, img := range []string{cortexImage, toolImage} {
if exec.Command("docker", "pull", img).Run() != nil {
continue // Generally the only failure is a missing image, which is good
}
buildnum, err := exec.Command("docker", "inspect", "--format", "{{index .Config.Labels \"buildnum\"}}", img).CombinedOutput()
if err != nil {
log.Fatalf("Failed to inspect container: %v\nOutput: %s", err, string(buildnum))
}
buildnum = bytes.TrimSpace(buildnum)

if len(buildnum) > 0 && len(env.Buildnum) > 0 {
oldnum, err := strconv.Atoi(string(buildnum))
if err != nil {
log.Fatalf("Failed to parse old image build number: %v", err)
}
newnum, err := strconv.Atoi(env.Buildnum)
if err != nil {
log.Fatalf("Failed to parse current build number: %v", err)
}
if oldnum > newnum {
log.Fatalf("Current build number %d not newer than existing %d", newnum, oldnum)
} else {
log.Printf("Updating %s from build %d to %d", img, oldnum, newnum)
}
}
}
build.MustRunCommand("docker", "image", "tag", fmt.Sprintf("%s:TAG", *upload), cortexImage)
build.MustRunCommand("docker", "image", "tag", fmt.Sprintf("%s:alltools-TAG", *upload), toolImage)
build.MustRunCommand("docker", "push", cortexImage)
build.MustRunCommand("docker", "push", toolImage)
}
}
// If multi-arch image manifest push is requested, assemble it
if len(*manifest) != 0 {
// Since different architectures are pushed by different builders, wait
// until all required images are updated.
var mismatch bool
for i := 0; i < 2; i++ { // 2 attempts, second is race check
mismatch = false // hope there's no mismatch now

for _, tag := range tags {
for _, arch := range strings.Split(*manifest, ",") {
cortexImage := fmt.Sprintf("%s:%s-%s", *upload, tag, arch)
toolImage := fmt.Sprintf("%s:alltools-%s-%s", *upload, tag, arch)

for _, img := range []string{cortexImage, toolImage} {
if out, err := exec.Command("docker", "pull", img).CombinedOutput(); err != nil {
log.Printf("Required image %s unavailable: %v\nOutput: %s", img, err, out)
mismatch = true
break
}
buildnum, err := exec.Command("docker", "inspect", "--format", "{{index .Config.Labels \"buildnum\"}}", img).CombinedOutput()
if err != nil {
log.Fatalf("Failed to inspect container: %v\nOutput: %s", err, string(buildnum))
}
buildnum = bytes.TrimSpace(buildnum)

if string(buildnum) != env.Buildnum {
log.Printf("Build number mismatch on %s: want %s, have %s", img, env.Buildnum, buildnum)
mismatch = true
break
}
}
if mismatch {
break
}
}
if mismatch {
break
}
}
if mismatch {
// Build numbers mismatching, retry in a short time to
// avoid concurrent failes in both publisher images. If
// however the retry failed too, it means the concurrent
// builder is still crunching, let that do the publish.
if i == 0 {
time.Sleep(30 * time.Second)
}
continue
}
break
}
if mismatch {
log.Println("Relinquishing publish to other builder")
return
}
// Assemble and push the Geth manifest image
for _, tag := range tags {
cortexImage := fmt.Sprintf("%s:%s", *upload, tag)

var cortexSubImages []string
for _, arch := range strings.Split(*manifest, ",") {
cortexSubImages = append(cortexSubImages, cortexImage+"-"+arch)
}
build.MustRunCommand("docker", append([]string{"manifest", "create", cortexImage}, cortexSubImages...)...)
build.MustRunCommand("docker", "manifest", "push", cortexImage)
}
// Assemble and push the alltools manifest image
for _, tag := range tags {
toolImage := fmt.Sprintf("%s:alltools-%s", *upload, tag)

var toolSubImages []string
for _, arch := range strings.Split(*manifest, ",") {
toolSubImages = append(toolSubImages, toolImage+"-"+arch)
// Need to create a mult-arch builder
check := exec.Command("docker", "buildx", "inspect", "multi-arch-builder")
if check.Run() != nil {
build.MustRunCommand("docker", "buildx", "create", "--use", "--name", "multi-arch-builder", "--platform", *platform)
}

for _, spec := range []struct {
file string
base string
}{
{file: "Dockerfile", base: fmt.Sprintf("%s:", *hubImage)},
{file: "Dockerfile.alltools", base: fmt.Sprintf("%s:alltools-", *hubImage)},
} {
for _, tag := range tags { // latest, stable etc
gethImage := fmt.Sprintf("%s%s", spec.base, tag)
cmd := exec.Command("docker", "buildx", "build",
"--build-arg", "COMMIT="+env.Commit,
"--build-arg", "VERSION="+version.WithMeta,
"--build-arg", "BUILDNUM="+env.Buildnum,
"--tag", gethImage,
"--platform", *platform,
"--file", spec.file,
)
if *upload {
cmd.Args = append(cmd.Args, "--push")
}
build.MustRunCommand("docker", append([]string{"manifest", "create", toolImage}, toolSubImages...)...)
build.MustRunCommand("docker", "manifest", "push", toolImage)
cmd.Args = append(cmd.Args, ".")
build.MustRun(cmd)
}
}
}
Expand Down