diff --git a/.travis.yml b/.travis.yml index 120ce02f2..48a2dc655 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,8 @@ jobs: script: - go run build/ci.go lint - # These builders create the Docker sub-images for multi-arch push + # These builders create the Docker sub-images for multi-arch push and each + # will attempt to push the multi-arch image if they are the last builder - stage: build if: type = push os: linux @@ -38,7 +39,7 @@ jobs: git: submodules: false # avoid cloning ethereum/tests script: - - go run build/ci.go docker -image -upload karalabe/geth-docker-test + - go run build/ci.go docker -image -manifest amd64,arm64 -upload karalabe/geth-docker-test - stage: build if: type = push @@ -53,24 +54,7 @@ jobs: git: submodules: false # avoid cloning ethereum/tests script: - - go run build/ci.go docker -image -upload karalabe/geth-docker-test - - # This builder does the Docker Hub multi-arch image - - stage: publish - if: type = push - os: linux - dist: bionic - go: 1.16.x - env: - - docker - services: - - docker - git: - submodules: false # avoid cloning ethereum/tests - before_install: - - export DOCKER_CLI_EXPERIMENTAL=enabled - script: - - go run build/ci.go docker -manifest amd64,arm64 -upload karalabe/geth-docker-test + - go run build/ci.go docker -image -manifest amd64,arm64 -upload karalabe/geth-docker-test # This builder does the Ubuntu PPA upload - stage: build diff --git a/Dockerfile b/Dockerfile index 6e0dea11b..c0243adc3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,8 @@ +# Support setting various labels on the final image +ARG COMMIT="" +ARG VERSION="" +ARG BUILDNUM="" + # Build Geth in a stock Go builder container FROM golang:1.16-alpine as builder @@ -14,3 +19,10 @@ COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/ EXPOSE 8545 8546 30303 30303/udp ENTRYPOINT ["geth"] + +# Add some metadata labels to help programatic image consumption +ARG COMMIT="" +ARG VERSION="" +ARG BUILDNUM="" + +LABEL commit="$COMMIT" version="$VERSION" buildnum="$BUILDNUM" diff --git a/Dockerfile.alltools b/Dockerfile.alltools index 483afad8c..1a2689690 100644 --- a/Dockerfile.alltools +++ b/Dockerfile.alltools @@ -1,3 +1,8 @@ +# Support setting various labels on the final image +ARG COMMIT="" +ARG VERSION="" +ARG BUILDNUM="" + # Build Geth in a stock Go builder container FROM golang:1.16-alpine as builder @@ -13,3 +18,10 @@ RUN apk add --no-cache ca-certificates COPY --from=builder /go-ethereum/build/bin/* /usr/local/bin/ EXPOSE 8545 8546 30303 30303/udp + +# Add some metadata labels to help programatic image consumption +ARG COMMIT="" +ARG VERSION="" +ARG BUILDNUM="" + +LABEL commit="$COMMIT" version="$VERSION" buildnum="$BUILDNUM" diff --git a/build/ci.go b/build/ci.go index d9ff77984..63dd60632 100644 --- a/build/ci.go +++ b/build/ci.go @@ -54,6 +54,7 @@ import ( "path/filepath" "regexp" "runtime" + "strconv" "strings" "time" @@ -495,14 +496,45 @@ func doDocker(cmdline []string) { } // If architecture specific image builds are requested, build and push them if *image { - build.MustRunCommand("docker", "build", "--tag", fmt.Sprintf("%s:TAG", *upload), ".") - build.MustRunCommand("docker", "build", "--tag", fmt.Sprintf("%s:alltools-TAG", *upload), "-f", "Dockerfile.alltools", ".") + build.MustRunCommand("docker", "build", "--build-arg", "COMMIT="+env.Commit, "--build-arg", "VERSION="+params.VersionWithMeta, "--build-arg", "BUILDNUM="+env.Buildnum, "--tag", fmt.Sprintf("%s:TAG", *upload), ".") + build.MustRunCommand("docker", "build", "--build-arg", "COMMIT="+env.Commit, "--build-arg", "VERSION="+params.VersionWithMeta, "--build-arg", "BUILDNUM="+env.Buildnum, "--tag", fmt.Sprintf("%s:alltools-TAG", *upload), "-f", "Dockerfile.alltools", ".") // Tag and upload the images to Docker Hub for _, tag := range tags { gethImage := fmt.Sprintf("%s:%s-%s", *upload, tag, runtime.GOARCH) toolImage := fmt.Sprintf("%s:alltools-%s-%s", *upload, tag, runtime.GOARCH) + // If the image already exists (non version tag), check the build + // number to prevent overwriting a newer commit if concurrent builds + // are running. This is still a tiny bit racey if two published are + // done at the same time, but that's extremely unlikely even on the + // master branch. + for _, img := range []string{gethImage, toolImage} { + if exec.Command("docker", "pull", img).Run() != nil { + continue // Generally the only failure is a missing image, which is good + } + buildnum, err := exec.Command("docker", "inspect", "--format", "{{index .Config.Labels \"buildnum\"}}", img).CombinedOutput() + if err != nil { + log.Fatalf("Failed to inspect container: %v\nOutput: %s", err, string(buildnum)) + } + buildnum = bytes.TrimSpace(buildnum) + + if len(buildnum) > 0 && len(env.Buildnum) > 0 { + oldnum, err := strconv.Atoi(string(buildnum)) + if err != nil { + log.Fatalf("Failed to parse old image build number: %v", err) + } + newnum, err := strconv.Atoi(env.Buildnum) + if err != nil { + log.Fatalf("Failed to parse current build number: %v", err) + } + if oldnum > newnum { + log.Fatalf("Current build number %d not newer than existing %d", newnum, oldnum) + } else { + log.Printf("Updating %s from build %d to %d", img, oldnum, newnum) + } + } + } build.MustRunCommand("docker", "image", "tag", fmt.Sprintf("%s:TAG", *upload), gethImage) build.MustRunCommand("docker", "image", "tag", fmt.Sprintf("%s:alltools-TAG", *upload), toolImage) build.MustRunCommand("docker", "push", gethImage) @@ -511,6 +543,59 @@ func doDocker(cmdline []string) { } // If multi-arch image manifest push is requested, assemble it if len(*manifest) != 0 { + // Since different architectures are pushed by different builders, wait + // until all required images are updated. + var mismatch bool + for i := 0; i < 2; i++ { // 2 attempts, second is race check + mismatch = false // hope there's no mismatch now + + for _, tag := range tags { + for _, arch := range strings.Split(*manifest, ",") { + gethImage := fmt.Sprintf("%s:%s-%s", *upload, tag, arch) + toolImage := fmt.Sprintf("%s:alltools-%s-%s", *upload, tag, arch) + + for _, img := range []string{gethImage, toolImage} { + if out, err := exec.Command("docker", "pull", img).CombinedOutput(); err != nil { + log.Printf("Required image %s unavailable: %v\nOutput: %s", img, err, out) + mismatch = true + break + } + buildnum, err := exec.Command("docker", "inspect", "--format", "{{index .Config.Labels \"buildnum\"}}", img).CombinedOutput() + if err != nil { + log.Fatalf("Failed to inspect container: %v\nOutput: %s", err, string(buildnum)) + } + buildnum = bytes.TrimSpace(buildnum) + + if string(buildnum) != env.Buildnum { + log.Printf("Build number mismatch on %s: want %s, have %s", img, env.Buildnum, buildnum) + mismatch = true + break + } + } + if mismatch { + break + } + } + if mismatch { + break + } + } + if mismatch { + // Build numbers mismatching, retry in a short time to + // avoid concurrent failes in both publisher images. If + // however the retry failed too, it means the concurrent + // builder is still crunching, let that do the publish. + if i == 0 { + time.Sleep(30 * time.Second) + } + continue + } + break + } + if mismatch { + log.Println("Relinquishing publish to other builder") + return + } // Assemble and push the Geth manifest image for _, tag := range tags { gethImage := fmt.Sprintf("%s:%s", *upload, tag)