From 606956b668c50e359adaadf6ee806e223609911d Mon Sep 17 00:00:00 2001 From: Alessio Treglia Date: Sun, 13 Sep 2020 20:55:35 +0100 Subject: [PATCH] Reproducible buildsystem (#7247) Provide a simple Docker-based mechanism for application developers to provide reproducible builds. Unlike gaia's current reproducible buildsystem, this does not depend on external tools, e.g. `gitian-builder`. `build-simd-linux` now builds `simd` in a deterministic Linux container. --- Makefile | 33 ++++++++++-- build.sh | 39 ++++++++++++++ contrib/images/Makefile | 5 +- contrib/images/rbuilder/Dockerfile | 19 +++++++ contrib/images/rbuilder/buildlib.sh | 80 +++++++++++++++++++++++++++++ 5 files changed, 171 insertions(+), 5 deletions(-) create mode 100755 build.sh create mode 100644 contrib/images/rbuilder/Dockerfile create mode 100644 contrib/images/rbuilder/buildlib.sh diff --git a/Makefile b/Makefile index 03dbff1e84..9df0b7647a 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PACKAGES_NOSIMULATION=$(shell go list ./... | grep -v '/simulation') PACKAGES_SIMTEST=$(shell go list ./... | grep '/simulation') -VERSION := $(shell echo $(shell git describe) | sed 's/^v//') +VERSION := $(shell echo $(shell git describe --always) | sed 's/^v//') COMMIT := $(shell git log -1 --format='%H') LEDGER_ENABLED ?= true BINDIR ?= $(GOPATH)/bin @@ -86,12 +86,35 @@ include contrib/devtools/Makefile build: go.sum go install -mod=readonly ./... -build-simd: go.sum +simd: mkdir -p $(BUILDDIR) go build -mod=readonly $(BUILD_FLAGS) -o $(BUILDDIR) ./simapp/simd +build-simd-all: go.sum + $(if $(shell docker inspect -f '{{ .Id }}' cosmossdk/rbuilder 2>/dev/null),$(info found image cosmossdk/rbuilder),docker pull cosmossdk/rbuilder:latest) + docker rm latest-build || true + docker run --volume=$(CURDIR):/sources:ro \ + --env TARGET_OS='darwin linux windows' \ + --env APP=simd \ + --env VERSION=$(VERSION) \ + --env COMMIT=$(COMMIT) \ + --env LEDGER_ENABLED=$(LEDGER_ENABLED) \ + --name latest-build cosmossdk/rbuilder:latest + docker cp -a latest-build:/home/builder/artifacts/ $(CURDIR)/ + build-simd-linux: go.sum - LEDGER_ENABLED=false GOOS=linux GOARCH=amd64 $(MAKE) build-simd + $(if $(shell docker inspect -f '{{ .Id }}' cosmossdk/rbuilder 2>/dev/null),$(info found image cosmossdk/rbuilder),docker pull cosmossdk/rbuilder:latest) + docker rm latest-build || true + docker run --volume=$(CURDIR):/sources:ro \ + --env TARGET_OS='linux' \ + --env APP=simd \ + --env VERSION=$(VERSION) \ + --env COMMIT=$(COMMIT) \ + --env LEDGER_ENABLED=false \ + --name latest-build cosmossdk/rbuilder:latest + docker cp -a latest-build:/home/builder/artifacts/ $(CURDIR)/ + mkdir -p $(BUILDDIR) + cp artifacts/simd-*-linux-amd64 $(BUILDDIR)/simd cosmovisor: $(MAKE) -C cosmovisor cosmovisor @@ -119,7 +142,9 @@ distclean: clean .gitian-builder-cache/ clean: - rm -rf $(BUILDDIR)/ + rm -rf \ + $(BUILDDIR)/ \ + artifacts/ .PHONY: distclean clean diff --git a/build.sh b/build.sh new file mode 100755 index 0000000000..e6dbb8513b --- /dev/null +++ b/build.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -ue + +# Expect the following envvars to be set: +# - APP +# - VERSION +# - COMMIT +# - TARGET_OS +# - LEDGER_ENABLED +# - DEBUG + +# Source builder's functions library +. /usr/local/share/cosmos-sdk/buildlib.sh + +# These variables are now available +# - BASEDIR +# - OUTDIR + +# Build for each os-architecture pair +for os in ${TARGET_OS} ; do + archs="`f_build_archs ${os}`" + exe_file_extension="`f_binary_file_ext ${os}`" + for arch in ${archs} ; do + make clean + GOOS="${os}" GOARCH="${arch}" GOROOT_FINAL="$(go env GOROOT)" \ + make ${APP} \ + LDFLAGS=-buildid=${VERSION} \ + VERSION=${VERSION} \ + COMMIT=${COMMIT} \ + LEDGER_ENABLED=${LEDGER_ENABLED} + mv ./build/${APP}${exe_file_extension} ${OUTDIR}/${APP}-${VERSION}-${os}-${arch}${exe_file_extension} + done + unset exe_file_extension +done + +# Generate and display build report +f_generate_build_report ${OUTDIR} +cat ${OUTDIR}/build_report diff --git a/contrib/images/Makefile b/contrib/images/Makefile index c0ec5240fd..8ab47103ad 100644 --- a/contrib/images/Makefile +++ b/contrib/images/Makefile @@ -3,4 +3,7 @@ all: simd-env simd-env: docker build --build-arg UID=$(shell id -u) --build-arg GID=$(shell id -g) --tag cosmossdk/simd-env simd-env -.PHONY: all simd-env +rbuilder: + docker build --tag cosmossdk/rbuilder rbuilder + +.PHONY: all simd-env rbuilder diff --git a/contrib/images/rbuilder/Dockerfile b/contrib/images/rbuilder/Dockerfile new file mode 100644 index 0000000000..03059bcef8 --- /dev/null +++ b/contrib/images/rbuilder/Dockerfile @@ -0,0 +1,19 @@ +FROM golang:1.15.0-buster +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get --no-install-recommends -y install \ + pciutils build-essential git wget \ + lsb-release dpkg-dev curl bsdmainutils fakeroot +RUN mkdir -p /usr/local/share/cosmos-sdk/ +COPY buildlib.sh /usr/local/share/cosmos-sdk/ +RUN useradd -ms /bin/bash -U builder +ARG APP +ARG DEBUG +ENV APP ${APP:-cosmos-sdk} +ENV DEBUG ${DEBUG} +ENV VERSION unknown +ENV COMMIT unknown +ENV LEDGER_ENABLE true +USER builder:builder +WORKDIR /sources +VOLUME [ "/sources" ] +ENTRYPOINT [ "/sources/build.sh" ] diff --git a/contrib/images/rbuilder/buildlib.sh b/contrib/images/rbuilder/buildlib.sh new file mode 100644 index 0000000000..5302e94e53 --- /dev/null +++ b/contrib/images/rbuilder/buildlib.sh @@ -0,0 +1,80 @@ +#/bin/bash + +f_make_release_tarball() { + SOURCEDIST=${BASEDIR}/${APP}-${VERSION}.tar.gz + + git archive --format tar.gz --prefix "${APP}-${VERSION}/" -o "${SOURCEDIST}" HEAD + + l_tempdir="$(mktemp -d)" + pushd "${l_tempdir}" >/dev/null + tar xf "${SOURCEDIST}" + rm "${SOURCEDIST}" + find ${APP}-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > "${SOURCEDIST}" + popd >/dev/null + rm -rf "${l_tempdir}" +} + +f_setup_pristine_src_dir() { + cd ${pristinesrcdir} + tar --strip-components=1 -xf "${SOURCEDIST}" + go mod download +} + +f_build_archs() { + local l_os + + l_os=$1 + + case "${l_os}" in + darwin | windows) + echo 'amd64' + ;; + linux) + echo 'amd64 arm64' + ;; + *) + echo "unknown OS -- ${l_os}" >&2 + return 1 + esac +} + +f_binary_file_ext() { + [ $1 = windows ] && printf '%s' '.exe' || printf '' +} + +f_generate_build_report() { + local l_tempfile + + l_tempfile="$(mktemp)" + + pushd "${OUTDIR}" >/dev/null + cat >>"${l_tempfile}" <> "${l_tempfile}" + md5sum * | sed 's/^/ /' >> "${l_tempfile}" + echo 'Checksums-Sha256:' >> "${l_tempfile}" + sha256sum * | sed 's/^/ /' >> "${l_tempfile}" + mv "${l_tempfile}" build_report + popd >/dev/null +} + +[ "x${DEBUG}" = "x" ] || set -x + +BASEDIR="$(mktemp -d)" +OUTDIR=$HOME/artifacts +rm -rfv ${OUTDIR}/ +mkdir -p ${OUTDIR}/ +pristinesrcdir=${BASEDIR}/buildsources +mkdir -p ${pristinesrcdir} + +# Make release tarball +f_make_release_tarball + +# Extract release tarball and cache dependencies +f_setup_pristine_src_dir + +# Move the release tarball to the out directory +mv ${SOURCEDIST} ${OUTDIR}/