From 734e00af9e9a5487f57b00ae33cbb52dbe0c9565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 5 Nov 2019 15:32:42 +0200 Subject: [PATCH] travis, build, internal: use own Go bundle for PPA builds (#20240) * build: bump PPAs to Go 1.13 (via longsleep), keep Trusty on 1.11 * travis, build, vendor: use own Go bundle for PPA builds * travis, build, internal, vendor: smarter Go bundler, own untar * build: updated ci-notes with new Go bundling, only make, don't test --- .travis.yml | 5 ++- build/ci-notes.md | 17 ++++--- build/ci.go | 70 ++++++++++++++++++++--------- build/deb/ethereum/deb.control | 2 +- build/deb/ethereum/deb.rules | 4 +- internal/build/archive.go | 46 +++++++++++++++++++ internal/build/gosrc.go | 81 ++++++++++++++++++++++++++++++++++ 7 files changed, 193 insertions(+), 32 deletions(-) create mode 100644 internal/build/gosrc.go diff --git a/.travis.yml b/.travis.yml index 996ec4fdc..a09450500 100644 --- a/.travis.yml +++ b/.travis.yml @@ -75,9 +75,12 @@ jobs: - fakeroot - python-bzrlib - python-paramiko + cache: + directories: + - $HOME/.gobundle script: - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts - - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " + - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " -goversion 1.13.4 -gohash 95dbeab442ee2746b9acf0934c8e2fc26414a0565c008631b04addb8c02e7624 -gobundle $HOME/.gobundle/go.tar.gz # This builder does the Linux Azure uploads - stage: build diff --git a/build/ci-notes.md b/build/ci-notes.md index 13e1fd230..edd9adc1c 100644 --- a/build/ci-notes.md +++ b/build/ci-notes.md @@ -22,19 +22,18 @@ variables `PPA_SIGNING_KEY` and `PPA_SSH_KEY` on Travis. We want to build go-ethereum with the most recent version of Go, irrespective of the Go version that is available in the main Ubuntu repository. In order to make this possible, -our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on -golang-1.11, which is co-installable alongside the regular golang package. PPA dependencies -can be edited at https://launchpad.net/%7Eethereum/+archive/ubuntu/ethereum/+edit-dependencies +we bundle the entire Go sources into our own source archive and start the built job by +compiling Go and then using that to build go-ethereum. On Trusty we have a special case +requiring the `~gophers/ubuntu/archive` PPA since Trusty can't even build Go itself. PPA +deps are set at https://launchpad.net/%7Eethereum/+archive/ubuntu/ethereum/+edit-dependencies ## Building Packages Locally (for testing) You need to run Ubuntu to do test packaging. -Add the gophers PPA and install Go 1.11 and Debian packaging tools: +Install any version of Go and Debian packaging tools: - $ sudo apt-add-repository ppa:gophers/ubuntu/archive - $ sudo apt-get update - $ sudo apt-get install build-essential golang-1.11 devscripts debhelper python-bzrlib python-paramiko + $ sudo apt-get install build-essential golang-go devscripts debhelper python-bzrlib python-paramiko Create the source packages: @@ -42,10 +41,10 @@ Create the source packages: Then go into the source package directory for your running distribution and build the package: - $ cd dist/ethereum-unstable-1.6.0+xenial + $ cd dist/ethereum-unstable-1.9.6+bionic $ dpkg-buildpackage Built packages are placed in the dist/ directory. $ cd .. - $ dpkg-deb -c geth-unstable_1.6.0+xenial_amd64.deb + $ dpkg-deb -c geth-unstable_1.9.6+bionic_amd64.deb diff --git a/build/ci.go b/build/ci.go index 347cf95e6..ac5c72b6b 100644 --- a/build/ci.go +++ b/build/ci.go @@ -58,6 +58,7 @@ import ( "strings" "time" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/internal/build" "github.com/ethereum/go-ethereum/params" ) @@ -138,7 +139,18 @@ var ( // Note: zesty is unsupported because it was officially deprecated on Launchpad. // Note: artful is unsupported because it was officially deprecated on Launchpad. // Note: cosmic is unsupported because it was officially deprecated on Launchpad. - debDistros = []string{"trusty", "xenial", "bionic", "disco", "eoan"} + debDistroGoBoots = map[string]string{ + "trusty": "golang-1.11", + "xenial": "golang-go", + "bionic": "golang-go", + "disco": "golang-go", + "eoan": "golang-go", + } + + debGoBootPaths = map[string]string{ + "golang-1.11": "/usr/lib/go-1.11", + "golang-go": "/usr/lib/go", + } ) var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) @@ -459,11 +471,14 @@ func maybeSkipArchive(env build.Environment) { // Debian Packaging func doDebianSource(cmdline []string) { var ( - signer = flag.String("signer", "", `Signing key name, also used as package author`) - upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`) - sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) - workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) - now = time.Now() + goversion = flag.String("goversion", "", `Go version to build with (will be included in the source package)`) + gobundle = flag.String("gobundle", "/tmp/go.tar.gz", `Filesystem path to cache the downloaded Go bundles at`) + gohash = flag.String("gohash", "", `SHA256 checksum of the Go sources requested to build with`) + signer = flag.String("signer", "", `Signing key name, also used as package author`) + upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`) + sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) + workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) + now = time.Now() ) flag.CommandLine.Parse(cmdline) *workdir = makeWorkdir(*workdir) @@ -476,12 +491,25 @@ func doDebianSource(cmdline []string) { gpg.Stdin = bytes.NewReader(key) build.MustRun(gpg) } - + // Download and verify the Go source package + if err := build.EnsureGoSources(*goversion, hexutil.MustDecode("0x"+*gohash), *gobundle); err != nil { + log.Fatalf("Failed to ensure Go source package: %v", err) + } // Create Debian packages and upload them for _, pkg := range debPackages { - for _, distro := range debDistros { - meta := newDebMetadata(distro, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables) + for distro, goboot := range debDistroGoBoots { + // Prepare the debian package with the go-ethereum sources + meta := newDebMetadata(distro, goboot, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables) pkgdir := stageDebianSource(*workdir, meta) + + // Ship the Go sources along so we have a proper thing to build with + if err := build.ExtractTarballArchive(*gobundle, pkgdir); err != nil { + log.Fatalf("Failed to extract Go sources: %v", err) + } + if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil { + log.Fatalf("Failed to rename Go source folder: %v", err) + } + // Run the packaging and upload to the PPA debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz") debuild.Dir = pkgdir build.MustRun(debuild) @@ -561,7 +589,9 @@ type debPackage struct { } type debMetadata struct { - Env build.Environment + Env build.Environment + GoBootPackage string + GoBootPath string PackageName string @@ -590,19 +620,21 @@ func (d debExecutable) Package() string { return d.BinaryName } -func newDebMetadata(distro, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata { +func newDebMetadata(distro, goboot, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata { if author == "" { // No signing key, use default author. author = "Ethereum Builds " } return debMetadata{ - PackageName: name, - Env: env, - Author: author, - Distro: distro, - Version: version, - Time: t.Format(time.RFC1123Z), - Executables: exes, + GoBootPackage: goboot, + GoBootPath: debGoBootPaths[goboot], + PackageName: name, + Env: env, + Author: author, + Distro: distro, + Version: version, + Time: t.Format(time.RFC1123Z), + Executables: exes, } } @@ -667,7 +699,6 @@ func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) { if err := os.Mkdir(pkgdir, 0755); err != nil { log.Fatal(err) } - // Copy the source code. build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator)) @@ -685,7 +716,6 @@ func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) { build.Render("build/deb/"+meta.PackageName+"/deb.install", install, 0644, exe) build.Render("build/deb/"+meta.PackageName+"/deb.docs", docs, 0644, exe) } - return pkgdir } diff --git a/build/deb/ethereum/deb.control b/build/deb/ethereum/deb.control index 5b3ff9354..501a32cb4 100644 --- a/build/deb/ethereum/deb.control +++ b/build/deb/ethereum/deb.control @@ -2,7 +2,7 @@ Source: {{.Name}} Section: science Priority: extra Maintainer: {{.Author}} -Build-Depends: debhelper (>= 8.0.0), golang-1.11 +Build-Depends: debhelper (>= 8.0.0), {{.GoBootPackage}} Standards-Version: 3.9.5 Homepage: https://ethereum.org Vcs-Git: git://github.com/ethereum/go-ethereum.git diff --git a/build/deb/ethereum/deb.rules b/build/deb/ethereum/deb.rules index 5280e0e55..1370a52f1 100644 --- a/build/deb/ethereum/deb.rules +++ b/build/deb/ethereum/deb.rules @@ -6,9 +6,11 @@ # Launchpad rejects Go's access to $HOME/.cache, use custom folder export GOCACHE=/tmp/go-build +export GOROOT_BOOTSTRAP={{.GoBootPath}} override_dh_auto_build: - build/env.sh /usr/lib/go-1.11/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}} + (cd .go/src && ./make.bash) + build/env.sh .go/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}} override_dh_auto_test: diff --git a/internal/build/archive.go b/internal/build/archive.go index ac680ba63..8571edd5a 100644 --- a/internal/build/archive.go +++ b/internal/build/archive.go @@ -183,3 +183,49 @@ func (a *TarballArchive) Close() error { } return a.file.Close() } + +func ExtractTarballArchive(archive string, dest string) error { + // We're only interested in gzipped archives, wrap the reader now + ar, err := os.Open(archive) + if err != nil { + return err + } + defer ar.Close() + + gzr, err := gzip.NewReader(ar) + if err != nil { + return err + } + defer gzr.Close() + + // Iterate over all the files in the tarball + tr := tar.NewReader(gzr) + for { + // Fetch the next tarball header and abort if needed + header, err := tr.Next() + if err != nil { + if err == io.EOF { + return nil + } + return err + } + // Figure out the target and create it + target := filepath.Join(dest, header.Name) + + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, 0755); err != nil { + return err + } + case tar.TypeReg: + file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) + if err != nil { + return err + } + if _, err := io.Copy(file, tr); err != nil { + return err + } + file.Close() + } + } +} diff --git a/internal/build/gosrc.go b/internal/build/gosrc.go new file mode 100644 index 000000000..c85e46968 --- /dev/null +++ b/internal/build/gosrc.go @@ -0,0 +1,81 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package build + +import ( + "bytes" + "crypto/sha256" + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strings" +) + +// EnsureGoSources ensures that path contains a file with the given SHA256 hash, +// and if not, it downloads a fresh Go source package from upstream and replaces +// path with it (if the hash matches). +func EnsureGoSources(version string, hash []byte, path string) error { + // Sanity check the destination path to ensure we don't do weird things + if !strings.HasSuffix(path, ".tar.gz") { + return fmt.Errorf("destination path (%s) must end with .tar.gz", path) + } + // If the file exists, validate it's hash + if archive, err := ioutil.ReadFile(path); err == nil { // Go sources are ~20MB, it's fine to read all + hasher := sha256.New() + hasher.Write(archive) + have := hasher.Sum(nil) + + if bytes.Equal(have, hash) { + fmt.Printf("Go %s [%x] available at %s\n", version, hash, path) + return nil + } + fmt.Printf("Go %s hash mismatch (have %x, want %x) at %s, deleting old archive\n", version, have, hash, path) + if err := os.Remove(path); err != nil { + return err + } + } + // Archive missing or bad hash, download a new one + fmt.Printf("Downloading Go %s [want %x] into %s\n", version, hash, path) + + res, err := http.Get(fmt.Sprintf("https://dl.google.com/go/go%s.src.tar.gz", version)) + if err != nil || res.StatusCode != http.StatusOK { + return fmt.Errorf("failed to access Go sources: code %d, err %v", res.StatusCode, err) + } + defer res.Body.Close() + + archive, err := ioutil.ReadAll(res.Body) + if err != nil { + return err + } + // Sanity check the downloaded archive, save if checks out + hasher := sha256.New() + hasher.Write(archive) + + if have := hasher.Sum(nil); !bytes.Equal(have, hash) { + return fmt.Errorf("downloaded Go %s hash mismatch (have %x, want %x)", version, have, hash) + } + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return err + } + if err := ioutil.WriteFile(path, archive, 0644); err != nil { + return err + } + fmt.Printf("Downloaded Go %s [%x] into %s\n", version, hash, path) + return nil +}