diff --git a/.travis.yml b/.travis.yml index ec63963ce..16c1e5174 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,6 @@ jobs: env: - GO111MODULE=on script: - - go run build/ci.go install - go run build/ci.go test -coverage $TEST_PACKAGES # These are the latest Go versions. @@ -43,7 +42,6 @@ jobs: env: - GO111MODULE=on script: - - go run build/ci.go install - go run build/ci.go test -coverage $TEST_PACKAGES - stage: build @@ -55,7 +53,6 @@ jobs: env: - GO111MODULE=on script: - - go run build/ci.go install - go run build/ci.go test -coverage $TEST_PACKAGES - stage: build @@ -74,7 +71,6 @@ jobs: - ulimit -S -n $NOFILE - ulimit -n - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703 - - go run build/ci.go install - go run build/ci.go test -coverage $TEST_PACKAGES # This builder does the Ubuntu PPA upload @@ -99,7 +95,7 @@ jobs: - python-paramiko script: - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts - - go run build/ci.go debsrc -goversion 1.15 -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " + - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " # This builder does the Linux Azure uploads - stage: build @@ -119,22 +115,22 @@ jobs: - gcc-multilib script: # Build for the primary platforms that Trusty can manage - - go run build/ci.go install + - go run build/ci.go install -dlgo - go run build/ci.go archive -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds - - go run build/ci.go install -arch 386 + - go run build/ci.go install -dlgo -arch 386 - go run build/ci.go archive -arch 386 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds # Switch over GCC to cross compilation (breaks 386, hence why do it here only) - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-arm-linux-gnueabihf libc6-dev-armhf-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross - sudo ln -s /usr/include/asm-generic /usr/include/asm - - GOARM=5 go run build/ci.go install -arch arm -cc arm-linux-gnueabi-gcc + - GOARM=5 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc - GOARM=5 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds - - GOARM=6 go run build/ci.go install -arch arm -cc arm-linux-gnueabi-gcc + - GOARM=6 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc - GOARM=6 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds - - GOARM=7 go run build/ci.go install -arch arm -cc arm-linux-gnueabihf-gcc + - GOARM=7 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabihf-gcc - GOARM=7 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds - - go run build/ci.go install -arch arm64 -cc aarch64-linux-gnu-gcc + - go run build/ci.go install -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds # This builder does the Linux Azure MIPS xgo uploads @@ -219,7 +215,7 @@ jobs: git: submodules: false # avoid cloning ethereum/tests script: - - go run build/ci.go install + - go run build/ci.go install -dlgo - go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -upload gethstore/builds # Build the iOS framework and upload it to CocoaPods and Azure diff --git a/appveyor.yml b/appveyor.yml index 7d6bf8763..eec726a65 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -30,7 +30,7 @@ install: - gcc --version build_script: - - go run build\ci.go install + - go run build\ci.go install -dlgo after_build: - go run build\ci.go archive -type zip -signer WINDOWS_SIGNING_KEY -upload gethstore/builds diff --git a/build/checksums.txt b/build/checksums.txt index 39f855cd0..d7a07d1ef 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -1,6 +1,13 @@ # This file contains sha256 checksums of optional build dependencies. -69438f7ed4f532154ffaf878f3dfd83747e7a00b70b3556eddabf7aaee28ac3a go1.15.src.tar.gz +063da6a9a4186b8118a0e584532c8c94e65582e2cd951ed078bfd595d27d2367 go1.15.4.src.tar.gz +aaf8c5323e0557211680960a8f51bedf98ab9a368775a687d6cf1f0079232b1d go1.15.4.darwin-amd64.tar.gz +6b2f6d8afddfb198bf0e36044084dc4db4cb0be1107375240b34d215aa5ff6ad go1.15.4.linux-386.tar.gz +eb61005f0b932c93b424a3a4eaa67d72196c79129d9a3ea8578047683e2c80d5 go1.15.4.linux-amd64.tar.gz +6f083b453484fc5f95afb345547a58ccc957cde91348b7a7c68f5b060e488c85 go1.15.4.linux-arm64.tar.gz +fe449ad3e121472e5db2f70becc0fef9d1a7188616c0605ada63f1e3bbad280e go1.15.4.linux-armv6l.tar.gz +3be3cfc08ccc7e7056fdee17b6f5d18e9d7f3d1351dcfec8de34b1c95cb05b50 go1.15.4.windows-386.zip +3593204e3851be577e4209900ece031b36f1e9ce1671f3f3221c9af7a090a941 go1.15.4.windows-amd64.zip d998a84eea42f2271aca792a7b027ca5c1edfcba229e8e5a844c9ac3f336df35 golangci-lint-1.27.0-linux-armv7.tar.gz bf781f05b0d393b4bf0a327d9e62926949a4f14d7774d950c4e009fc766ed1d4 golangci-lint.exe-1.27.0-windows-amd64.zip diff --git a/build/ci.go b/build/ci.go index 9522d29e4..4b6df88a4 100644 --- a/build/ci.go +++ b/build/ci.go @@ -46,12 +46,11 @@ import ( "encoding/base64" "flag" "fmt" - "go/parser" - "go/token" "io/ioutil" "log" "os" "os/exec" + "path" "path/filepath" "regexp" "runtime" @@ -148,6 +147,11 @@ var ( "golang-1.11": "/usr/lib/go-1.11", "golang-go": "/usr/lib/go", } + + // This is the version of go that will be downloaded by + // + // go run ci.go install -dlgo + dlgoVersion = "1.15.4" ) var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) @@ -198,19 +202,19 @@ func main() { func doInstall(cmdline []string) { var ( + dlgo = flag.Bool("dlgo", false, "Download Go and build with it") arch = flag.String("arch", "", "Architecture to cross build for") cc = flag.String("cc", "", "C compiler to cross build with") ) flag.CommandLine.Parse(cmdline) env := build.Env() - // Check Go version. People regularly open issues about compilation + // Check local Go version. People regularly open issues about compilation // failure with outdated Go. This should save them the trouble. if !strings.Contains(runtime.Version(), "devel") { // Figure out the minor version number since we can't textually compare (1.10 < 1.9) var minor int fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor) - if minor < 13 { log.Println("You have Go version", runtime.Version()) log.Println("go-ethereum requires at least Go version 1.13 and cannot") @@ -218,90 +222,108 @@ func doInstall(cmdline []string) { os.Exit(1) } } - // Compile packages given as arguments, or everything if there are no arguments. - packages := []string{"./..."} - if flag.NArg() > 0 { - packages = flag.Args() + + // Choose which go command we're going to use. + var gobuild *exec.Cmd + if !*dlgo { + // Default behavior: use the go version which runs ci.go right now. + gobuild = goTool("build") + } else { + // Download of Go requested. This is for build environments where the + // installed version is too old and cannot be upgraded easily. + cachedir := filepath.Join("build", "cache") + goroot := downloadGo(runtime.GOARCH, runtime.GOOS, cachedir) + gobuild = localGoTool(goroot, "build") } - if *arch == "" || *arch == runtime.GOARCH { - goinstall := goTool("install", buildFlags(env)...) - if runtime.GOARCH == "arm64" { - goinstall.Args = append(goinstall.Args, "-p", "1") - } - goinstall.Args = append(goinstall.Args, "-trimpath") - goinstall.Args = append(goinstall.Args, "-v") - goinstall.Args = append(goinstall.Args, packages...) - build.MustRun(goinstall) - return + // Configure environment for cross build. + if *arch != "" || *arch != runtime.GOARCH { + gobuild.Env = append(gobuild.Env, "CGO_ENABLED=1") + gobuild.Env = append(gobuild.Env, "GOARCH="+*arch) } - // Seems we are cross compiling, work around forbidden GOBIN - goinstall := goToolArch(*arch, *cc, "install", buildFlags(env)...) - goinstall.Args = append(goinstall.Args, "-trimpath") - goinstall.Args = append(goinstall.Args, "-v") - goinstall.Args = append(goinstall.Args, []string{"-buildmode", "archive"}...) - goinstall.Args = append(goinstall.Args, packages...) - build.MustRun(goinstall) + // Configure C compiler. + if *cc == "" { + gobuild.Env = append(gobuild.Env, "CC="+*cc) + } else if os.Getenv("CC") != "" { + gobuild.Env = append(gobuild.Env, "CC="+os.Getenv("CC")) + } - if cmds, err := ioutil.ReadDir("cmd"); err == nil { - for _, cmd := range cmds { - pkgs, err := parser.ParseDir(token.NewFileSet(), filepath.Join(".", "cmd", cmd.Name()), nil, parser.PackageClauseOnly) - if err != nil { - log.Fatal(err) - } - for name := range pkgs { - if name == "main" { - gobuild := goToolArch(*arch, *cc, "build", buildFlags(env)...) - gobuild.Args = append(gobuild.Args, "-v") - gobuild.Args = append(gobuild.Args, []string{"-o", executablePath(cmd.Name())}...) - gobuild.Args = append(gobuild.Args, "."+string(filepath.Separator)+filepath.Join("cmd", cmd.Name())) - build.MustRun(gobuild) - break - } - } - } + // arm64 CI builders are memory-constrained and can't handle concurrent builds, + // better disable it. This check isn't the best, it should probably + // check for something in env instead. + if runtime.GOARCH == "arm64" { + gobuild.Args = append(gobuild.Args, "-p", "1") + } + + // Put the default settings in. + gobuild.Args = append(gobuild.Args, buildFlags(env)...) + + // Show packages during build. + gobuild.Args = append(gobuild.Args, "-v") + + // Now we choose what we're even building. + // Default: collect all 'main' packages in cmd/ and build those. + packages := flag.Args() + if len(packages) == 0 { + packages = build.FindMainPackages("./cmd") + } + + // Do the build! + for _, pkg := range packages { + args := make([]string, len(gobuild.Args)) + copy(args, gobuild.Args) + args = append(args, "-o", executablePath(path.Base(pkg))) + args = append(args, pkg) + build.MustRun(&exec.Cmd{Path: gobuild.Path, Args: args, Env: gobuild.Env}) } } +// buildFlags returns the go tool flags for building. func buildFlags(env build.Environment) (flags []string) { var ld []string if env.Commit != "" { ld = append(ld, "-X", "main.gitCommit="+env.Commit) ld = append(ld, "-X", "main.gitDate="+env.Date) } + // Strip DWARF on darwin. This used to be required for certain things, + // and there is no downside to this, so we just keep doing it. if runtime.GOOS == "darwin" { ld = append(ld, "-s") } - if len(ld) > 0 { flags = append(flags, "-ldflags", strings.Join(ld, " ")) } + // We use -trimpath to avoid leaking local paths into the built executables. + flags = append(flags, "-trimpath") return flags } +// goTool returns the go tool. This uses the Go version which runs ci.go. func goTool(subcmd string, args ...string) *exec.Cmd { - return goToolArch(runtime.GOARCH, os.Getenv("CC"), subcmd, args...) + cmd := build.GoTool(subcmd, args...) + goToolSetEnv(cmd) + return cmd } -func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd { - cmd := build.GoTool(subcmd, args...) - if arch == "" || arch == runtime.GOARCH { - cmd.Env = append(cmd.Env, "GOBIN="+GOBIN) - } else { - cmd.Env = append(cmd.Env, "CGO_ENABLED=1") - cmd.Env = append(cmd.Env, "GOARCH="+arch) - } - if cc != "" { - cmd.Env = append(cmd.Env, "CC="+cc) - } +// localGoTool returns the go tool from the given GOROOT. +func localGoTool(goroot string, subcmd string, args ...string) *exec.Cmd { + gotool := filepath.Join(goroot, "bin", "go") + cmd := exec.Command(gotool, subcmd) + goToolSetEnv(cmd) + cmd.Env = append(cmd.Env, "GOROOT="+goroot) + cmd.Args = append(cmd.Args, args...) + return cmd +} + +// goToolSetEnv forwards the build environment to the go tool. +func goToolSetEnv(cmd *exec.Cmd) { for _, e := range os.Environ() { - if strings.HasPrefix(e, "GOBIN=") { + if strings.HasPrefix(e, "GOBIN=") || strings.HasPrefix(e, "CC=") { continue } cmd.Env = append(cmd.Env, e) } - return cmd } // Running The Tests @@ -363,7 +385,7 @@ func downloadLinter(cachedir string) string { if err := csdb.DownloadFile(url, archivePath); err != nil { log.Fatal(err) } - if err := build.ExtractTarballArchive(archivePath, cachedir); err != nil { + if err := build.ExtractArchive(archivePath, cachedir); err != nil { log.Fatal(err) } return filepath.Join(cachedir, base, "golangci-lint") @@ -469,13 +491,12 @@ func maybeSkipArchive(env build.Environment) { // Debian Packaging func doDebianSource(cmdline []string) { var ( - goversion = flag.String("goversion", "", `Go version to build with (will be included in the source package)`) - cachedir = flag.String("cachedir", "./build/cache", `Filesystem path to cache the downloaded Go bundles at`) - signer = flag.String("signer", "", `Signing key name, also used as package author`) - upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`) - sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) - workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) - now = time.Now() + cachedir = flag.String("cachedir", "./build/cache", `Filesystem path to cache the downloaded Go bundles at`) + signer = flag.String("signer", "", `Signing key name, also used as package author`) + upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`) + sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) + workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) + now = time.Now() ) flag.CommandLine.Parse(cmdline) *workdir = makeWorkdir(*workdir) @@ -490,7 +511,7 @@ func doDebianSource(cmdline []string) { } // Download and verify the Go source package. - gobundle := downloadGoSources(*goversion, *cachedir) + gobundle := downloadGoSources(*cachedir) // Download all the dependencies needed to build the sources and run the ci script srcdepfetch := goTool("install", "-n", "./...") @@ -509,7 +530,7 @@ func doDebianSource(cmdline []string) { pkgdir := stageDebianSource(*workdir, meta) // Add Go source code - if err := build.ExtractTarballArchive(gobundle, pkgdir); err != nil { + if err := build.ExtractArchive(gobundle, pkgdir); err != nil { log.Fatalf("Failed to extract Go sources: %v", err) } if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil { @@ -541,9 +562,10 @@ func doDebianSource(cmdline []string) { } } -func downloadGoSources(version string, cachedir string) string { +// downloadGoSources downloads the Go source tarball. +func downloadGoSources(cachedir string) string { csdb := build.MustLoadChecksums("build/checksums.txt") - file := fmt.Sprintf("go%s.src.tar.gz", version) + file := fmt.Sprintf("go%s.src.tar.gz", dlgoVersion) url := "https://dl.google.com/go/" + file dst := filepath.Join(cachedir, file) if err := csdb.DownloadFile(url, dst); err != nil { @@ -552,6 +574,41 @@ func downloadGoSources(version string, cachedir string) string { return dst } +// downloadGo downloads the Go binary distribution and unpacks it into a temporary +// directory. It returns the GOROOT of the unpacked toolchain. +func downloadGo(goarch, goos, cachedir string) string { + if goarch == "arm" { + goarch = "armv6l" + } + + csdb := build.MustLoadChecksums("build/checksums.txt") + file := fmt.Sprintf("go%s.%s-%s", dlgoVersion, goos, goarch) + if goos == "windows" { + file += ".zip" + } else { + file += ".tar.gz" + } + url := "https://golang.org/dl/" + file + dst := filepath.Join(cachedir, file) + if err := csdb.DownloadFile(url, dst); err != nil { + log.Fatal(err) + } + + ucache, err := os.UserCacheDir() + if err != nil { + log.Fatal(err) + } + godir := filepath.Join(ucache, fmt.Sprintf("geth-go-%s-%s-%s", dlgoVersion, goos, goarch)) + if err := build.ExtractArchive(dst, godir); err != nil { + log.Fatal(err) + } + goroot, err := filepath.Abs(filepath.Join(godir, "go")) + if err != nil { + log.Fatal(err) + } + return goroot +} + func ppaUpload(workdir, ppa, sshUser string, files []string) { p := strings.Split(ppa, "/") if len(p) != 2 { diff --git a/internal/build/archive.go b/internal/build/archive.go index a00258d99..8b3ac23d1 100644 --- a/internal/build/archive.go +++ b/internal/build/archive.go @@ -184,24 +184,35 @@ func (a *TarballArchive) Close() error { return a.file.Close() } -func ExtractTarballArchive(archive string, dest string) error { - // We're only interested in gzipped archives, wrap the reader now +// ExtractArchive unpacks a .zip or .tar.gz archive to the destination directory. +func ExtractArchive(archive string, dest string) error { ar, err := os.Open(archive) if err != nil { return err } defer ar.Close() + switch { + case strings.HasSuffix(archive, ".tar.gz"): + return extractTarball(ar, dest) + case strings.HasSuffix(archive, ".zip"): + return extractZip(ar, dest) + default: + return fmt.Errorf("unhandled archive type %s", archive) + } +} + +// extractTarball unpacks a .tar.gz file. +func extractTarball(ar io.Reader, dest string) error { gzr, err := gzip.NewReader(ar) if err != nil { return err } defer gzr.Close() - // Iterate over all the files in the tarball tr := tar.NewReader(gzr) for { - // Fetch the next tarball header and abort if needed + // Move to the next file header. header, err := tr.Next() if err != nil { if err == io.EOF { @@ -209,22 +220,69 @@ func ExtractTarballArchive(archive string, dest string) error { } return err } - // Figure out the target and create it - target := filepath.Join(dest, header.Name) - - switch header.Typeflag { - case tar.TypeReg: - if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { - return err - } - file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) + // We only care about regular files, directory modes + // and special file types are not supported. + if header.Typeflag == tar.TypeReg { + armode := header.FileInfo().Mode() + err := extractFile(header.Name, armode, tr, dest) if err != nil { - return err + return fmt.Errorf("extract %s: %v", header.Name, err) } - if _, err := io.Copy(file, tr); err != nil { - return err - } - file.Close() } } } + +// extractZip unpacks the given .zip file. +func extractZip(ar *os.File, dest string) error { + info, err := ar.Stat() + if err != nil { + return err + } + zr, err := zip.NewReader(ar, info.Size()) + if err != nil { + return err + } + + for _, zf := range zr.File { + if !zf.Mode().IsRegular() { + continue + } + + data, err := zf.Open() + if err != nil { + return err + } + err = extractFile(zf.Name, zf.Mode(), data, dest) + data.Close() + if err != nil { + return fmt.Errorf("extract %s: %v", zf.Name, err) + } + } + return nil +} + +// extractFile extracts a single file from an archive. +func extractFile(arpath string, armode os.FileMode, data io.Reader, dest string) error { + // Check that path is inside destination directory. + target := filepath.Join(dest, filepath.FromSlash(arpath)) + if !strings.HasPrefix(target, filepath.Clean(dest)+string(os.PathSeparator)) { + return fmt.Errorf("path %q escapes archive destination", target) + } + + // Ensure the destination directory exists. + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return err + } + + // Copy file data. + file, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, armode) + if err != nil { + return err + } + if _, err := io.Copy(file, data); err != nil { + file.Close() + os.Remove(target) + return err + } + return file.Close() +} diff --git a/internal/build/util.go b/internal/build/util.go index fc559760b..91149926f 100644 --- a/internal/build/util.go +++ b/internal/build/util.go @@ -20,6 +20,8 @@ import ( "bytes" "flag" "fmt" + "go/parser" + "go/token" "io" "io/ioutil" "log" @@ -152,3 +154,28 @@ func UploadSFTP(identityFile, host, dir string, files []string) error { stdin.Close() return sftp.Wait() } + +// FindMainPackages finds all 'main' packages in the given directory and returns their +// package paths. +func FindMainPackages(dir string) []string { + var commands []string + cmds, err := ioutil.ReadDir(dir) + if err != nil { + log.Fatal(err) + } + for _, cmd := range cmds { + pkgdir := filepath.Join(dir, cmd.Name()) + pkgs, err := parser.ParseDir(token.NewFileSet(), pkgdir, nil, parser.PackageClauseOnly) + if err != nil { + log.Fatal(err) + } + for name := range pkgs { + if name == "main" { + path := "./" + filepath.ToSlash(pkgdir) + commands = append(commands, path) + break + } + } + } + return commands +}