travis, build: aggregate and upload go mod dependencies for PPA

.travis.yml

@@ -91,6 +91,10 @@ jobs:
             - python-bzrlib
             - python-paramiko
       script:
+        # Build once locally to download all the go modeule dependencies
+        - go run build/ci.go install
+
+        # Assemble the sources, dependencies and Go SDK into a deb source and upload to Launchpad
         - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
         - go run build/ci.go debsrc -goversion 1.13.4 -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
 

build/ci.go

@@ -46,6 +46,7 @@ import (
 	"encoding/base64"
 	"flag"
 	"fmt"
+	gobuild "go/build"
 	"go/parser"
 	"go/token"
 	"io/ioutil"
@@ -507,13 +508,15 @@ func doDebianSource(cmdline []string) {
 			meta := newDebMetadata(distro, goboot, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables)
 			pkgdir := stageDebianSource(*workdir, meta)
 
-			// Add Go source code.
+			// Add Go source code
 			if err := build.ExtractTarballArchive(gobundle, pkgdir); err != nil {
 				log.Fatalf("Failed to extract Go sources: %v", err)
 			}
 			if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil {
 				log.Fatalf("Failed to rename Go source folder: %v", err)
 			}
+			// Add all dependency modules in compressed form
+			build.CopyFolder(filepath.Join(pkgdir, ".mod", "cache", "download"), filepath.Join(gobuild.Default.GOPATH, "pkg", "mod", "cache", "download"), 0755)
 
 			// Run the packaging and upload to the PPA
 			debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz")

build/deb/ethereum/deb.rules

@@ -9,8 +9,17 @@ export GOCACHE=/tmp/go-build
 export GOROOT_BOOTSTRAP={{.GoBootPath}}
 
 override_dh_auto_build:
-	(cd .go/src && ./make.bash)
+	# We can't download a fresh Go within Launchpad, so we're shipping and building
-	build/env.sh .go/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
+	# one on the fly. However, we can't build it inside the go-ethereum folder as
+	# bootstrapping clashes with go modules, so build in a sibling folder.
+	(mv .go ../ && cd ../.go/src && ./make.bash)
+
+	# We can't download external go modules within Launchpad, so we're shipping the
+	# entire dependency source cache with go-ethereum.
+	(mkdir -p build/_workspace/pkg/mod && mv .mod/* build/_workspace/pkg/mod)
+
+	# A fresh Go was built, all dependency downloads faked, hope build works now
+	build/env.sh ../.go/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
 
 override_dh_auto_test:
 

internal/build/util.go

@@ -140,6 +140,29 @@ func CopyFile(dst, src string, mode os.FileMode) {
 	}
 }
 
+// CopyFolder copies a folder.
+func CopyFolder(dst, src string, mode os.FileMode) {
+	if err := os.MkdirAll(dst, mode); err != nil {
+		log.Fatal(err)
+	}
+	dir, _ := os.Open(src)
+
+	objects, err := dir.Readdir(-1)
+	if err != nil {
+		log.Fatal(err)
+	}
+	for _, obj := range objects {
+		srcPath := filepath.Join(src, obj.Name())
+		dstPath := filepath.Join(dst, obj.Name())
+
+		if obj.IsDir() {
+			CopyFolder(dstPath, srcPath, mode)
+		} else {
+			CopyFile(dstPath, srcPath, mode)
+		}
+	}
+}
+
 // GoTool returns the command that runs a go tool. This uses go from GOROOT instead of PATH
 // so that go commands executed by build use the same version of Go as the 'host' that runs
 // build code. e.g.