travis, build: implement uploading archives to azure

2016-11-02 18:21:05 +02:00 · 2016-11-02 18:21:05 +02:00 · 3b62c145f8
commit 3b62c145f8
parent 1b73c79234
4 changed files with 180 additions and 13 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -6,6 +6,8 @@ matrix:
    - os: linux
      dist: trusty
      go: 1.5.4
+      env:
+        - GO15VENDOREXPERIMENT=1
    - os: linux
      dist: trusty
      go: 1.6.2
@ -15,11 +17,13 @@ matrix:
    - os: osx
      go: 1.7

-    # This builder does the PPA upload (and nothing else).
+    # This builder does the Ubuntu PPA and Linux Azure uploads
    - os: linux
      dist: trusty
      go: 1.7
-      env: PPA
+      env:
+        - ubuntu-ppa
+        - azure-linux-amd64
      addons:
        apt:
          packages:
@ -28,14 +32,23 @@ matrix:
            - dput
      script:
        - go run build/ci.go debsrc -signer "Felix Lange (Geth CI Testing Key) <fjl@twurst.com>" -upload ppa:lp-fjl/geth-ci-testing
+        - go run build/ci.go install
+        - go run build/ci.go archive -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+
+    # This builder does the OSX Azure uploads
+    - os: osx
+      go: 1.7
+      env:
+        - azure-osx-amd64
+      script:
+        - go run build/ci.go install
+        - go run build/ci.go archive -type zip -signer OSX_SIGNING_KEY -upload gethstore/builds

 install:
  - go get golang.org/x/tools/cmd/cover
 script:
  - go run build/ci.go install
  - go run build/ci.go test -coverage -vet
-after_success:
-  # - go run build/ci.go archive -type tar

 notifications:
  webhooks:
--- a/build/ci.go
+++ b/build/ci.go
@ -23,12 +23,12 @@ Usage: go run ci.go <command> <command flags/arguments>

 Available commands are:

-   install    [ packages... ]                          -- builds packages and executables
-   test       [ -coverage ] [ -vet ] [ packages... ]   -- runs the tests
-   archive    [ -type zip|tar ]                        -- archives build artefacts
-   importkeys                                          -- imports signing keys from env
-   debsrc     [ -sign key-id ] [ -upload dest ]        -- creates a debian source package
-   xgo        [ options ]                              -- cross builds according to options
+   install    [ packages... ]                                           -- builds packages and executables
+   test       [ -coverage ] [ -vet ] [ packages... ]                    -- runs the tests
+   archive    [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artefacts
+   importkeys                                                           -- imports signing keys from env
+   debsrc     [ -signer key-id ] [ -upload dest ]                       -- creates a debian source package
+   xgo        [ options ]                                               -- cross builds according to options

 For all commands, -n prevents execution of external programs (dry run mode).

@ -49,7 +49,7 @@ import (
 	"strings"
 	"time"

-	"../internal/build"
+	"github.com/ethereum/go-ethereum/internal/build"
 )

 var (
@ -239,8 +239,10 @@ func doTest(cmdline []string) {

 func doArchive(cmdline []string) {
 	var (
-		atype = flag.String("type", "zip", "Type of archive to write (zip|tar)")
-		ext   string
+		atype  = flag.String("type", "zip", "Type of archive to write (zip|tar)")
+		signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. LINUX_SIGNING_KEY)`)
+		upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`)
+		ext    string
 	)
 	flag.CommandLine.Parse(cmdline)
 	switch *atype {
@ -262,6 +264,12 @@ func doArchive(cmdline []string) {
 	if err := build.WriteArchive("geth-alltools-"+base, ext, allToolsArchiveFiles); err != nil {
 		log.Fatal(err)
 	}
+
+	for _, archive := range []string{"geth-" + base + ext, "geth-alltools-" + base + ext} {
+		if err := archiveUpload(archive, *upload, *signer); err != nil {
+			log.Fatal(err)
+		}
+	}
 }

 func archiveBasename(env build.Environment) string {
@ -274,6 +282,36 @@ func archiveBasename(env build.Environment) string {
 	return archive
 }

+func archiveUpload(archive string, blobstore string, signer string) error {
+	// If signing was requested, generate the signature files
+	if signer != "" {
+		pgpkey, err := base64.StdEncoding.DecodeString(os.Getenv(signer))
+		if err != nil {
+			return fmt.Errorf("invalid base64 %s", signer)
+		}
+		if err := build.PGPSignFile(archive, archive+".asc", string(pgpkey)); err != nil {
+			return err
+		}
+	}
+	// If uploading to Azure was requested, push the archive possibly with its signature
+	if blobstore != "" {
+		auth := build.AzureBlobstoreConfig{
+			Account:   strings.Split(blobstore, "/")[0],
+			Token:     os.Getenv("AZURE_BLOBSTORE_TOKEN"),
+			Container: strings.SplitN(blobstore, "/", 2)[1],
+		}
+		if err := build.AzureBlobstoreUpload(archive, archive, auth); err != nil {
+			return err
+		}
+		if signer != "" {
+			if err := build.AzureBlobstoreUpload(archive+".asc", archive+".asc", auth); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 // skips archiving for some build configurations.
 func maybeSkipArchive(env build.Environment) {
 	if env.IsPullRequest {
--- a/internal/build/azure.go
+++ b/internal/build/azure.go
@ -0,0 +1,58 @@
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package build
+
+import (
+	"os"
+
+	"github.com/Azure/azure-sdk-for-go/storage"
+)
+
+// AzureBlobstoreConfig is an authentication and configuration struct containing
+// the data needed by the Azure SDK to interact with a speicifc container in the
+// blobstore.
+type AzureBlobstoreConfig struct {
+	Account   string // Account name to authorize API requests with
+	Token     string // Access token for the above account
+	Container string // Blob container to upload files into
+}
+
+// AzureBlobstoreUpload uploads a local file to the Azure Blob Storage. Note, this
+// method assumes a max file size of 64MB (Azure limitation). Larger files will
+// need a multi API call approach implemented.
+//
+// See: https://msdn.microsoft.com/en-us/library/azure/dd179451.aspx#Anchor_3
+func AzureBlobstoreUpload(path string, name string, config AzureBlobstoreConfig) error {
+	// Create an authenticated client against the Azure cloud
+	rawClient, err := storage.NewBasicClient(config.Account, config.Token)
+	if err != nil {
+		return err
+	}
+	client := rawClient.GetBlobService()
+
+	// Stream the file to upload into the designated blobstore container
+	in, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	info, err := in.Stat()
+	if err != nil {
+		return err
+	}
+	return client.CreateBlockBlobFromReader(config.Container, name, uint64(info.Size()), in, nil)
+}
--- a/internal/build/pgp.go
+++ b/internal/build/pgp.go
@ -0,0 +1,58 @@
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// signFile reads the contents of an input file and signs it (in armored format)
+// with the key provided, placing the signature into the output file.
+
+package build
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+
+	"golang.org/x/crypto/openpgp"
+)
+
+// PGPSignFile parses a PGP private key from the specified string and creates a
+// signature file into the output parameter of the input file.
+//
+// Note, this method assumes a single key will be container in the pgpkey arg,
+// furthermore that it is in armored format.
+func PGPSignFile(input string, output string, pgpkey string) error {
+	// Parse the keyring and make sure we only have a single private key in it
+	keys, err := openpgp.ReadArmoredKeyRing(bytes.NewBufferString(pgpkey))
+	if err != nil {
+		return err
+	}
+	if len(keys) != 1 {
+		return fmt.Errorf("key count mismatch: have %d, want %d", len(keys), 1)
+	}
+	// Create the input and output streams for signing
+	in, err := os.Open(input)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	out, err := os.Create(output)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	// Generate the signature and return
+	return openpgp.ArmoredDetachSignText(out, keys[0], in, nil)
+}