paramfetch: check checksums in parallel

build/paramfetch.go

@@ -8,10 +8,13 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 
 	rice "github.com/GeertJohan/go.rice"
 	logging "github.com/ipfs/go-log"
 	"github.com/minio/blake2b-simd"
+	"go.uber.org/multierr"
+	"golang.org/x/xerrors"
 	pb "gopkg.in/cheggaaa/pb.v1"
 )
 
@@ -27,6 +30,13 @@ type paramFile struct {
 	SectorSize uint64 `json:"sector_size"`
 }
 
+type fetch struct {
+	wg      sync.WaitGroup
+	fetchLk sync.Mutex
+
+	errs []error
+}
+
 func GetParams(storage bool) error {
 	if err := os.Mkdir(paramdir, 0755); err != nil && !os.IsExist(err) {
 		return err
@@ -39,6 +49,8 @@ func GetParams(storage bool) error {
 		return err
 	}
 
+	ft := &fetch{}
+
 	for name, info := range params {
 		if !SupportedSectorSize(info.SectorSize) {
 			continue
@@ -47,18 +59,42 @@ func GetParams(storage bool) error {
 			continue
 		}
 
-		if err := maybeFetch(name, info); err != nil {
+		ft.maybeFetchAsync(name, info)
+	}
+
+	return ft.wait()
+}
+
+func (ft *fetch) maybeFetchAsync(name string, info paramFile) {
+	ft.wg.Add(1)
+
+	go func() {
+		defer ft.wg.Done()
+
+		path := filepath.Join(paramdir, name)
+
+		err := ft.checkFile(path, info)
+		if !os.IsNotExist(err) && err != nil {
+			log.Warn(err)
+		}
+		if err == nil {
+			return
+		}
+
+		ft.fetchLk.Lock()
+		defer ft.fetchLk.Unlock()
+
+		if err := doFetch(path, info); err != nil {
+			ft.errs = append(ft.errs, xerrors.Errorf("fetching file %s: %w", path, err))
+		}
+	}()
+}
+
+func (ft *fetch) checkFile(path string, info paramFile) error {
+	f, err := os.Open(path)
+	if err != nil {
 		return err
 	}
-	}
-
-	return nil
-}
-
-func maybeFetch(name string, info paramFile) error {
-	path := filepath.Join(paramdir, name)
-	f, err := os.Open(path)
-	if err == nil {
 	defer f.Close()
 
 	h := blake2b.New512()
@@ -72,10 +108,12 @@ func maybeFetch(name string, info paramFile) error {
 		return nil
 	}
 
-		log.Warnf("Checksum mismatch in param file %s, %s != %s", name, strSum, info.Digest)
+	return xerrors.Errorf("checksum mismatch in param file %s, %s != %s", path, strSum, info.Digest)
 }
 
-	return doFetch(path, info)
+func (ft *fetch) wait() error {
+	ft.wg.Wait()
+	return multierr.Combine(ft.errs...)
 }
 
 func doFetch(out string, info paramFile) error {

chain/gen/utils.go

@@ -372,7 +372,7 @@ func MakeGenesisBlock(bs bstore.Blockstore, balances map[address.Address]types.B
 	log.Infof("Empty Genesis root: %s", emptyroot)
 
 	genesisticket := &types.Ticket{
-		VRFProof: []byte("vrf proof"),
+		VRFProof: []byte("vrf proof0000000vrf proof0000000"),
 	}
 
 	b := &types.BlockHeader{

chain/types/blockheader_test.go

@@ -27,7 +27,7 @@ func testBlockHeader(t testing.TB) *BlockHeader {
 		ElectionProof: []byte("cats won the election"),
 		Tickets: []*Ticket{
 			&Ticket{
-				VRFProof: []byte("vrf proof"),
+				VRFProof: []byte("vrf proof0000000vrf proof0000000"),
 			},
 		},
 		Parents:               []cid.Cid{c, c},

go.mod

@@ -79,6 +79,7 @@ require (
 	go.uber.org/dig v1.7.0 // indirect
 	go.uber.org/fx v1.9.0
 	go.uber.org/goleak v0.10.0 // indirect
+	go.uber.org/multierr v1.1.0
 	go.uber.org/zap v1.10.0
 	go4.org v0.0.0-20190313082347-94abd6928b1d // indirect
 	golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 // indirect

node/modules/storageminer.go

@@ -181,7 +181,6 @@ func SealTicketGen(api api.FullNode) sector.TicketFn {
 			return nil, xerrors.Errorf("unexpected randomness len: %d (expected %d)", n, sectorbuilder.CommLen)
 		}
 
-
 		return &sectorbuilder.SealTicket{
 			BlockHeight: ts.Height(),
 			TicketBytes: tkt,