lotus/build/paramfetch.go

190 lines
3.7 KiB
Go
Raw Normal View History

2019-10-02 17:20:30 +00:00
package build
import (
"encoding/hex"
"encoding/json"
"io"
"net/http"
2019-11-16 11:52:50 +00:00
"net/url"
2019-10-02 17:20:30 +00:00
"os"
"path/filepath"
2019-11-16 11:52:50 +00:00
"strconv"
2019-10-02 17:20:30 +00:00
"strings"
"sync"
2019-10-02 17:20:30 +00:00
rice "github.com/GeertJohan/go.rice"
logging "github.com/ipfs/go-log"
"github.com/minio/blake2b-simd"
"go.uber.org/multierr"
"golang.org/x/xerrors"
2019-10-02 17:20:30 +00:00
pb "gopkg.in/cheggaaa/pb.v1"
)
2019-10-02 20:29:40 +00:00
var log = logging.Logger("build")
2019-10-02 17:20:30 +00:00
2019-10-22 06:43:14 +00:00
//const gateway = "http://198.211.99.118/ipfs/"
2019-10-31 10:21:37 +00:00
const gateway = "https://ipfs.io/ipfs/"
2019-10-02 17:20:30 +00:00
const paramdir = "/var/tmp/filecoin-proof-parameters"
const dirEnv = "FIL_PROOFS_PARAMETER_CACHE"
2019-10-02 17:20:30 +00:00
2019-10-02 20:29:40 +00:00
type paramFile struct {
Cid string `json:"cid"`
Digest string `json:"digest"`
2019-10-02 17:20:30 +00:00
SectorSize uint64 `json:"sector_size"`
}
type fetch struct {
wg sync.WaitGroup
fetchLk sync.Mutex
errs []error
}
func getParamDir() string {
if os.Getenv(dirEnv) == "" {
return paramdir
}
return os.Getenv(dirEnv)
}
2019-12-04 19:44:15 +00:00
func GetParams(storageSize uint64) error {
if err := os.Mkdir(getParamDir(), 0755); err != nil && !os.IsExist(err) {
2019-10-02 23:36:33 +00:00
return err
}
2019-10-02 17:20:30 +00:00
var params map[string]paramFile
paramBytes := rice.MustFindBox("proof-params").MustBytes("parameters.json")
if err := json.Unmarshal(paramBytes, &params); err != nil {
return err
}
ft := &fetch{}
2019-10-02 17:20:30 +00:00
for name, info := range params {
2019-12-04 19:44:15 +00:00
if storageSize != info.SectorSize && strings.HasSuffix(name, ".params") {
2019-10-02 17:20:30 +00:00
continue
}
2019-10-31 10:21:37 +00:00
ft.maybeFetchAsync(name, info)
2019-10-02 17:20:30 +00:00
}
return ft.wait()
2019-10-02 17:20:30 +00:00
}
2019-10-31 10:21:37 +00:00
func (ft *fetch) maybeFetchAsync(name string, info paramFile) {
ft.wg.Add(1)
2019-10-02 17:20:30 +00:00
go func() {
defer ft.wg.Done()
path := filepath.Join(getParamDir(), name)
err := ft.checkFile(path, info)
if !os.IsNotExist(err) && err != nil {
log.Warn(err)
}
if err == nil {
return
2019-10-02 17:20:30 +00:00
}
2019-12-05 09:58:57 +00:00
ft.fetchLk.Lock()
defer ft.fetchLk.Unlock()
2019-10-31 10:21:37 +00:00
if err := doFetch(path, info); err != nil {
2019-11-18 03:00:31 +00:00
ft.errs = append(ft.errs, xerrors.Errorf("fetching file %s failed: %w", path, err))
return
}
err = ft.checkFile(path, info)
if err != nil {
2019-11-18 03:00:31 +00:00
ft.errs = append(ft.errs, xerrors.Errorf("checking file %s failed: %w", path, err))
err := os.Remove(path)
if err != nil {
ft.errs = append(ft.errs, xerrors.Errorf("remove file %s failed: %w", path, err))
}
2019-10-02 17:20:30 +00:00
}
}()
}
2019-10-02 17:20:30 +00:00
func (ft *fetch) checkFile(path string, info paramFile) error {
if os.Getenv("TRUST_PARAMS") == "1" {
log.Warn("Assuming parameter files are ok. DO NOT USE IN PRODUCTION")
2019-11-06 23:09:48 +00:00
return nil
}
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
h := blake2b.New512()
if _, err := io.Copy(h, f); err != nil {
return err
}
sum := h.Sum(nil)
strSum := hex.EncodeToString(sum[:16])
if strSum == info.Digest {
log.Infof("Parameter file %s is ok", path)
return nil
2019-10-02 17:20:30 +00:00
}
return xerrors.Errorf("checksum mismatch in param file %s, %s != %s", path, strSum, info.Digest)
}
func (ft *fetch) wait() error {
ft.wg.Wait()
return multierr.Combine(ft.errs...)
2019-10-02 17:20:30 +00:00
}
2019-10-31 10:21:37 +00:00
func doFetch(out string, info paramFile) error {
gw := os.Getenv("IPFS_GATEWAY")
if gw == "" {
gw = gateway
}
log.Infof("Fetching %s from %s", out, gw)
2019-10-02 17:20:30 +00:00
2019-11-19 15:51:12 +00:00
outf, err := os.OpenFile(out, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
2019-10-02 20:29:40 +00:00
if err != nil {
return err
}
2019-11-16 11:52:50 +00:00
defer outf.Close()
2019-10-02 20:29:40 +00:00
2019-11-16 11:52:50 +00:00
fStat, err := outf.Stat()
2019-10-02 20:29:40 +00:00
if err != nil {
return err
}
2019-11-16 11:52:50 +00:00
header := http.Header{}
2019-11-19 15:51:12 +00:00
header.Set("Range", "bytes="+strconv.FormatInt(fStat.Size(), 10)+"-")
2019-11-16 11:52:50 +00:00
url, err := url.Parse(gw + info.Cid)
if err != nil {
return err
}
2019-11-27 17:10:34 +00:00
log.Infof("GET %s", url)
2019-11-16 11:52:50 +00:00
req := http.Request{
Method: "GET",
2019-11-19 15:51:12 +00:00
URL: url,
2019-11-16 11:52:50 +00:00
Header: header,
2019-11-19 15:51:12 +00:00
Close: true,
2019-11-16 11:52:50 +00:00
}
resp, err := http.DefaultClient.Do(&req)
if err != nil {
return err
}
defer resp.Body.Close()
2019-10-02 20:29:40 +00:00
bar := pb.New64(resp.ContentLength)
2019-10-02 17:20:30 +00:00
bar.Units = pb.U_BYTES
bar.ShowSpeed = true
bar.Start()
2019-10-02 20:29:40 +00:00
_, err = io.Copy(outf, bar.NewProxyReader(resp.Body))
2019-10-02 17:20:30 +00:00
2019-10-02 20:29:40 +00:00
bar.Finish()
2019-10-02 17:20:30 +00:00
return err
}