From a3145bae07f3658183d95d9a6e56d2f3ae38ce76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 28 Sep 2020 12:17:54 +0200
Subject: [PATCH 1/2] daemon cmd: Add progress bar to chain import

---
 cmd/lotus/daemon.go | 18 +++++++++++++++++-
 go.mod              |  1 +
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go
index e0fee6564..93f6e4b8c 100644
--- a/cmd/lotus/daemon.go
+++ b/cmd/lotus/daemon.go
@@ -23,6 +23,7 @@ import (
 	"go.opencensus.io/stats/view"
 	"go.opencensus.io/tag"
 	"golang.org/x/xerrors"
+	"gopkg.in/cheggaaa/pb.v1"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
@@ -333,6 +334,11 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) error {
 	}
 	defer fi.Close() //nolint:errcheck
 
+	st, err := os.Stat(fname)
+	if err != nil {
+		return err
+	}
+
 	lr, err := r.Lock(repo.FullNode)
 	if err != nil {
 		return err
@@ -354,7 +360,17 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) error {
 	cst := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier))
 
 	log.Info("importing chain from file...")
-	ts, err := cst.Import(fi)
+
+	bar := pb.New64(st.Size())
+	br := bar.NewProxyReader(fi)
+	bar.ShowTimeLeft = true
+	bar.ShowPercent = true
+	bar.Units = pb.U_BYTES
+
+	bar.Start()
+	ts, err := cst.Import(br)
+	bar.Finish()
+
 	if err != nil {
 		return xerrors.Errorf("importing chain failed: %w", err)
 	}
diff --git a/go.mod b/go.mod
index 67d87347f..2c0322ecc 100644
--- a/go.mod
+++ b/go.mod
@@ -125,6 +125,7 @@ require (
 	golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
+	gopkg.in/cheggaaa/pb.v1 v1.0.28
 	gotest.tools v2.2.0+incompatible
 	launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect
 )

From f8095296997f6d5d8ba05d66acf291cfa048ae50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 28 Sep 2020 12:28:12 +0200
Subject: [PATCH 2/2] daemon cmd: Support imports straight from http

---
 cmd/lotus/daemon.go | 64 +++++++++++++++++++++++++++++++--------------
 1 file changed, 45 insertions(+), 19 deletions(-)

diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go
index 93f6e4b8c..b976fde79 100644
--- a/cmd/lotus/daemon.go
+++ b/cmd/lotus/daemon.go
@@ -3,11 +3,14 @@
 package main
 
 import (
+	"bufio"
 	"context"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"runtime/pprof"
 	"strings"
@@ -101,11 +104,11 @@ var DaemonCmd = &cli.Command{
 		},
 		&cli.StringFlag{
 			Name:  "import-chain",
-			Usage: "on first run, load chain from given file and validate",
+			Usage: "on first run, load chain from given file or url and validate",
 		},
 		&cli.StringFlag{
 			Name:  "import-snapshot",
-			Usage: "import chain state from a given chain export file",
+			Usage: "import chain state from a given chain export file or url",
 		},
 		&cli.BoolFlag{
 			Name:  "halt-after-import",
@@ -207,11 +210,6 @@ var DaemonCmd = &cli.Command{
 				issnapshot = true
 			}
 
-			chainfile, err := homedir.Expand(chainfile)
-			if err != nil {
-				return err
-			}
-
 			if err := ImportChain(r, chainfile, issnapshot); err != nil {
 				return err
 			}
@@ -327,16 +325,41 @@ func importKey(ctx context.Context, api api.FullNode, f string) error {
 	return nil
 }
 
-func ImportChain(r repo.Repo, fname string, snapshot bool) error {
-	fi, err := os.Open(fname)
-	if err != nil {
-		return err
-	}
-	defer fi.Close() //nolint:errcheck
+func ImportChain(r repo.Repo, fname string, snapshot bool) (err error) {
+	var rd io.Reader
+	var l int64
+	if strings.HasPrefix(fname, "http://") || strings.HasPrefix(fname, "https://") {
+		resp, err := http.Get(fname) //nolint:gosec
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close() //nolint:errcheck
 
-	st, err := os.Stat(fname)
-	if err != nil {
-		return err
+		if resp.StatusCode != http.StatusOK {
+			return xerrors.Errorf("non-200 response: %d", resp.StatusCode)
+		}
+
+		rd = resp.Body
+		l = resp.ContentLength
+	} else {
+		fname, err = homedir.Expand(fname)
+		if err != nil {
+			return err
+		}
+
+		fi, err := os.Open(fname)
+		if err != nil {
+			return err
+		}
+		defer fi.Close() //nolint:errcheck
+
+		st, err := os.Stat(fname)
+		if err != nil {
+			return err
+		}
+
+		rd = fi
+		l = st.Size()
 	}
 
 	lr, err := r.Lock(repo.FullNode)
@@ -359,12 +382,15 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) error {
 
 	cst := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier))
 
-	log.Info("importing chain from file...")
+	log.Infof("importing chain from %s...", fname)
 
-	bar := pb.New64(st.Size())
-	br := bar.NewProxyReader(fi)
+	bufr := bufio.NewReaderSize(rd, 1<<20)
+
+	bar := pb.New64(l)
+	br := bar.NewProxyReader(bufr)
 	bar.ShowTimeLeft = true
 	bar.ShowPercent = true
+	bar.ShowSpeed = true
 	bar.Units = pb.U_BYTES
 
 	bar.Start()