remote ipfs support & automatic env

Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com>

go.mod

@@ -68,6 +68,7 @@ require (
 	github.com/ipfs/interface-go-ipfs-core v0.2.3
 	github.com/ipld/go-car v0.1.1-0.20200430185908-8ff2e52a4c88
 	github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e
+	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/lib/pq v1.2.0
 	github.com/libp2p/go-eventbus v0.1.0
 	github.com/libp2p/go-libp2p v0.9.2

go.sum

@@ -482,6 +482,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/kabukky/httpscerts v0.0.0-20150320125433-617593d7dcb3 h1:Iy7Ifq2ysilWU4QlCx/97OoI4xT1IV7i8byT/EyIT/M=
 github.com/kabukky/httpscerts v0.0.0-20150320125433-617593d7dcb3/go.mod h1:BYpt4ufZiIGv2nXn4gMxnfKV306n3mWXgNu/d2TqdTU=
 github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0=
+github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
+github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=

lib/ipfsbstore/ipfsbstore.go

@@ -7,6 +7,7 @@ import (
 
 	"golang.org/x/xerrors"
 
+	"github.com/multiformats/go-multiaddr"
 	"github.com/multiformats/go-multihash"
 
 	blocks "github.com/ipfs/go-block-format"
@@ -35,6 +36,18 @@ func NewIpfsBstore(ctx context.Context) (*IpfsBstore, error) {
 	}, nil
 }
 
+func NewRemoteIpfsBstore(ctx context.Context, maddr multiaddr.Multiaddr) (*IpfsBstore, error) {
+	api, err := httpapi.NewApi(maddr)
+	if err != nil {
+		return nil, xerrors.Errorf("getting remote ipfs api: %w", err)
+	}
+
+	return &IpfsBstore{
+		ctx: ctx,
+		api: api,
+	}, nil
+}
+
 func (i *IpfsBstore) DeleteBlock(cid cid.Cid) error {
 	return xerrors.Errorf("not supported")
 }

node/builder.go

@@ -378,11 +378,17 @@ func ConfigFullNode(c interface{}) Option {
 		return Error(xerrors.Errorf("invalid config from repo, got: %T", c))
 	}
 
+	remoteIpfsMaddrNotEmpty := func(s *Settings) bool {
+		return len(cfg.Client.RemoteIpfsMAddr) > 0
+	}
+
 	return Options(
 		ConfigCommon(&cfg.Common),
 		If(cfg.Client.UseIpfs,
 			Override(new(dtypes.ClientBlockstore), modules.IpfsClientBlockstore),
 		),
+		ApplyIf(remoteIpfsMaddrNotEmpty,
+			Override(new(dtypes.ClientBlockstore), modules.IpfsRemoteClientBlockstore(cfg.Client.RemoteIpfsMAddr))),
 
 		If(cfg.Metrics.HeadNotifs,
 			Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)),

node/config/def.go

@@ -63,6 +63,7 @@ type Metrics struct {
 
 type Client struct {
 	UseIpfs         bool
+	RemoteIpfsMAddr string
 }
 
 func defCommon() Common {

node/config/load.go

@@ -2,10 +2,12 @@ package config
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"os"
 
 	"github.com/BurntSushi/toml"
+	"github.com/kelseyhightower/envconfig"
 	"golang.org/x/xerrors"
 )
 
@@ -32,6 +34,11 @@ func FromReader(reader io.Reader, def interface{}) (interface{}, error) {
 		return nil, err
 	}
 
+	err = envconfig.Process("LOTUS", cfg)
+	if err != nil {
+		return nil, fmt.Errorf("processing env vars overrides: %s", err)
+	}
+
 	return cfg, nil
 }
 

node/modules/ipfsclient.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/ipfs/go-filestore"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	"github.com/multiformats/go-multiaddr"
 
 	"github.com/filecoin-project/lotus/lib/bufbstore"
 	"github.com/filecoin-project/lotus/lib/ipfsbstore"
@@ -24,3 +25,21 @@ func IpfsClientBlockstore(mctx helpers.MetricsCtx, lc fx.Lifecycle, fstore dtype
 		blockstore.NewIdStore((*filestore.Filestore)(fstore)),
 	), nil
 }
+
+func IpfsRemoteClientBlockstore(ipfsMaddr string) func(helpers.MetricsCtx, fx.Lifecycle, dtypes.ClientFilestore) (dtypes.ClientBlockstore, error) {
+	return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, fstore dtypes.ClientFilestore) (dtypes.ClientBlockstore, error) {
+		ma, err := multiaddr.NewMultiaddr(ipfsMaddr)
+		if err != nil {
+			return nil, xerrors.Errorf("parsing ipfs multiaddr: %w", err)
+		}
+		ipfsbs, err := ipfsbstore.NewRemoteIpfsBstore(helpers.LifecycleCtx(mctx, lc), ma)
+		if err != nil {
+			return nil, xerrors.Errorf("constructing ipfs blockstore: %w", err)
+		}
+
+		return bufbstore.NewTieredBstore(
+			ipfsbs,
+			blockstore.NewIdStore((*filestore.Filestore)(fstore)),
+		), nil
+	}
+}

node/repo/fsrepo.go

@@ -3,7 +3,6 @@ package repo
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/filecoin-project/sector-storage/stores"
 	"io"
 	"io/ioutil"
 	"os"
@@ -11,6 +10,8 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/filecoin-project/sector-storage/stores"
+
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/namespace"
 	badger "github.com/ipfs/go-ds-badger2"
@@ -276,7 +277,7 @@ func (fsr *fsLockedRepo) Datastore(ns string) (datastore.Batching, error) {
 	return namespace.Wrap(fsr.ds, datastore.NewKey(ns)), nil
 }
 
-func (fsr *fsLockedRepo) Config() (interface{}, error) {
+func (fsr *fsLockedRepo) Config() (df interface{}, err error) {
 	if err := fsr.stillValid(); err != nil {
 		return nil, err
 	}