From 3cd3785385a326248e097ba831284c0ab028dd78 Mon Sep 17 00:00:00 2001 From: vyzo Date: Tue, 26 May 2020 20:47:00 +0300 Subject: [PATCH 01/11] gomod: update go-libp2p-pubsub@v0.3.1 --- go.mod | 2 +- go.sum | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7b04d39be..5dc2ae55c 100644 --- a/go.mod +++ b/go.mod @@ -78,7 +78,7 @@ require ( github.com/libp2p/go-libp2p-mplex v0.2.3 github.com/libp2p/go-libp2p-peer v0.2.0 github.com/libp2p/go-libp2p-peerstore v0.2.4 - github.com/libp2p/go-libp2p-pubsub v0.3.0 + github.com/libp2p/go-libp2p-pubsub v0.3.1 github.com/libp2p/go-libp2p-quic-transport v0.1.1 github.com/libp2p/go-libp2p-record v0.1.2 github.com/libp2p/go-libp2p-routing-helpers v0.2.1 diff --git a/go.sum b/go.sum index b407f56ba..4edb733c4 100644 --- a/go.sum +++ b/go.sum @@ -175,6 +175,7 @@ github.com/filecoin-project/sector-storage v0.0.0-20200522011946-a59ca7536a95 h1 github.com/filecoin-project/sector-storage v0.0.0-20200522011946-a59ca7536a95/go.mod h1:1UpuWCUC7XNfgyufyQQq+/40JSzhfGdgcwauK55C8gk= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= +github.com/filecoin-project/specs-actors v0.2.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= github.com/filecoin-project/specs-actors v0.5.2/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= github.com/filecoin-project/specs-actors v0.5.4-0.20200521014528-0df536f7e461 h1:yyAoJ9dNboljDWj0uBdJEbdaTak/YhkjYUQt0GzlY0A= @@ -654,8 +655,8 @@ github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYc github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= -github.com/libp2p/go-libp2p-pubsub v0.3.0 h1:K5FSYyfcSrJWrGExgdbogCLMqwC3pQaXEVt2CaUy1SA= -github.com/libp2p/go-libp2p-pubsub v0.3.0/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato= +github.com/libp2p/go-libp2p-pubsub v0.3.1 h1:7Hyv2d8BK/x1HGRJTZ8X++VQEP+WqDTSwpUSZGTVLYA= +github.com/libp2p/go-libp2p-pubsub v0.3.1/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato= github.com/libp2p/go-libp2p-quic-transport v0.1.1 h1:MFMJzvsxIEDEVKzO89BnB/FgvMj9WI4GDGUW2ArDPUA= github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU= github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFAymLDLm7lxMgU79Q= From c8f28103f3b0172798f3b6c9f5c8b3af9f8ffd0b Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 21 May 2020 11:17:34 -0700 Subject: [PATCH 02/11] update chain-val syscalls interface --- chain/validation/applier.go | 8 +++++--- chain/validation/factories.go | 19 +++---------------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/chain/validation/applier.go b/chain/validation/applier.go index 5289f840b..f30c543bb 100644 --- a/chain/validation/applier.go +++ b/chain/validation/applier.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/crypto" "github.com/filecoin-project/specs-actors/actors/puppet" + "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/ipfs/go-cid" vtypes "github.com/filecoin-project/chain-validation/chain/types" @@ -24,12 +25,13 @@ import ( // Applier applies messages to state trees and storage. type Applier struct { stateWrapper *StateWrapper + syscalls runtime.Syscalls } var _ vstate.Applier = &Applier{} -func NewApplier(sw *StateWrapper) *Applier { - return &Applier{sw} +func NewApplier(sw *StateWrapper, syscalls runtime.Syscalls) *Applier { + return &Applier{sw, syscalls} } func (a *Applier) ApplyMessage(epoch abi.ChainEpoch, message *vtypes.Message) (vtypes.ApplyMessageResult, error) { @@ -65,7 +67,7 @@ func (a *Applier) ApplySignedMessage(epoch abi.ChainEpoch, msg *vtypes.SignedMes } func (a *Applier) ApplyTipSetMessages(epoch abi.ChainEpoch, blocks []vtypes.BlockMessagesInfo, rnd vstate.RandomnessSource) (vtypes.ApplyTipSetResult, error) { - cs := store.NewChainStore(a.stateWrapper.bs, a.stateWrapper.ds, vdrivers.NewChainValidationSyscalls()) + cs := store.NewChainStore(a.stateWrapper.bs, a.stateWrapper.ds, a.syscalls) sm := stmgr.NewStateManager(cs) var bms []stmgr.BlockMessages diff --git a/chain/validation/factories.go b/chain/validation/factories.go index 223382538..d3771d87d 100644 --- a/chain/validation/factories.go +++ b/chain/validation/factories.go @@ -1,11 +1,9 @@ package validation import ( - "context" + "github.com/filecoin-project/specs-actors/actors/runtime" vstate "github.com/filecoin-project/chain-validation/state" - "github.com/filecoin-project/specs-actors/actors/abi" - acrypto "github.com/filecoin-project/specs-actors/actors/crypto" ) type Factories struct { @@ -18,26 +16,15 @@ func NewFactories() *Factories { return &Factories{} } -func (f *Factories) NewStateAndApplier() (vstate.VMWrapper, vstate.Applier) { +func (f *Factories) NewStateAndApplier(syscalls runtime.Syscalls) (vstate.VMWrapper, vstate.Applier) { st := NewState() - return st, NewApplier(st) + return st, NewApplier(st, syscalls) } func (f *Factories) NewKeyManager() vstate.KeyManager { return newKeyManager() } -type fakeRandSrc struct { -} - -func (r fakeRandSrc) Randomness(_ context.Context, _ acrypto.DomainSeparationTag, _ abi.ChainEpoch, _ []byte) (abi.Randomness, error) { - return abi.Randomness("sausages"), nil -} - -func (f *Factories) NewRandomnessSource() vstate.RandomnessSource { - return &fakeRandSrc{} -} - func (f *Factories) NewValidationConfig() vstate.ValidationConfig { trackGas := true checkExit := true From 8ce6921b1811005c0a1e1aa366c15aaf541a9624 Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 26 May 2020 10:32:21 -0700 Subject: [PATCH 03/11] update chain-validation - has new syscalls --- chain/validation/applier.go | 3 +-- go.mod | 4 ++-- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/chain/validation/applier.go b/chain/validation/applier.go index f30c543bb..6c1629104 100644 --- a/chain/validation/applier.go +++ b/chain/validation/applier.go @@ -13,7 +13,6 @@ import ( "github.com/ipfs/go-cid" vtypes "github.com/filecoin-project/chain-validation/chain/types" - vdrivers "github.com/filecoin-project/chain-validation/drivers" vstate "github.com/filecoin-project/chain-validation/state" "github.com/filecoin-project/lotus/chain/stmgr" @@ -136,7 +135,7 @@ func (a *Applier) applyMessage(epoch abi.ChainEpoch, lm types.ChainMsg) (vtypes. ctx := context.TODO() base := a.stateWrapper.Root() - lotusVM, err := vm.NewVM(base, epoch, &vmRand{}, a.stateWrapper.bs, vdrivers.NewChainValidationSyscalls()) + lotusVM, err := vm.NewVM(base, epoch, &vmRand{}, a.stateWrapper.bs, a.syscalls) // need to modify the VM invoker to add the puppet actor chainValInvoker := vm.NewInvoker() chainValInvoker.Register(puppet.PuppetActorCodeID, puppet.Actor{}, puppet.State{}) diff --git a/go.mod b/go.mod index 7b04d39be..4f2e67a76 100644 --- a/go.mod +++ b/go.mod @@ -13,8 +13,8 @@ require ( github.com/docker/go-units v0.4.0 github.com/drand/drand v0.8.2-0.20200518165838-d61135e6e2c8 github.com/fatih/color v1.8.0 - github.com/filecoin-project/chain-validation v0.0.6-0.20200518190139-483332336e8e - github.com/filecoin-project/filecoin-ffi v0.0.0-20200427223233-a0014b17f124 + github.com/filecoin-project/chain-validation v0.0.6-0.20200526171800-c56c1882dc99 + github.com/filecoin-project/filecoin-ffi v0.26.1-0.20200508175440-05b30afeb00d github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2 github.com/filecoin-project/go-bitfield v0.0.1 diff --git a/go.sum b/go.sum index b407f56ba..8f01e6170 100644 --- a/go.sum +++ b/go.sum @@ -133,8 +133,8 @@ github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:Jp github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= -github.com/filecoin-project/chain-validation v0.0.6-0.20200518190139-483332336e8e h1:3x2eL2t3ZkMOHt1b5WS5aVWyJeo5+WjWCT77QdPGSwk= -github.com/filecoin-project/chain-validation v0.0.6-0.20200518190139-483332336e8e/go.mod h1:6B3uenDcH8n+PKqgzUtZmgyCzKy4qpiLwJ5aw7Rj2xQ= +github.com/filecoin-project/chain-validation v0.0.6-0.20200526171800-c56c1882dc99 h1:jVuyHbsCX/w4vZ+3pO0cqzOyqvwfo8tB94wzmcwaD2k= +github.com/filecoin-project/chain-validation v0.0.6-0.20200526171800-c56c1882dc99/go.mod h1:hMqMCTt3z+wZ+GL74uy7X4NptnucZTTrb5SUTt1L/0g= github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef h1:Wi5E+P1QfHP8IF27eUiTx5vYfqQZwfPxzq3oFEq8w8U= From d5346f83262b98f8d82211f4f206f3ae8ecae08e Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Fri, 22 May 2020 11:51:18 -0300 Subject: [PATCH 04/11] remote ipfs support & automatic env Signed-off-by: Ignacio Hagopian --- go.mod | 1 + go.sum | 2 ++ lib/ipfsbstore/ipfsbstore.go | 13 +++++++++++++ node/builder.go | 6 ++++++ node/config/def.go | 3 ++- node/config/load.go | 7 +++++++ node/modules/ipfsclient.go | 19 +++++++++++++++++++ node/repo/fsrepo.go | 5 +++-- 8 files changed, 53 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7b04d39be..5d0f21b64 100644 --- a/go.mod +++ b/go.mod @@ -68,6 +68,7 @@ require ( github.com/ipfs/interface-go-ipfs-core v0.2.3 github.com/ipld/go-car v0.1.1-0.20200430185908-8ff2e52a4c88 github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e + github.com/kelseyhightower/envconfig v1.4.0 github.com/lib/pq v1.2.0 github.com/libp2p/go-eventbus v0.1.0 github.com/libp2p/go-libp2p v0.9.2 diff --git a/go.sum b/go.sum index b407f56ba..e241cd3c8 100644 --- a/go.sum +++ b/go.sum @@ -482,6 +482,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/kabukky/httpscerts v0.0.0-20150320125433-617593d7dcb3 h1:Iy7Ifq2ysilWU4QlCx/97OoI4xT1IV7i8byT/EyIT/M= github.com/kabukky/httpscerts v0.0.0-20150320125433-617593d7dcb3/go.mod h1:BYpt4ufZiIGv2nXn4gMxnfKV306n3mWXgNu/d2TqdTU= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= diff --git a/lib/ipfsbstore/ipfsbstore.go b/lib/ipfsbstore/ipfsbstore.go index a79407f5a..33110e295 100644 --- a/lib/ipfsbstore/ipfsbstore.go +++ b/lib/ipfsbstore/ipfsbstore.go @@ -7,6 +7,7 @@ import ( "golang.org/x/xerrors" + "github.com/multiformats/go-multiaddr" "github.com/multiformats/go-multihash" blocks "github.com/ipfs/go-block-format" @@ -35,6 +36,18 @@ func NewIpfsBstore(ctx context.Context) (*IpfsBstore, error) { }, nil } +func NewRemoteIpfsBstore(ctx context.Context, maddr multiaddr.Multiaddr) (*IpfsBstore, error) { + api, err := httpapi.NewApi(maddr) + if err != nil { + return nil, xerrors.Errorf("getting remote ipfs api: %w", err) + } + + return &IpfsBstore{ + ctx: ctx, + api: api, + }, nil +} + func (i *IpfsBstore) DeleteBlock(cid cid.Cid) error { return xerrors.Errorf("not supported") } diff --git a/node/builder.go b/node/builder.go index 0154f1494..5fe1e4346 100644 --- a/node/builder.go +++ b/node/builder.go @@ -378,11 +378,17 @@ func ConfigFullNode(c interface{}) Option { return Error(xerrors.Errorf("invalid config from repo, got: %T", c)) } + remoteIpfsMaddrNotEmpty := func(s *Settings) bool { + return len(cfg.Client.RemoteIpfsMAddr) > 0 + } + return Options( ConfigCommon(&cfg.Common), If(cfg.Client.UseIpfs, Override(new(dtypes.ClientBlockstore), modules.IpfsClientBlockstore), ), + ApplyIf(remoteIpfsMaddrNotEmpty, + Override(new(dtypes.ClientBlockstore), modules.IpfsRemoteClientBlockstore(cfg.Client.RemoteIpfsMAddr))), If(cfg.Metrics.HeadNotifs, Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)), diff --git a/node/config/def.go b/node/config/def.go index 72fdde4ff..eea38965b 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -62,7 +62,8 @@ type Metrics struct { } type Client struct { - UseIpfs bool + UseIpfs bool + RemoteIpfsMAddr string } func defCommon() Common { diff --git a/node/config/load.go b/node/config/load.go index 352d17197..1590efedb 100644 --- a/node/config/load.go +++ b/node/config/load.go @@ -2,10 +2,12 @@ package config import ( "bytes" + "fmt" "io" "os" "github.com/BurntSushi/toml" + "github.com/kelseyhightower/envconfig" "golang.org/x/xerrors" ) @@ -32,6 +34,11 @@ func FromReader(reader io.Reader, def interface{}) (interface{}, error) { return nil, err } + err = envconfig.Process("LOTUS", cfg) + if err != nil { + return nil, fmt.Errorf("processing env vars overrides: %s", err) + } + return cfg, nil } diff --git a/node/modules/ipfsclient.go b/node/modules/ipfsclient.go index 8898e4cd1..373f8531f 100644 --- a/node/modules/ipfsclient.go +++ b/node/modules/ipfsclient.go @@ -6,6 +6,7 @@ import ( "github.com/ipfs/go-filestore" blockstore "github.com/ipfs/go-ipfs-blockstore" + "github.com/multiformats/go-multiaddr" "github.com/filecoin-project/lotus/lib/bufbstore" "github.com/filecoin-project/lotus/lib/ipfsbstore" @@ -24,3 +25,21 @@ func IpfsClientBlockstore(mctx helpers.MetricsCtx, lc fx.Lifecycle, fstore dtype blockstore.NewIdStore((*filestore.Filestore)(fstore)), ), nil } + +func IpfsRemoteClientBlockstore(ipfsMaddr string) func(helpers.MetricsCtx, fx.Lifecycle, dtypes.ClientFilestore) (dtypes.ClientBlockstore, error) { + return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, fstore dtypes.ClientFilestore) (dtypes.ClientBlockstore, error) { + ma, err := multiaddr.NewMultiaddr(ipfsMaddr) + if err != nil { + return nil, xerrors.Errorf("parsing ipfs multiaddr: %w", err) + } + ipfsbs, err := ipfsbstore.NewRemoteIpfsBstore(helpers.LifecycleCtx(mctx, lc), ma) + if err != nil { + return nil, xerrors.Errorf("constructing ipfs blockstore: %w", err) + } + + return bufbstore.NewTieredBstore( + ipfsbs, + blockstore.NewIdStore((*filestore.Filestore)(fstore)), + ), nil + } +} diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 322adbdef..830de3b24 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -3,7 +3,6 @@ package repo import ( "encoding/json" "fmt" - "github.com/filecoin-project/sector-storage/stores" "io" "io/ioutil" "os" @@ -11,6 +10,8 @@ import ( "strings" "sync" + "github.com/filecoin-project/sector-storage/stores" + "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" badger "github.com/ipfs/go-ds-badger2" @@ -276,7 +277,7 @@ func (fsr *fsLockedRepo) Datastore(ns string) (datastore.Batching, error) { return namespace.Wrap(fsr.ds, datastore.NewKey(ns)), nil } -func (fsr *fsLockedRepo) Config() (interface{}, error) { +func (fsr *fsLockedRepo) Config() (df interface{}, err error) { if err := fsr.stillValid(); err != nil { return nil, err } From 61daaa774bf7e072bb419b93d292a3fde1401217 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Tue, 26 May 2020 09:50:35 -0300 Subject: [PATCH 05/11] allow using ipfs for retrieval Signed-off-by: Ignacio Hagopian --- node/builder.go | 11 +++-------- node/config/def.go | 5 +++-- node/modules/ipfsclient.go | 36 +++++++++++++++++------------------- 3 files changed, 23 insertions(+), 29 deletions(-) diff --git a/node/builder.go b/node/builder.go index 5fe1e4346..d054be7a5 100644 --- a/node/builder.go +++ b/node/builder.go @@ -378,18 +378,13 @@ func ConfigFullNode(c interface{}) Option { return Error(xerrors.Errorf("invalid config from repo, got: %T", c)) } - remoteIpfsMaddrNotEmpty := func(s *Settings) bool { - return len(cfg.Client.RemoteIpfsMAddr) > 0 - } - + ipfsMaddr := cfg.Client.IpfsMAddr + useForRetrieval := cfg.Client.IpfsUseForRetrieval return Options( ConfigCommon(&cfg.Common), If(cfg.Client.UseIpfs, - Override(new(dtypes.ClientBlockstore), modules.IpfsClientBlockstore), + Override(new(dtypes.ClientBlockstore), modules.IpfsClientBlockstore(ipfsMaddr, useForRetrieval)), ), - ApplyIf(remoteIpfsMaddrNotEmpty, - Override(new(dtypes.ClientBlockstore), modules.IpfsRemoteClientBlockstore(cfg.Client.RemoteIpfsMAddr))), - If(cfg.Metrics.HeadNotifs, Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)), ), diff --git a/node/config/def.go b/node/config/def.go index eea38965b..d5c64d40d 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -62,8 +62,9 @@ type Metrics struct { } type Client struct { - UseIpfs bool - RemoteIpfsMAddr string + UseIpfs bool + IpfsMAddr string + IpfsUseForRetrieval bool } func defCommon() Common { diff --git a/node/modules/ipfsclient.go b/node/modules/ipfsclient.go index 373f8531f..c3710de1c 100644 --- a/node/modules/ipfsclient.go +++ b/node/modules/ipfsclient.go @@ -14,32 +14,30 @@ import ( "github.com/filecoin-project/lotus/node/modules/helpers" ) -func IpfsClientBlockstore(mctx helpers.MetricsCtx, lc fx.Lifecycle, fstore dtypes.ClientFilestore) (dtypes.ClientBlockstore, error) { - ipfsbs, err := ipfsbstore.NewIpfsBstore(helpers.LifecycleCtx(mctx, lc)) - if err != nil { - return nil, xerrors.Errorf("constructing ipfs blockstore: %w", err) - } - - return bufbstore.NewTieredBstore( - ipfsbs, - blockstore.NewIdStore((*filestore.Filestore)(fstore)), - ), nil -} - -func IpfsRemoteClientBlockstore(ipfsMaddr string) func(helpers.MetricsCtx, fx.Lifecycle, dtypes.ClientFilestore) (dtypes.ClientBlockstore, error) { +// IpfsClientBlockstore returns a ClientBlockstore implementation backed by an IPFS node. +// If ipfsMaddr is empty, a local IPFS node is assumed considering IPFS_PATH configuration. +// If ipfsMaddr is not empty, it will connect to the remote IPFS node with the provided multiaddress. +// The flag useForRetrieval indicates if the IPFS node will also be used for storing retrieving deals. +func IpfsClientBlockstore(ipfsMaddr string, useForRetrieval bool) func(helpers.MetricsCtx, fx.Lifecycle, dtypes.ClientFilestore) (dtypes.ClientBlockstore, error) { return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, fstore dtypes.ClientFilestore) (dtypes.ClientBlockstore, error) { ma, err := multiaddr.NewMultiaddr(ipfsMaddr) if err != nil { return nil, xerrors.Errorf("parsing ipfs multiaddr: %w", err) } - ipfsbs, err := ipfsbstore.NewRemoteIpfsBstore(helpers.LifecycleCtx(mctx, lc), ma) + var ipfsbs *ipfsbstore.IpfsBstore + if ipfsMaddr != "" { + ipfsbs, err = ipfsbstore.NewRemoteIpfsBstore(helpers.LifecycleCtx(mctx, lc), ma) + } else { + ipfsbs, err = ipfsbstore.NewIpfsBstore(helpers.LifecycleCtx(mctx, lc)) + } if err != nil { return nil, xerrors.Errorf("constructing ipfs blockstore: %w", err) } - - return bufbstore.NewTieredBstore( - ipfsbs, - blockstore.NewIdStore((*filestore.Filestore)(fstore)), - ), nil + var ws blockstore.Blockstore + ws = ipfsbs + if !useForRetrieval { + ws = blockstore.NewIdStore((*filestore.Filestore)(fstore)) + } + return bufbstore.NewTieredBstore(ipfsbs, ws), nil } } From bcd84cbe9deaf71d108a73fea4b8db18041541ce Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Tue, 26 May 2020 12:36:21 -0300 Subject: [PATCH 06/11] retrieve: make fileref optional Signed-off-by: Ignacio Hagopian --- api/api_full.go | 2 +- api/apistruct/struct.go | 4 ++-- api/test/deals.go | 2 +- cli/client.go | 2 +- node/impl/client/client.go | 7 ++++++- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 9026bf2d5..c6d442e2b 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -117,7 +117,7 @@ type FullNode interface { ClientListDeals(ctx context.Context) ([]DealInfo, error) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) - ClientRetrieve(ctx context.Context, order RetrievalOrder, ref FileRef) error + ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *FileRef) error ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) ClientCalcCommP(ctx context.Context, inpath string, miner address.Address) (*CommPRet, error) ClientGenCar(ctx context.Context, ref FileRef, outpath string) error diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 368a2b145..f0be44070 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -110,7 +110,7 @@ type FullNodeStruct struct { ClientStartDeal func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) `perm:"admin"` ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error) `perm:"read"` ClientListDeals func(ctx context.Context) ([]api.DealInfo, error) `perm:"write"` - ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, ref api.FileRef) error `perm:"admin"` + ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error `perm:"admin"` ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) `perm:"read"` ClientCalcCommP func(ctx context.Context, inpath string, miner address.Address) (*api.CommPRet, error) `perm:"read"` ClientGenCar func(ctx context.Context, ref api.FileRef, outpath string) error `perm:"write"` @@ -321,7 +321,7 @@ func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]api.DealInfo, e return c.Internal.ClientListDeals(ctx) } -func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref api.FileRef) error { +func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error { return c.Internal.ClientRetrieve(ctx, order, ref) } diff --git a/api/test/deals.go b/api/test/deals.go index 2414d3fe2..625be4583 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -200,7 +200,7 @@ func testRetrieval(t *testing.T, ctx context.Context, err error, client *impl.Fu t.Fatal(err) } - ref := api.FileRef{ + ref := &api.FileRef{ Path: filepath.Join(rpath, "ret"), IsCAR: carExport, } diff --git a/cli/client.go b/cli/client.go index 7f7e5e6c4..3bfc5de70 100644 --- a/cli/client.go +++ b/cli/client.go @@ -384,7 +384,7 @@ var clientRetrieveCmd = &cli.Command{ return nil } - ref := lapi.FileRef{ + ref := &lapi.FileRef{ Path: cctx.Args().Get(1), IsCAR: cctx.Bool("car"), } diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 320579ce1..ff974e8b2 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -302,7 +302,7 @@ func (a *API) ClientListImports(ctx context.Context) ([]api.Import, error) { } } -func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref api.FileRef) error { +func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error { if order.MinerPeerID == "" { mi, err := a.StateMinerInfo(ctx, order.Miner, types.EmptyTSK) if err != nil { @@ -353,6 +353,11 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref unsubscribe() + // If ref is nil, it only fetches the data into the configured blockstore. + if ref == nil { + return nil + } + if ref.IsCAR { f, err := os.OpenFile(ref.Path, os.O_CREATE|os.O_WRONLY, 0644) if err != nil { From a10690359cdcf693abfdee270f3fd165505c5f74 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Tue, 26 May 2020 15:35:20 -0300 Subject: [PATCH 07/11] revert file Signed-off-by: Ignacio Hagopian --- node/repo/fsrepo.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 830de3b24..322adbdef 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -3,6 +3,7 @@ package repo import ( "encoding/json" "fmt" + "github.com/filecoin-project/sector-storage/stores" "io" "io/ioutil" "os" @@ -10,8 +11,6 @@ import ( "strings" "sync" - "github.com/filecoin-project/sector-storage/stores" - "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" badger "github.com/ipfs/go-ds-badger2" @@ -277,7 +276,7 @@ func (fsr *fsLockedRepo) Datastore(ns string) (datastore.Batching, error) { return namespace.Wrap(fsr.ds, datastore.NewKey(ns)), nil } -func (fsr *fsLockedRepo) Config() (df interface{}, err error) { +func (fsr *fsLockedRepo) Config() (interface{}, error) { if err := fsr.stillValid(); err != nil { return nil, err } From 02afd7d71f1205c57fc68053f978ba5aaa746368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 27 May 2020 10:13:06 +0200 Subject: [PATCH 08/11] cli: Trim spaces from token file --- node/repo/fsrepo.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 322adbdef..396fbf67e 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -1,9 +1,9 @@ package repo import ( + "bytes" "encoding/json" "fmt" - "github.com/filecoin-project/sector-storage/stores" "io" "io/ioutil" "os" @@ -21,6 +21,8 @@ import ( "github.com/multiformats/go-multiaddr" "golang.org/x/xerrors" + "github.com/filecoin-project/sector-storage/stores" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/config" ) @@ -191,7 +193,12 @@ func (fsr *FsRepo) APIToken() ([]byte, error) { } defer f.Close() //nolint: errcheck // Read only op - return ioutil.ReadAll(f) + tb, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + + return bytes.TrimSpace(tb), nil } // Lock acquires exclusive lock on this repo From 009cd920104f6e479c00ad39e7b95fd89ffeb8a9 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Wed, 27 May 2020 09:50:32 -0300 Subject: [PATCH 09/11] fix Signed-off-by: Ignacio Hagopian --- node/modules/ipfsclient.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/node/modules/ipfsclient.go b/node/modules/ipfsclient.go index c3710de1c..f405cb4c6 100644 --- a/node/modules/ipfsclient.go +++ b/node/modules/ipfsclient.go @@ -20,12 +20,14 @@ import ( // The flag useForRetrieval indicates if the IPFS node will also be used for storing retrieving deals. func IpfsClientBlockstore(ipfsMaddr string, useForRetrieval bool) func(helpers.MetricsCtx, fx.Lifecycle, dtypes.ClientFilestore) (dtypes.ClientBlockstore, error) { return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, fstore dtypes.ClientFilestore) (dtypes.ClientBlockstore, error) { - ma, err := multiaddr.NewMultiaddr(ipfsMaddr) - if err != nil { - return nil, xerrors.Errorf("parsing ipfs multiaddr: %w", err) - } + var err error var ipfsbs *ipfsbstore.IpfsBstore if ipfsMaddr != "" { + var ma multiaddr.Multiaddr + ma, err = multiaddr.NewMultiaddr(ipfsMaddr) + if err != nil { + return nil, xerrors.Errorf("parsing ipfs multiaddr: %w", err) + } ipfsbs, err = ipfsbstore.NewRemoteIpfsBstore(helpers.LifecycleCtx(mctx, lc), ma) } else { ipfsbs, err = ipfsbstore.NewIpfsBstore(helpers.LifecycleCtx(mctx, lc)) From 78e0c96f22f6caf8cc267fdda5f1501a6ce24562 Mon Sep 17 00:00:00 2001 From: Mike Greenberg Date: Wed, 27 May 2020 17:12:45 -0400 Subject: [PATCH 10/11] update systemd services; include make directives and docs for use --- Makefile | 12 ++++++++ documentation/en/install-systemd-services.md | 29 ++++++++++++++++++++ scripts/lotus-daemon.service | 4 +++ 3 files changed, 45 insertions(+) create mode 100644 documentation/en/install-systemd-services.md diff --git a/Makefile b/Makefile index 1af851082..572c19c17 100644 --- a/Makefile +++ b/Makefile @@ -103,6 +103,18 @@ install: install -C ./lotus-storage-miner /usr/local/bin/lotus-storage-miner install -C ./lotus-seal-worker /usr/local/bin/lotus-seal-worker +install-services: + install -C -m 0644 ./scripts/lotus-daemon.service /etc/systemd/system/lotus-daemon.service + install -C -m 0644 ./scripts/lotus-miner.service /etc/systemd/system/lotus-miner.service + systemctl daemon-reload + @echo + @echo "lotus and lotus-miner services installed. Don't forget to 'systemctl enable lotus|lotus-miner' for it to be enabled on startup." + +clean-services: + rm -f /etc/systemd/system/lotus-daemon.service + rm -f /etc/systemd/system/lotus-miner.service + systemctl daemon-reload + # TOOLS lotus-seed: $(BUILD_DEPS) diff --git a/documentation/en/install-systemd-services.md b/documentation/en/install-systemd-services.md new file mode 100644 index 000000000..8c83d56a1 --- /dev/null +++ b/documentation/en/install-systemd-services.md @@ -0,0 +1,29 @@ +# Lotus and Miner as a systemd service + +Lotus is capable of running as a systemd service daemon. You can find installable service files for systemd in the [lotus repo](https://github.com/filecoin-project/lotus/tree/master/scripts) as files with `.service` extension. In order to install these service files, you can copy these `.service` files to the default systemd service path. + +## Installing via `make` + +NOTE: Before using lotus and lotus-miner as systemd services, don't forget to `sudo make install` to ensure the binaries are accessible by the root user. + +If your host uses the default systemd service path, it can be installed with `sudo make install-services`: + +```sh +$ sudo make install-services +``` + +## Interacting with service logs + +Logs from the services can be reviewed using `journalctl`. + +### Follow logs from a specific service unit + +```sh +$ sudo journalctl -u lotus-daemon -f +``` + +### View logs in reverse order + +```sh +$ sudo journalctl -u lotus-miner -r +``` diff --git a/scripts/lotus-daemon.service b/scripts/lotus-daemon.service index 44b2058b0..86f0dff27 100644 --- a/scripts/lotus-daemon.service +++ b/scripts/lotus-daemon.service @@ -6,6 +6,10 @@ After=network.target ExecStart=/usr/local/bin/lotus daemon Environment=GOLOG_FILE="/var/log/lotus-daemon" Environment=GOLOG_LOG_FMT="json" +MemoryAccounting=true +MemoryHigh=8G +MemoryMax=10G +LimitNOFILE=8192:10240 [Install] WantedBy=multiuser.target From b807bdbea07cebdb41c1e78e5ffbf7714f80880f Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 28 May 2020 08:45:34 -0700 Subject: [PATCH 11/11] check neq zero exit code --- storage/wdpost_run.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index cfee3cde3..63109e5f5 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -4,9 +4,10 @@ import ( "bytes" "context" "errors" - "github.com/filecoin-project/go-bitfield" "time" + "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/builtin" @@ -171,7 +172,7 @@ func (s *WindowPoStScheduler) checkRecoveries(ctx context.Context, deadline uint return xerrors.Errorf("declare faults recovered wait error: %w", err) } - if rec.Receipt.ExitCode == 0 { + if rec.Receipt.ExitCode != 0 { return xerrors.Errorf("declare faults recovered wait non-0 exit code: %d", rec.Receipt.ExitCode) }