From a8915f0331c08e3d3bca74e8db5b1950fb518532 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 3 Jun 2020 08:45:53 -0700 Subject: [PATCH 01/15] add a doc on amd vs intel cpus --- documentation/en/sealing-procs.md | 70 +++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 documentation/en/sealing-procs.md diff --git a/documentation/en/sealing-procs.md b/documentation/en/sealing-procs.md new file mode 100644 index 000000000..1b4f2877c --- /dev/null +++ b/documentation/en/sealing-procs.md @@ -0,0 +1,70 @@ +# Why does Filecoin mining work best on AMD? +Currently, Filecoin's Proof of Replication (PoRep) prefers to be run on AMD +processors. More accurately, it runs much much slower on Intel CPUs (it runs +competitively fast on some ARM processors, like the ones in newer Samsung +phones, but they lack the RAM to seal the larger sector sizes). The main reason +that we see this benefit on AMD processors is due to their implementation of +the SHA hardware instructions. Now, why do we use the SHA instruction? + +## PoRep security assumptions +Our research team has two different models for the security of Proofs of +Replication. These are the Latency Assumption, and the Cost Assumption. These +assumptions are arguments for why an attacker cannot pull off a 'regeneration +attack'. That is, the attacker cannot seal and commit random data (generated by +a function), delete it, and then reseal it on the fly to respond to PoSt +challenges, without actually storing the data for that time period. + +### Cost Assumptions +The cost assumption states that the real money cost (hardware, electricity, +etc) of generating a sector is higher than the real money cost of simply +storing it on disks. NSE is a new PoRep our research team is working on that is +based on the cost assumption, and is thus able to be very parallelizable (In +comparison to schemes based on a latency assumption, as will be explained +next). However, cost assumptions vary greatly with available and hypothetical +hardware. For example, someone making an ASIC for NSE could break the cost +assumption by lowering the cost of sealing too much. This is one of our main +hesitations around shipping NSE. + +### Latency Assumptions +A Proof of Replication that is secure under a latency assumption is secure +because an attacker cannot regenerate the data in time. We use this assumption +for SDR, where we assume that an attacker cannot regenerate enough of a sector +fast enough to respond to a PoSt. The way we achieve this is through the use +of depth-robust graphs. Without going into too much detail, depth-robust +graphs guarantee a minimum number of serial operations to compute an encoding +based on the graph. Each edge in the graph represents an operation we need to +perform. We thus have a guarantee that someone has to perform some operation +N times in a row in order to compute the encoding. That means that the +computation of the encoding must take at least as long as N times the fastest +someone can do that operation. + +Now, to make this secure, we need to choose an operation that can't be made +much faster. There are many potential candidates here, depending on what +hardware you want to require. We opted not to require ASICs in order to mine +Filecoin, so that limits our choices severely. We have to look at what +operations CPUs are really good at. One candidate was AES encryption, which +also has hardware instructions. However, the difference between the performance +of CPU AES instructions, and the hypothetical 'best' performance you get was +still too great. This gap is generally called 'Amax', an attacker’s maximum +advantage. The higher the Amax of an algorithm we choose, the more expensive +the overall process has to become in order to bound how fast the attacker could +do it. +As we were doing our research, we noticed that AMD shipped their new processors +with a builtin SHA function, and we looked into how fast someone could possibly +compute a SHA hash. We found that AMD’s implementation is only around 3 times +slower than anyone could reasonably do (given estimates by the hardware +engineers at [Supranational](https://www.supranational.net/) ). This is +incredibly impressive for something you can get in consumer hardware. With +this, we were able to make SDR sealing reasonably performant for people with +off-the-shelf hardware. + +## Super Optimized CPUs + +Given all of the above, with a latency assumption that we're basing our proofs +on right now, you need a processor that can do iterated SHA hashes really fast. +As mentioned earlier, this isn’t just AMD processors, but many ARM processors +also have support for this. Hopefully, new Intel processors also follow suit. +But for now, Filecoin works best on AMD processors. + + + From c01f70105fda789a060343cf0eb4541e424f09bd Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 3 Jun 2020 17:14:36 -0700 Subject: [PATCH 02/15] implement chain index to make lookbacks faster --- chain/store/index.go | 144 +++++++++++++++++++++++++++++++++++++++++++ chain/store/store.go | 32 +++++----- 2 files changed, 159 insertions(+), 17 deletions(-) create mode 100644 chain/store/index.go diff --git a/chain/store/index.go b/chain/store/index.go new file mode 100644 index 000000000..36096e58a --- /dev/null +++ b/chain/store/index.go @@ -0,0 +1,144 @@ +package store + +import ( + "context" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/abi" + lru "github.com/hashicorp/golang-lru" + "golang.org/x/xerrors" +) + +type ChainIndex struct { + skipCache *lru.ARCCache + + loadTipSet loadTipSetFunc + + skipLength abi.ChainEpoch +} +type loadTipSetFunc func(types.TipSetKey) (*types.TipSet, error) + +func NewChainIndex(lts loadTipSetFunc) *ChainIndex { + sc, _ := lru.NewARC(8192) + return &ChainIndex{ + skipCache: sc, + loadTipSet: lts, + skipLength: 20, + } +} + +type lbEntry struct { + ts *types.TipSet + parentHeight abi.ChainEpoch + target types.TipSetKey +} + +func (ci *ChainIndex) GetTipsetByHeight(ctx context.Context, from *types.TipSet, to abi.ChainEpoch) (*types.TipSet, error) { + if from.Height()-to <= ci.skipLength { + return ci.walkBack(from, to) + } + + rounded, err := ci.roundDown(from) + if err != nil { + return nil, err + } + + cur := rounded.Key() + for { + cval, ok := ci.skipCache.Get(cur) + if !ok { + fc, err := ci.fillCache(cur) + if err != nil { + return nil, err + } + cval = fc + } + + lbe := cval.(*lbEntry) + if lbe.ts.Height() == to || lbe.parentHeight < to { + return lbe.ts, nil + } else if to > lbe.ts.Height()-ci.skipLength { + return ci.walkBack(from, to) + } + + cur = lbe.target + } +} + +func (ci *ChainIndex) fillCache(tsk types.TipSetKey) (*lbEntry, error) { + ts, err := ci.loadTipSet(tsk) + if err != nil { + return nil, err + } + + // will either be equal to ts.Height, or at least > ts.Parent.Height() + rheight := ci.roundHeight(ts.Height()) + + parent, err := ci.loadTipSet(ts.Parents()) + if err != nil { + return nil, err + } + + if parent.Height() > rheight { + return nil, xerrors.Errorf("cache is inconsistent") + } + + rheight -= ci.skipLength + + skipTarget, err := ci.walkBack(parent, rheight) + if err != nil { + return nil, err + } + + lbe := &lbEntry{ + ts: ts, + parentHeight: parent.Height(), + target: skipTarget.Key(), + } + ci.skipCache.Add(tsk, lbe) + + return lbe, nil +} + +func (ci *ChainIndex) roundHeight(h abi.ChainEpoch) abi.ChainEpoch { + return abi.ChainEpoch(h/ci.skipLength) * ci.skipLength +} + +func (ci *ChainIndex) roundDown(ts *types.TipSet) (*types.TipSet, error) { + target := ci.roundHeight(ts.Height()) + + rounded, err := ci.walkBack(ts, target) + if err != nil { + return nil, err + } + + return rounded, nil +} + +func (ci *ChainIndex) walkBack(from *types.TipSet, to abi.ChainEpoch) (*types.TipSet, error) { + if to > from.Height() { + return nil, xerrors.Errorf("looking for tipset with height greater than start point") + } + + if to == from.Height() { + return from, nil + } + + ts := from + + for { + pts, err := ci.loadTipSet(ts.Parents()) + if err != nil { + return nil, err + } + + if to > pts.Height() { + return ts, nil + } + if to == pts.Height() { + return pts, nil + } + + ts = pts + } +} diff --git a/chain/store/store.go b/chain/store/store.go index 1d0668032..f5a6ee0f6 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -62,6 +62,8 @@ type ChainStore struct { tstLk sync.Mutex tipsets map[abi.ChainEpoch][]cid.Cid + cindex *ChainIndex + reorgCh chan<- reorg headChangeNotifs []func(rev, app []*types.TipSet) error @@ -84,6 +86,10 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls runtime.Sys vmcalls: vmcalls, } + ci := NewChainIndex(cs.LoadTipSet) + + cs.cindex = ci + cs.reorgCh = cs.reorgWorker(context.TODO()) hcnf := func(rev, app []*types.TipSet) error { @@ -951,24 +957,16 @@ func (cs *ChainStore) GetTipsetByHeight(ctx context.Context, h abi.ChainEpoch, t log.Warnf("expensive call to GetTipsetByHeight, seeking %d levels", ts.Height()-h) } - for { - pts, err := cs.LoadTipSet(ts.Parents()) - if err != nil { - return nil, err - } - - if h > pts.Height() { - if prev { - return pts, nil - } - return ts, nil - } - if h == pts.Height() { - return pts, nil - } - - ts = pts + lbts, err := cs.cindex.GetTipsetByHeight(ctx, ts, h) + if err != nil { + return nil, err } + + if lbts.Height() == h || !prev { + return lbts, nil + } + + return cs.LoadTipSet(lbts.Parents()) } func recurseLinks(bs blockstore.Blockstore, root cid.Cid, in []cid.Cid) ([]cid.Cid, error) { From 86083531c62e1fb93346a7582d45dbf4597e5494 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 3 Jun 2020 18:25:41 -0700 Subject: [PATCH 03/15] a couple bugfixes --- chain/store/index.go | 9 ++++++++- chain/store/store.go | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/chain/store/index.go b/chain/store/index.go index 36096e58a..7d642eb10 100644 --- a/chain/store/index.go +++ b/chain/store/index.go @@ -58,7 +58,7 @@ func (ci *ChainIndex) GetTipsetByHeight(ctx context.Context, from *types.TipSet, if lbe.ts.Height() == to || lbe.parentHeight < to { return lbe.ts, nil } else if to > lbe.ts.Height()-ci.skipLength { - return ci.walkBack(from, to) + return ci.walkBack(lbe.ts, to) } cur = lbe.target @@ -71,6 +71,13 @@ func (ci *ChainIndex) fillCache(tsk types.TipSetKey) (*lbEntry, error) { return nil, err } + if ts.Height() == 0 { + return &lbEntry{ + ts: ts, + parentHeight: 0, + }, nil + } + // will either be equal to ts.Height, or at least > ts.Parent.Height() rheight := ci.roundHeight(ts.Height()) diff --git a/chain/store/store.go b/chain/store/store.go index f5a6ee0f6..dcd1a5ead 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -946,7 +946,7 @@ func (cs *ChainStore) GetTipsetByHeight(ctx context.Context, h abi.ChainEpoch, t } if h > ts.Height() { - return nil, xerrors.Errorf("looking for tipset with height less than start point") + return nil, xerrors.Errorf("looking for tipset with height greater than start point") } if h == ts.Height() { From 656b285195008bcd047b90e983c38fa0efed2df5 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 4 Jun 2020 17:56:57 -0400 Subject: [PATCH 04/15] Bugfix: Begin walkback when lookback target's height is too low --- chain/store/index.go | 8 +++++++- chain/store/store.go | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/chain/store/index.go b/chain/store/index.go index 7d642eb10..15d5d7025 100644 --- a/chain/store/index.go +++ b/chain/store/index.go @@ -30,6 +30,7 @@ func NewChainIndex(lts loadTipSetFunc) *ChainIndex { type lbEntry struct { ts *types.TipSet parentHeight abi.ChainEpoch + targetHeight abi.ChainEpoch target types.TipSetKey } @@ -57,7 +58,7 @@ func (ci *ChainIndex) GetTipsetByHeight(ctx context.Context, from *types.TipSet, lbe := cval.(*lbEntry) if lbe.ts.Height() == to || lbe.parentHeight < to { return lbe.ts, nil - } else if to > lbe.ts.Height()-ci.skipLength { + } else if to > lbe.targetHeight { return ci.walkBack(lbe.ts, to) } @@ -65,6 +66,10 @@ func (ci *ChainIndex) GetTipsetByHeight(ctx context.Context, from *types.TipSet, } } +func (ci *ChainIndex) GetTipsetByHeightWithoutCache(from *types.TipSet, to abi.ChainEpoch) (*types.TipSet, error) { + return ci.walkBack(from, to) +} + func (ci *ChainIndex) fillCache(tsk types.TipSetKey) (*lbEntry, error) { ts, err := ci.loadTipSet(tsk) if err != nil { @@ -100,6 +105,7 @@ func (ci *ChainIndex) fillCache(tsk types.TipSetKey) (*lbEntry, error) { lbe := &lbEntry{ ts: ts, parentHeight: parent.Height(), + targetHeight: skipTarget.Height(), target: skipTarget.Key(), } ci.skipCache.Add(tsk, lbe) diff --git a/chain/store/store.go b/chain/store/store.go index dcd1a5ead..94d94da79 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -962,6 +962,14 @@ func (cs *ChainStore) GetTipsetByHeight(ctx context.Context, h abi.ChainEpoch, t return nil, err } + if lbts.Height() < h { + log.Warnf("chain index returned the wrong tipset at height %d, using slow retrieval", h) + lbts, err = cs.cindex.GetTipsetByHeightWithoutCache(ts, h) + if err != nil { + return nil, err + } + } + if lbts.Height() == h || !prev { return lbts, nil } From 4855045ea2abd84fc78a76d60fae6eb7fa435876 Mon Sep 17 00:00:00 2001 From: laser Date: Fri, 5 Jun 2020 10:19:33 -0700 Subject: [PATCH 05/15] pass context to paramfetch.GetParams Fixes #1209 --- cli/params.go | 3 ++- cmd/lotus-bench/main.go | 5 +++-- cmd/lotus-seal-worker/main.go | 4 ++-- cmd/lotus-shed/params.go | 3 ++- cmd/lotus-storage-miner/init.go | 6 ++++-- cmd/lotus/daemon.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- node/modules/storageminer.go | 2 +- 9 files changed, 18 insertions(+), 13 deletions(-) diff --git a/cli/params.go b/cli/params.go index 518add665..95596ff57 100644 --- a/cli/params.go +++ b/cli/params.go @@ -24,7 +24,8 @@ var fetchParamCmd = &cli.Command{ return err } sectorSize := uint64(sectorSizeInt) - err = paramfetch.GetParams(build.ParametersJson(), sectorSize) + + err = paramfetch.GetParams(ReqContext(cctx), build.ParametersJson(), sectorSize) if err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 23471500d..dad92c55d 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -20,6 +20,7 @@ import ( "github.com/filecoin-project/go-address" paramfetch "github.com/filecoin-project/go-paramfetch" + lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/sector-storage/ffiwrapper" "github.com/filecoin-project/sector-storage/ffiwrapper/basicfs" "github.com/filecoin-project/sector-storage/stores" @@ -209,7 +210,7 @@ var sealBenchCmd = &cli.Command{ // Only fetch parameters if actually needed if !c.Bool("skip-commit2") { - if err := paramfetch.GetParams(build.ParametersJson(), uint64(sectorSize)); err != nil { + if err := paramfetch.GetParams(lcli.ReqContext(c), build.ParametersJson(), uint64(sectorSize)); err != nil { return xerrors.Errorf("getting params: %w", err) } } @@ -616,7 +617,7 @@ var proveCmd = &cli.Command{ return xerrors.Errorf("unmarshalling input file: %w", err) } - if err := paramfetch.GetParams(build.ParametersJson(), c2in.SectorSize); err != nil { + if err := paramfetch.GetParams(lcli.ReqContext(c), build.ParametersJson(), c2in.SectorSize); err != nil { return xerrors.Errorf("getting params: %w", err) } diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index 8271656d0..6f9d94da1 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -26,7 +26,7 @@ import ( lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/lotuslog" "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/sector-storage" + sectorstorage "github.com/filecoin-project/sector-storage" "github.com/filecoin-project/sector-storage/sealtasks" "github.com/filecoin-project/sector-storage/stores" ) @@ -146,7 +146,7 @@ var runCmd = &cli.Command{ } if cctx.Bool("commit") { - if err := paramfetch.GetParams(build.ParametersJson(), uint64(ssize)); err != nil { + if err := paramfetch.GetParams(ctx, build.ParametersJson(), uint64(ssize)); err != nil { return xerrors.Errorf("get params: %w", err) } } diff --git a/cmd/lotus-shed/params.go b/cmd/lotus-shed/params.go index ff00f2b4f..cc271d112 100644 --- a/cmd/lotus-shed/params.go +++ b/cmd/lotus-shed/params.go @@ -3,6 +3,7 @@ package main import ( "github.com/docker/go-units" paramfetch "github.com/filecoin-project/go-paramfetch" + lcli "github.com/filecoin-project/lotus/cli" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" @@ -24,7 +25,7 @@ var fetchParamCmd = &cli.Command{ return err } sectorSize := uint64(sectorSizeInt) - err = paramfetch.GetParams(build.ParametersJson(), sectorSize) + err = paramfetch.GetParams(lcli.ReqContext(cctx), build.ParametersJson(), sectorSize) if err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index f834a09a5..bee9858e9 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -126,8 +126,11 @@ var initCmd = &cli.Command{ log.Info("will attempt to symlink to imported sectors") } + ctx := lcli.ReqContext(cctx) + log.Info("Checking proof parameters") - if err := paramfetch.GetParams(build.ParametersJson(), uint64(ssize)); err != nil { + + if err := paramfetch.GetParams(ctx, build.ParametersJson(), uint64(ssize)); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } @@ -138,7 +141,6 @@ var initCmd = &cli.Command{ return err } defer closer() - ctx := lcli.ReqContext(cctx) log.Info("Checking full node sync status") diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 6783ef0b1..51a1fd37f 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -155,7 +155,7 @@ var DaemonCmd = &cli.Command{ return xerrors.Errorf("repo init error: %w", err) } - if err := paramfetch.GetParams(build.ParametersJson(), 0); err != nil { + if err := paramfetch.GetParams(lcli.ReqContext(cctx), build.ParametersJson(), 0); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/go.mod b/go.mod index 0ae19cc93..89818fce5 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/filecoin-project/go-fil-markets v0.2.7 github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 - github.com/filecoin-project/go-paramfetch v0.0.2-0.20200505180321-973f8949ea8e + github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/sector-storage v0.0.0-20200522011946-a59ca7536a95 diff --git a/go.sum b/go.sum index c1236d113..4c4ddf12f 100644 --- a/go.sum +++ b/go.sum @@ -177,8 +177,8 @@ github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:9 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6/go.mod h1:0HgYnrkeSU4lu1p+LEOeDpFsNBssa0OGGriWdA4hvaE= github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= -github.com/filecoin-project/go-paramfetch v0.0.2-0.20200505180321-973f8949ea8e h1:R+HNoQWirMBOhQC+L1OpYUVbvMjB+jq1hx5LmLFvNfA= -github.com/filecoin-project/go-paramfetch v0.0.2-0.20200505180321-973f8949ea8e/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= +github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca h1:OGykrCr6mSn/ckk2IFbIlkc76nsgEs7tSLhZXQt7+z4= +github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 h1:k9qVR9ItcziSB2rxtlkN/MDWNlbsI6yzec+zjUatLW0= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 47a00eb38..0dc93e395 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -72,7 +72,7 @@ func GetParams(sbc *ffiwrapper.Config) error { return err } - if err := paramfetch.GetParams(build.ParametersJson(), uint64(ssize)); err != nil { + if err := paramfetch.GetParams(context.TODO(), build.ParametersJson(), uint64(ssize)); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } From 76b5c8b145cea0c0b4f3849938c3411848dde9f7 Mon Sep 17 00:00:00 2001 From: shannonwells Date: Fri, 5 Jun 2020 10:59:28 -0700 Subject: [PATCH 06/15] fix unhandled nil return --- paychmgr/simple.go | 1 + 1 file changed, 1 insertion(+) diff --git a/paychmgr/simple.go b/paychmgr/simple.go index ff537d425..65ab856e1 100644 --- a/paychmgr/simple.go +++ b/paychmgr/simple.go @@ -58,6 +58,7 @@ func (pm *Manager) waitForPaychCreateMsg(ctx context.Context, mcid cid.Cid) { mwait, err := pm.state.StateWaitMsg(ctx, mcid) if err != nil { log.Errorf("wait msg: %w", err) + return } if mwait.Receipt.ExitCode != 0 { From e4bed4d9636e51edb229fc2b981f10f39c95ee02 Mon Sep 17 00:00:00 2001 From: shannonwells Date: Fri, 5 Jun 2020 11:06:42 -0700 Subject: [PATCH 07/15] return on all err conditions --- paychmgr/simple.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/paychmgr/simple.go b/paychmgr/simple.go index 65ab856e1..24a662a50 100644 --- a/paychmgr/simple.go +++ b/paychmgr/simple.go @@ -63,18 +63,21 @@ func (pm *Manager) waitForPaychCreateMsg(ctx context.Context, mcid cid.Cid) { if mwait.Receipt.ExitCode != 0 { log.Errorf("payment channel creation failed (exit code %d)", mwait.Receipt.ExitCode) + return } var decodedReturn init_.ExecReturn err = decodedReturn.UnmarshalCBOR(bytes.NewReader(mwait.Receipt.Return)) if err != nil { log.Error(err) + return } paychaddr := decodedReturn.RobustAddress ci, err := pm.loadOutboundChannelInfo(ctx, paychaddr) if err != nil { log.Errorf("loading channel info: %w", err) + return } if err := pm.store.trackChannel(ci); err != nil { From 2347ba6f39e37d1523cd36ddb80bad7d747bef67 Mon Sep 17 00:00:00 2001 From: laser Date: Fri, 5 Jun 2020 13:04:28 -0700 Subject: [PATCH 08/15] print "client"-command CIDs as base32-encoded strs Fixes #916 --- cli/client.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/cli/client.go b/cli/client.go index 3bfc5de70..5c8e7a957 100644 --- a/cli/client.go +++ b/cli/client.go @@ -9,6 +9,7 @@ import ( "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" + "github.com/multiformats/go-multibase" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" @@ -67,7 +68,14 @@ var clientImportCmd = &cli.Command{ if err != nil { return err } - fmt.Println(c.String()) + + encoded, err := c.StringOfBase(multibase.Base32) + if err != nil { + return err + } + + fmt.Println(encoded) + return nil }, } @@ -150,7 +158,12 @@ var clientLocalCmd = &cli.Command{ return err } for _, v := range list { - fmt.Printf("%s %s %d %s\n", v.Key, v.FilePath, v.Size, v.Status) + encoded, err := v.Key.StringOfBase(multibase.Base32) + if err != nil { + return err + } + + fmt.Printf("%s %s %d %s\n", encoded, v.FilePath, v.Size, v.Status) } return nil }, From b3eef76cddc53005b0e4c36eb05feab7a42b7a96 Mon Sep 17 00:00:00 2001 From: laser Date: Fri, 5 Jun 2020 13:13:16 -0700 Subject: [PATCH 09/15] client commP outputs Base32-encoded string --- cli/client.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cli/client.go b/cli/client.go index 5c8e7a957..1750aa7bb 100644 --- a/cli/client.go +++ b/cli/client.go @@ -102,11 +102,16 @@ var clientCommPCmd = &cli.Command{ } ret, err := api.ClientCalcCommP(ctx, cctx.Args().Get(0), miner) - if err != nil { return err } - fmt.Println("CID: ", ret.Root) + + encoded, err := ret.Root.StringOfBase(multibase.Base32) + if err != nil { + return err + } + + fmt.Println("CID: ", encoded) fmt.Println("Piece size: ", ret.Size) return nil }, From 55a9c9ae0a119991c5e70333540ad216177408b2 Mon Sep 17 00:00:00 2001 From: laser Date: Fri, 5 Jun 2020 13:13:33 -0700 Subject: [PATCH 10/15] client deal command outputs Base32-encoded CID --- cli/client.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cli/client.go b/cli/client.go index 1750aa7bb..afd133523 100644 --- a/cli/client.go +++ b/cli/client.go @@ -281,7 +281,13 @@ var clientDealCmd = &cli.Command{ return err } - fmt.Println(proposal) + encoded, err := proposal.StringOfBase(multibase.Base32) + if err != nil { + return err + } + + fmt.Println(encoded) + return nil }, } From 28e454fd4aeaf14d1d586bc8b668defaa8b1baba Mon Sep 17 00:00:00 2001 From: laser Date: Fri, 5 Jun 2020 13:13:53 -0700 Subject: [PATCH 11/15] go mod tidy --- go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/go.mod b/go.mod index 89818fce5..d83e6a845 100644 --- a/go.mod +++ b/go.mod @@ -95,6 +95,7 @@ require ( github.com/multiformats/go-multiaddr v0.2.2 github.com/multiformats/go-multiaddr-dns v0.2.0 github.com/multiformats/go-multiaddr-net v0.1.5 + github.com/multiformats/go-multibase v0.0.2 github.com/multiformats/go-multihash v0.0.13 github.com/opentracing/opentracing-go v1.1.0 github.com/stretchr/objx v0.2.0 // indirect From b167e73acb0c0aeb0f88e90b21f8747ae432dd7d Mon Sep 17 00:00:00 2001 From: laser Date: Fri, 5 Jun 2020 13:33:45 -0700 Subject: [PATCH 12/15] allow CLI consumer to specify CID encoding for output --- cli/client.go | 26 +++++++++++++------------- cli/cmd.go | 21 +++++++++++++++++++++ cmd/lotus/main.go | 7 +++++++ go.mod | 1 + go.sum | 2 ++ 5 files changed, 44 insertions(+), 13 deletions(-) diff --git a/cli/client.go b/cli/client.go index afd133523..49002d67e 100644 --- a/cli/client.go +++ b/cli/client.go @@ -9,7 +9,6 @@ import ( "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" - "github.com/multiformats/go-multibase" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" @@ -69,12 +68,12 @@ var clientImportCmd = &cli.Command{ return err } - encoded, err := c.StringOfBase(multibase.Base32) + encoder, err := GetCidEncoder(cctx) if err != nil { return err } - fmt.Println(encoded) + fmt.Println(encoder.Encode(c)) return nil }, @@ -106,12 +105,12 @@ var clientCommPCmd = &cli.Command{ return err } - encoded, err := ret.Root.StringOfBase(multibase.Base32) + encoder, err := GetCidEncoder(cctx) if err != nil { return err } - fmt.Println("CID: ", encoded) + fmt.Println("CID: ", encoder.Encode(ret.Root)) fmt.Println("Piece size: ", ret.Size) return nil }, @@ -162,13 +161,14 @@ var clientLocalCmd = &cli.Command{ if err != nil { return err } - for _, v := range list { - encoded, err := v.Key.StringOfBase(multibase.Base32) - if err != nil { - return err - } - fmt.Printf("%s %s %d %s\n", encoded, v.FilePath, v.Size, v.Status) + encoder, err := GetCidEncoder(cctx) + if err != nil { + return err + } + + for _, v := range list { + fmt.Printf("%s %s %d %s\n", encoder.Encode(v.Key), v.FilePath, v.Size, v.Status) } return nil }, @@ -281,12 +281,12 @@ var clientDealCmd = &cli.Command{ return err } - encoded, err := proposal.StringOfBase(multibase.Base32) + encoder, err := GetCidEncoder(cctx) if err != nil { return err } - fmt.Println(encoded) + fmt.Println(encoder.Encode(*proposal)) return nil }, diff --git a/cli/cmd.go b/cli/cmd.go index fd0ab1e78..2d4ff7a75 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -9,6 +9,9 @@ import ( "strings" "syscall" + "github.com/multiformats/go-multibase" + + cidenc "github.com/ipfs/go-cidutil/cidenc" logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" @@ -193,6 +196,24 @@ func DaemonContext(cctx *cli.Context) context.Context { return context.Background() } +// GetCidEncoder returns an encoder using the cid-base flag if provided, or +// the default (Base32) encoder if not. +func GetCidEncoder(cctx *cli.Context) (cidenc.Encoder, error) { + val := cctx.String("cid-base") + + e := cidenc.Default() + + if val != "" { + var err error + e.Base, err = multibase.EncoderByName(val) + if err != nil { + return e, err + } + } + + return e, nil +} + // ReqContext returns context for cli execution. Calling it for the first time // installs SIGTERM handler that will close returned context. // Not safe for concurrent execution. diff --git a/cmd/lotus/main.go b/cmd/lotus/main.go index 33ff56ac3..22d121364 100644 --- a/cmd/lotus/main.go +++ b/cmd/lotus/main.go @@ -61,6 +61,13 @@ func main() { Hidden: true, Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME }, + &cli.StringFlag{ + Name: "cid-base", + Hidden: true, + Value: "base32", + Usage: "Multibase encoding used for version 1 CIDs in output.", + DefaultText: "base32", + }, }, Commands: append(local, lcli.Commands...), diff --git a/go.mod b/go.mod index d83e6a845..a41fcec7d 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.3 github.com/ipfs/go-cid v0.0.6-0.20200501230655-7c82f3b81c00 + github.com/ipfs/go-cidutil v0.0.2 github.com/ipfs/go-datastore v0.4.4 github.com/ipfs/go-ds-badger2 v0.1.0 github.com/ipfs/go-filestore v1.0.0 diff --git a/go.sum b/go.sum index 4c4ddf12f..f78d58bd7 100644 --- a/go.sum +++ b/go.sum @@ -345,6 +345,8 @@ github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/ipfs/go-cid v0.0.6-0.20200501230655-7c82f3b81c00 h1:QN88Q0kT2QiDaLxpR/SDsqOBtNIEF/F3n96gSDUimkA= github.com/ipfs/go-cid v0.0.6-0.20200501230655-7c82f3b81c00/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= +github.com/ipfs/go-cidutil v0.0.2 h1:CNOboQf1t7Qp0nuNh8QMmhJs0+Q//bRL1axtCnIB1Yo= +github.com/ipfs/go-cidutil v0.0.2/go.mod h1:ewllrvrxG6AMYStla3GD7Cqn+XYSLqjK0vc+086tB6s= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= From 963e4dd2fc2da19f01ec55681711a16e8720e83e Mon Sep 17 00:00:00 2001 From: laser Date: Fri, 5 Jun 2020 13:52:36 -0700 Subject: [PATCH 13/15] set a Base32 default explicitly --- cli/cmd.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/cmd.go b/cli/cmd.go index 2d4ff7a75..82b75801c 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -196,12 +196,12 @@ func DaemonContext(cctx *cli.Context) context.Context { return context.Background() } -// GetCidEncoder returns an encoder using the cid-base flag if provided, or +// GetCidEncoder returns an encoder using the `cid-base` flag if provided, or // the default (Base32) encoder if not. func GetCidEncoder(cctx *cli.Context) (cidenc.Encoder, error) { val := cctx.String("cid-base") - e := cidenc.Default() + e := cidenc.Encoder{Base: multibase.MustNewEncoder(multibase.Base32)} if val != "" { var err error From fb09d1ad2b4b365c13efb81a22767e0d05fcab39 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 5 Jun 2020 14:13:08 -0700 Subject: [PATCH 14/15] update to new hamt code with improved perf --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 89818fce5..b594dd8d0 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.1 github.com/ipfs/go-graphsync v0.0.6-0.20200504202014-9d5f2c26a103 - github.com/ipfs/go-hamt-ipld v0.1.1-0.20200501020327-d53d20a7063e + github.com/ipfs/go-hamt-ipld v0.1.1-0.20200605182717-0310ad2b0b1f github.com/ipfs/go-ipfs-blockstore v1.0.0 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-ds-help v1.0.0 diff --git a/go.sum b/go.sum index 4c4ddf12f..c05655afa 100644 --- a/go.sum +++ b/go.sum @@ -377,6 +377,8 @@ github.com/ipfs/go-hamt-ipld v0.0.15-0.20200131012125-dd88a59d3f2e/go.mod h1:9aQ github.com/ipfs/go-hamt-ipld v0.0.15-0.20200204200533-99b8553ef242/go.mod h1:kq3Pi+UP3oHhAdKexE+kHHYRKMoFNuGero0R7q3hWGg= github.com/ipfs/go-hamt-ipld v0.1.1-0.20200501020327-d53d20a7063e h1:Klv6s+kbuhh0JVpGFmFK2t6AtZxJfAnVneQHh1DlFOo= github.com/ipfs/go-hamt-ipld v0.1.1-0.20200501020327-d53d20a7063e/go.mod h1:giiPqWYCnRBYpNTsJ/EX1ojldX5kTXrXYckSJQ7ko9M= +github.com/ipfs/go-hamt-ipld v0.1.1-0.20200605182717-0310ad2b0b1f h1:mchhWiYYUSoCuE3wDfRCo8cho5kqSoxkgnOtGcnNMZw= +github.com/ipfs/go-hamt-ipld v0.1.1-0.20200605182717-0310ad2b0b1f/go.mod h1:phOFBB7W73N9dg1glcb1fQ9HtQFDUpeyJgatW8ns0bw= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ= From a80dce2e3ba990cad549b9a6fb7511f50b4e353d Mon Sep 17 00:00:00 2001 From: laser Date: Fri, 5 Jun 2020 14:22:29 -0700 Subject: [PATCH 15/15] move to client command --- cli/client.go | 36 ++++++++++++++++++++++++++++++++++++ cli/cmd.go | 21 --------------------- cmd/lotus/main.go | 7 ------- 3 files changed, 36 insertions(+), 28 deletions(-) diff --git a/cli/client.go b/cli/client.go index 49002d67e..9787df37e 100644 --- a/cli/client.go +++ b/cli/client.go @@ -8,7 +8,9 @@ import ( "text/tabwriter" "github.com/ipfs/go-cid" + "github.com/ipfs/go-cidutil/cidenc" "github.com/libp2p/go-libp2p-core/peer" + "github.com/multiformats/go-multibase" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" @@ -21,6 +23,32 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +var CidBaseFlag = cli.StringFlag{ + Name: "cid-base", + Hidden: true, + Value: "base32", + Usage: "Multibase encoding used for version 1 CIDs in output.", + DefaultText: "base32", +} + +// GetCidEncoder returns an encoder using the `cid-base` flag if provided, or +// the default (Base32) encoder if not. +func GetCidEncoder(cctx *cli.Context) (cidenc.Encoder, error) { + val := cctx.String("cid-base") + + e := cidenc.Encoder{Base: multibase.MustNewEncoder(multibase.Base32)} + + if val != "" { + var err error + e.Base, err = multibase.EncoderByName(val) + if err != nil { + return e, err + } + } + + return e, nil +} + var clientCmd = &cli.Command{ Name: "client", Usage: "Make deals, store data, retrieve data", @@ -46,6 +74,7 @@ var clientImportCmd = &cli.Command{ Name: "car", Usage: "import from a car file instead of a regular file", }, + &CidBaseFlag, }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -83,6 +112,9 @@ var clientCommPCmd = &cli.Command{ Name: "commP", Usage: "calculate the piece-cid (commP) of a CAR file", ArgsUsage: "[inputFile minerAddress]", + Flags: []cli.Flag{ + &CidBaseFlag, + }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) if err != nil { @@ -149,6 +181,9 @@ var clientCarGenCmd = &cli.Command{ var clientLocalCmd = &cli.Command{ Name: "local", Usage: "List locally imported data", + Flags: []cli.Flag{ + &CidBaseFlag, + }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) if err != nil { @@ -196,6 +231,7 @@ var clientDealCmd = &cli.Command{ Usage: "specify the epoch that the deal should start at", Value: -1, }, + &CidBaseFlag, }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) diff --git a/cli/cmd.go b/cli/cmd.go index 82b75801c..fd0ab1e78 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -9,9 +9,6 @@ import ( "strings" "syscall" - "github.com/multiformats/go-multibase" - - cidenc "github.com/ipfs/go-cidutil/cidenc" logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" @@ -196,24 +193,6 @@ func DaemonContext(cctx *cli.Context) context.Context { return context.Background() } -// GetCidEncoder returns an encoder using the `cid-base` flag if provided, or -// the default (Base32) encoder if not. -func GetCidEncoder(cctx *cli.Context) (cidenc.Encoder, error) { - val := cctx.String("cid-base") - - e := cidenc.Encoder{Base: multibase.MustNewEncoder(multibase.Base32)} - - if val != "" { - var err error - e.Base, err = multibase.EncoderByName(val) - if err != nil { - return e, err - } - } - - return e, nil -} - // ReqContext returns context for cli execution. Calling it for the first time // installs SIGTERM handler that will close returned context. // Not safe for concurrent execution. diff --git a/cmd/lotus/main.go b/cmd/lotus/main.go index 22d121364..33ff56ac3 100644 --- a/cmd/lotus/main.go +++ b/cmd/lotus/main.go @@ -61,13 +61,6 @@ func main() { Hidden: true, Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME }, - &cli.StringFlag{ - Name: "cid-base", - Hidden: true, - Value: "base32", - Usage: "Multibase encoding used for version 1 CIDs in output.", - DefaultText: "base32", - }, }, Commands: append(local, lcli.Commands...),