diff --git a/.gitignore b/.gitignore index 67359f1ff..c54df6a7a 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,10 @@ build/.* build/paramfetch.sh /vendor +/blocks.dot +/blocks.svg +/chainwatch +/chainwatch.db *-fuzz.zip /chain/types/work_msg/ diff --git a/Makefile b/Makefile index af80fb0e6..7f1630fef 100644 --- a/Makefile +++ b/Makefile @@ -44,9 +44,12 @@ CLEAN+=build/.update-modules deps: $(BUILD_DEPS) .PHONY: deps +debug: GOFLAGS=-tags=debug +debug: lotus lotus-storage-miner + lotus: $(BUILD_DEPS) rm -f lotus - go build -o lotus ./cmd/lotus + go build $(GOFLAGS) -o lotus ./cmd/lotus go run github.com/GeertJohan/go.rice/rice append --exec lotus -i ./build .PHONY: lotus @@ -54,7 +57,7 @@ CLEAN+=lotus lotus-storage-miner: $(BUILD_DEPS) rm -f lotus-storage-miner - go build -o lotus-storage-miner ./cmd/lotus-storage-miner + go build $(GOFLAGS) -o lotus-storage-miner ./cmd/lotus-storage-miner go run github.com/GeertJohan/go.rice/rice append --exec lotus-storage-miner -i ./build .PHONY: lotus-storage-miner @@ -100,6 +103,12 @@ fountain: go run github.com/GeertJohan/go.rice/rice append --exec fountain -i ./cmd/lotus-fountain .PHONY: fountain +chainwatch: + rm -f chainwatch + go build -o chainwatch ./cmd/lotus-chainwatch + go run github.com/GeertJohan/go.rice/rice append --exec chainwatch -i ./cmd/lotus-chainwatch +.PHONY: chainwatch + stats: rm -f stats go build -o stats ./tools/stats diff --git a/api/api_full.go b/api/api_full.go index a5582b093..fc23cf43b 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -38,12 +38,14 @@ type FullNode interface { // syncer SyncState(context.Context) (*SyncState, error) SyncSubmitBlock(ctx context.Context, blk *types.BlockMsg) error + SyncIncomingBlocks(ctx context.Context) (<-chan *types.BlockHeader, error) // messages MpoolPending(context.Context, *types.TipSet) ([]*types.SignedMessage, error) MpoolPush(context.Context, *types.SignedMessage) error // TODO: remove MpoolPushMessage(context.Context, *types.Message) (*types.SignedMessage, error) // get nonce, sign, push MpoolGetNonce(context.Context, address.Address) (uint64, error) + MpoolSub(context.Context) (<-chan MpoolUpdate, error) // FullNodeStruct @@ -107,6 +109,9 @@ type FullNode interface { StateMarketParticipants(context.Context, *types.TipSet) (map[string]actors.StorageParticipantBalance, error) StateMarketDeals(context.Context, *types.TipSet) (map[string]actors.OnChainDeal, error) StateMarketStorageDeal(context.Context, uint64, *types.TipSet) (*actors.OnChainDeal, error) + StateLookupID(context.Context, address.Address, *types.TipSet) (address.Address, error) + StateChangedActors(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) + StateGetReceipt(context.Context, cid.Cid, *types.TipSet) (*types.MessageReceipt, error) MarketEnsureAvailable(context.Context, address.Address, types.BigInt) error // MarketFreeBalance @@ -271,3 +276,15 @@ const ( StageMessages StageSyncComplete ) + +type MpoolChange int + +const ( + MpoolAdd MpoolChange = iota + MpoolRemove +) + +type MpoolUpdate struct { + Type MpoolChange + Message *types.SignedMessage +} diff --git a/api/struct.go b/api/struct.go index 8ba0bc3b9..1356bff2f 100644 --- a/api/struct.go +++ b/api/struct.go @@ -51,12 +51,15 @@ type FullNodeStruct struct { ChainGetGenesis func(context.Context) (*types.TipSet, error) `perm:"read"` ChainTipSetWeight func(context.Context, *types.TipSet) (types.BigInt, error) `perm:"read"` - SyncState func(context.Context) (*SyncState, error) `perm:"read"` - SyncSubmitBlock func(ctx context.Context, blk *types.BlockMsg) error `perm:"write"` + SyncState func(context.Context) (*SyncState, error) `perm:"read"` + SyncSubmitBlock func(ctx context.Context, blk *types.BlockMsg) error `perm:"write"` + SyncIncomingBlocks func(ctx context.Context) (<-chan *types.BlockHeader, error) `perm:"read"` MpoolPending func(context.Context, *types.TipSet) ([]*types.SignedMessage, error) `perm:"read"` MpoolPush func(context.Context, *types.SignedMessage) error `perm:"write"` MpoolPushMessage func(context.Context, *types.Message) (*types.SignedMessage, error) `perm:"sign"` + MpoolGetNonce func(context.Context, address.Address) (uint64, error) `perm:"read"` + MpoolSub func(context.Context) (<-chan MpoolUpdate, error) `perm:"read"` MinerCreateBlock func(context.Context, address.Address, *types.TipSet, *types.Ticket, *types.EPostProof, []*types.SignedMessage, uint64, uint64) (*types.BlockMsg, error) `perm:"write"` @@ -71,8 +74,6 @@ type FullNodeStruct struct { WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` - MpoolGetNonce func(context.Context, address.Address) (uint64, error) `perm:"read"` - ClientImport func(ctx context.Context, path string) (cid.Cid, error) `perm:"admin"` ClientListImports func(ctx context.Context) ([]Import, error) `perm:"write"` ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` @@ -102,6 +103,9 @@ type FullNodeStruct struct { StateMarketParticipants func(context.Context, *types.TipSet) (map[string]actors.StorageParticipantBalance, error) `perm:"read"` StateMarketDeals func(context.Context, *types.TipSet) (map[string]actors.OnChainDeal, error) `perm:"read"` StateMarketStorageDeal func(context.Context, uint64, *types.TipSet) (*actors.OnChainDeal, error) `perm:"read"` + StateLookupID func(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) `perm:"read"` + StateChangedActors func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) `perm:"read"` + StateGetReceipt func(context.Context, cid.Cid, *types.TipSet) (*types.MessageReceipt, error) `perm:"read"` MarketEnsureAvailable func(context.Context, address.Address, types.BigInt) error `perm:"sign"` @@ -222,6 +226,10 @@ func (c *FullNodeStruct) MpoolPushMessage(ctx context.Context, msg *types.Messag return c.Internal.MpoolPushMessage(ctx, msg) } +func (c *FullNodeStruct) MpoolSub(ctx context.Context) (<-chan MpoolUpdate, error) { + return c.Internal.MpoolSub(ctx) +} + func (c *FullNodeStruct) MinerCreateBlock(ctx context.Context, addr address.Address, base *types.TipSet, ticket *types.Ticket, eproof *types.EPostProof, msgs []*types.SignedMessage, height, ts uint64) (*types.BlockMsg, error) { return c.Internal.MinerCreateBlock(ctx, addr, base, ticket, eproof, msgs, height, ts) } @@ -330,6 +338,10 @@ func (c *FullNodeStruct) SyncSubmitBlock(ctx context.Context, blk *types.BlockMs return c.Internal.SyncSubmitBlock(ctx, blk) } +func (c *FullNodeStruct) SyncIncomingBlocks(ctx context.Context) (<-chan *types.BlockHeader, error) { + return c.Internal.SyncIncomingBlocks(ctx) +} + func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*ChainSectorInfo, error) { return c.Internal.StateMinerSectors(ctx, addr, ts) } @@ -405,6 +417,18 @@ func (c *FullNodeStruct) StateMarketStorageDeal(ctx context.Context, dealid uint return c.Internal.StateMarketStorageDeal(ctx, dealid, ts) } +func (c *FullNodeStruct) StateLookupID(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) { + return c.Internal.StateLookupID(ctx, addr, ts) +} + +func (c *FullNodeStruct) StateChangedActors(ctx context.Context, olnstate cid.Cid, newstate cid.Cid) (map[string]types.Actor, error) { + return c.Internal.StateChangedActors(ctx, olnstate, newstate) +} + +func (c *FullNodeStruct) StateGetReceipt(ctx context.Context, msg cid.Cid, ts *types.TipSet) (*types.MessageReceipt, error) { + return c.Internal.StateGetReceipt(ctx, msg, ts) +} + func (c *FullNodeStruct) MarketEnsureAvailable(ctx context.Context, addr address.Address, amt types.BigInt) error { return c.Internal.MarketEnsureAvailable(ctx, addr, amt) } diff --git a/build/bootstrap.go b/build/bootstrap.go index 9f8dd6883..9d9c3e9f1 100644 --- a/build/bootstrap.go +++ b/build/bootstrap.go @@ -2,10 +2,11 @@ package build import ( "context" - "github.com/filecoin-project/lotus/lib/addrutil" "os" "strings" + "github.com/filecoin-project/lotus/lib/addrutil" + rice "github.com/GeertJohan/go.rice" "github.com/libp2p/go-libp2p-core/peer" ) diff --git a/build/params_debug.go b/build/params_debug.go new file mode 100644 index 000000000..3337c827f --- /dev/null +++ b/build/params_debug.go @@ -0,0 +1,15 @@ +// +build debug + +package build + +import "os" + +// Seconds +const BlockDelay = 2 + +// Blocks +const ProvingPeriodDuration uint64 = 40 + +func init() { + os.Setenv("TRUST_PARAMS", "1") +} diff --git a/build/params_devnet.go b/build/params_devnet.go new file mode 100644 index 000000000..58aa48321 --- /dev/null +++ b/build/params_devnet.go @@ -0,0 +1,9 @@ +// +build !debug + +package build + +// Seconds +const BlockDelay = 12 + +// Blocks +const ProvingPeriodDuration uint64 = 300 diff --git a/build/params.go b/build/params_shared.go similarity index 96% rename from build/params.go rename to build/params_shared.go index d316d8b5e..44b66248d 100644 --- a/build/params.go +++ b/build/params_shared.go @@ -37,9 +37,6 @@ const PaymentChannelClosingDelay = 6 * 60 * 2 // six hours // ///// // Consensus / Network -// Seconds -const BlockDelay = 12 - // Seconds const AllowableClockDrift = BlockDelay * 2 @@ -60,9 +57,6 @@ const WRatioDen = 2 // ///// // Proofs -// Blocks -const ProvingPeriodDuration uint64 = 300 - // PoStChallangeTime sets the window in which post computation should happen // Blocks const PoStChallangeTime = ProvingPeriodDuration - 6 diff --git a/chain/address/address.go b/chain/address/address.go index c2c6c5943..801da6147 100644 --- a/chain/address/address.go +++ b/chain/address/address.go @@ -12,6 +12,7 @@ import ( cbor "github.com/ipfs/go-ipld-cbor" "github.com/minio/blake2b-simd" "github.com/polydawn/refmt/obj/atlas" + "golang.org/x/xerrors" cbg "github.com/whyrusleeping/cbor-gen" ) @@ -147,6 +148,22 @@ func (a Address) Format(f fmt.State, c rune) { } } +func (a *Address) Scan(value interface{}) error { + switch value := value.(type) { + case string: + a1, err := decode(value) + if err != nil { + return err + } + + *a = a1 + + return nil + default: + return xerrors.New("non-string types unsupported") + } +} + // NewIDAddress returns an address using the ID protocol. func NewIDAddress(id uint64) (Address, error) { return newAddress(ID, leb128.FromUInt64(id)) diff --git a/chain/address/constants.go b/chain/address/constants.go index 8fc66c678..9eabb862b 100644 --- a/chain/address/constants.go +++ b/chain/address/constants.go @@ -2,9 +2,9 @@ package address import ( "encoding/base32" + "errors" "github.com/minio/blake2b-simd" - errors "github.com/pkg/errors" ) func init() { diff --git a/chain/deals/client.go b/chain/deals/client.go index 720926274..946e6cb93 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -74,7 +74,12 @@ type clientDealUpdate struct { mut func(*ClientDeal) } -func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, dag dtypes.ClientDAG, dataTransfer dtypes.ClientDataTransfer, discovery *discovery.Local, fm *market.FundMgr, deals dtypes.ClientDealStore, chainapi full.ChainAPI) *Client { +type clientApi struct { + full.ChainAPI + full.StateAPI +} + +func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, dag dtypes.ClientDAG, dataTransfer dtypes.ClientDataTransfer, discovery *discovery.Local, fm *market.FundMgr, deals dtypes.ClientDealStore, chainapi full.ChainAPI, stateapi full.StateAPI) *Client { c := &Client{ sm: sm, chain: chain, @@ -84,7 +89,7 @@ func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w * dag: dag, discovery: discovery, fm: fm, - events: events.NewEvents(context.TODO(), &chainapi), + events: events.NewEvents(context.TODO(), &clientApi{chainapi, stateapi}), deals: deals, conns: map[cid.Cid]inet.Stream{}, diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go index c6438c236..bac7aab47 100644 --- a/chain/deals/client_states.go +++ b/chain/deals/client_states.go @@ -153,7 +153,7 @@ func (c *Client) sealing(ctx context.Context, deal ClientDeal) (func(*ClientDeal return false, true, nil } - called := func(msg *types.Message, ts *types.TipSet, curH uint64) (more bool, err error) { + called := func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (more bool, err error) { defer func() { if err != nil { select { diff --git a/chain/events/events.go b/chain/events/events.go index fb0297ead..80028965b 100644 --- a/chain/events/events.go +++ b/chain/events/events.go @@ -11,6 +11,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" ) @@ -32,6 +33,9 @@ type eventApi interface { ChainNotify(context.Context) (<-chan []*store.HeadChange, error) ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error) ChainGetTipSetByHeight(context.Context, uint64, *types.TipSet) (*types.TipSet, error) + StateGetReceipt(context.Context, cid.Cid, *types.TipSet) (*types.MessageReceipt, error) + + StateGetActor(ctx context.Context, actor address.Address, ts *types.TipSet) (*types.Actor, error) // optional / for CalledMsg } type Events struct { diff --git a/chain/events/events_called.go b/chain/events/events_called.go index 321eca840..00b86bfee 100644 --- a/chain/events/events_called.go +++ b/chain/events/events_called.go @@ -6,7 +6,9 @@ import ( "sync" "github.com/ipfs/go-cid" + "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" ) @@ -23,7 +25,7 @@ type triggerH = uint64 // `ts` is the tipset, in which the `msg` is included. // `curH`-`ts.Height` = `confidence` -type CalledHandler func(msg *types.Message, ts *types.TipSet, curH uint64) (more bool, err error) +type CalledHandler func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (more bool, err error) // CheckFunc is used for atomicity guarantees. If the condition the callbacks // wait for has already happened in tipset `ts` @@ -186,7 +188,13 @@ func (e *calledEvents) applyWithConfidence(ts *types.TipSet) { continue } - more, err := trigger.handle(event.msg, triggerTs, ts.Height()) + rec, err := e.cs.StateGetReceipt(e.ctx, event.msg.Cid(), ts) + if err != nil { + log.Error(err) + return + } + + more, err := trigger.handle(event.msg, rec, triggerTs, ts.Height()) if err != nil { log.Errorf("chain trigger (call %s.%d() @H %d, called @ %d) failed: %s", event.msg.To, event.msg.Method, origH, ts.Height(), err) continue // don't revert failed calls @@ -224,7 +232,7 @@ func (e *calledEvents) applyTimeouts(ts *types.TipSet) { log.Errorf("events: applyTimeouts didn't find tipset for event; wanted %d; current %d", ts.Height()-uint64(trigger.confidence), ts.Height()) } - more, err := trigger.handle(nil, timeoutTs, ts.Height()) + more, err := trigger.handle(nil, nil, timeoutTs, ts.Height()) if err != nil { log.Errorf("chain trigger (call @H %d, called @ %d) failed: %s", timeoutTs.Height(), ts.Height(), err) continue // don't revert failed calls @@ -296,9 +304,10 @@ func (e *calledEvents) Called(check CheckFunc, hnd CalledHandler, rev RevertHand e.lk.Lock() defer e.lk.Unlock() - done, more, err := check(e.tsc.best()) + ts := e.tsc.best() + done, more, err := check(ts) if err != nil { - return err + return xerrors.Errorf("called check error (h: %d): %w", ts.Height(), err) } if done { timeout = NoTimeout @@ -328,3 +337,7 @@ func (e *calledEvents) Called(check CheckFunc, hnd CalledHandler, rev RevertHand return nil } + +func (e *calledEvents) CalledMsg(ctx context.Context, hnd CalledHandler, rev RevertHandler, confidence int, timeout uint64, msg store.ChainMsg) error { + return e.Called(e.CheckMsg(ctx, msg, hnd), hnd, rev, confidence, timeout, e.MatchMsg(msg.VMMessage())) +} diff --git a/chain/events/events_height.go b/chain/events/events_height.go index a05a3e1be..e7a81d735 100644 --- a/chain/events/events_height.go +++ b/chain/events/events_height.go @@ -96,7 +96,7 @@ func (e *heightEvents) headChangeAt(rev, app []*types.TipSet) error { span.End() if err != nil { - log.Errorf("chain trigger (@H %d, called @ %d) failed: %s", triggerH, ts.Height(), err) + log.Errorf("chain trigger (@H %d, called @ %d) failed: %+v", triggerH, ts.Height(), err) } } return nil diff --git a/chain/events/events_test.go b/chain/events/events_test.go index 6ceeb8db0..927653357 100644 --- a/chain/events/events_test.go +++ b/chain/events/events_test.go @@ -40,6 +40,14 @@ type fakeCS struct { sub func(rev, app []*types.TipSet) } +func (fcs *fakeCS) StateGetReceipt(context.Context, cid.Cid, *types.TipSet) (*types.MessageReceipt, error) { + return nil, nil +} + +func (fcs *fakeCS) StateGetActor(ctx context.Context, actor address.Address, ts *types.TipSet) (*types.Actor, error) { + panic("Not Implemented") +} + func (fcs *fakeCS) ChainGetTipSetByHeight(context.Context, uint64, *types.TipSet) (*types.TipSet, error) { panic("Not Implemented") } @@ -514,7 +522,7 @@ func TestCalled(t *testing.T) { err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil - }, func(msg *types.Message, ts *types.TipSet, curH uint64) (bool, error) { + }, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (bool, error) { require.Equal(t, false, applied) applied = true appliedMsg = msg @@ -709,7 +717,7 @@ func TestCalledTimeout(t *testing.T) { err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil - }, func(msg *types.Message, ts *types.TipSet, curH uint64) (bool, error) { + }, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (bool, error) { called = true require.Nil(t, msg) require.Equal(t, uint64(20), ts.Height()) @@ -744,7 +752,7 @@ func TestCalledTimeout(t *testing.T) { err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { return true, true, nil - }, func(msg *types.Message, ts *types.TipSet, curH uint64) (bool, error) { + }, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (bool, error) { called = true require.Nil(t, msg) require.Equal(t, uint64(20), ts.Height()) @@ -783,7 +791,7 @@ func TestCalledOrder(t *testing.T) { err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil - }, func(msg *types.Message, ts *types.TipSet, curH uint64) (bool, error) { + }, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (bool, error) { switch at { case 0: require.Equal(t, uint64(1), msg.Nonce) diff --git a/chain/events/utils.go b/chain/events/utils.go new file mode 100644 index 000000000..ba8083f9b --- /dev/null +++ b/chain/events/utils.go @@ -0,0 +1,45 @@ +package events + +import ( + "context" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types" +) + +func (e *calledEvents) CheckMsg(ctx context.Context, smsg store.ChainMsg, hnd CalledHandler) CheckFunc { + msg := smsg.VMMessage() + + return func(ts *types.TipSet) (done bool, more bool, err error) { + fa, err := e.cs.StateGetActor(ctx, msg.From, ts) + if err != nil { + return false, true, err + } + + // >= because actor nonce is actually the next nonce that is expected to appear on chain + if msg.Nonce >= fa.Nonce { + return false, true, nil + } + + rec, err := e.cs.StateGetReceipt(ctx, smsg.VMMessage().Cid(), ts) + if err != nil { + return false, true, xerrors.Errorf("getting receipt in CheckMsg: %w", err) + } + + more, err = hnd(msg, rec, ts, ts.Height()) + + return true, more, err + } +} + +func (e *calledEvents) MatchMsg(inmsg *types.Message) MatchFunc { + return func(msg *types.Message) (bool, error) { + if msg.From == inmsg.From && msg.Nonce == inmsg.Nonce && !inmsg.Equals(msg) { + return false, xerrors.Errorf("matching msg %s from %s, nonce %d: got duplicate origin/nonce msg %s", inmsg.Cid(), inmsg.From, inmsg.Nonce, msg.Nonce) + } + + return inmsg.Equals(msg), nil + } +} diff --git a/chain/messagepool.go b/chain/messagepool.go index 53f6f282e..f0a3ed383 100644 --- a/chain/messagepool.go +++ b/chain/messagepool.go @@ -1,21 +1,28 @@ package chain import ( + "bytes" + "context" + "errors" "sort" "sync" "time" - "errors" - lru "github.com/hashicorp/golang-lru" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + "github.com/ipfs/go-datastore/query" pubsub "github.com/libp2p/go-libp2p-pubsub" + lps "github.com/whyrusleeping/pubsub" "go.uber.org/multierr" "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) var ( @@ -32,6 +39,10 @@ var ( const ( msgTopic = "/fil/messages" + + localMsgsDs = "/mpool/local" + + localUpdates = "update" ) type MessagePool struct { @@ -54,6 +65,10 @@ type MessagePool struct { maxTxPoolSize int blsSigCache *lru.TwoQueueCache + + changes *lps.PubSub + + localMsgs datastore.Datastore } type msgSet struct { @@ -83,7 +98,7 @@ func (ms *msgSet) add(m *types.SignedMessage) error { return nil } -func NewMessagePool(sm *stmgr.StateManager, ps *pubsub.PubSub) *MessagePool { +func NewMessagePool(sm *stmgr.StateManager, ps *pubsub.PubSub, ds dtypes.MetadataDS) (*MessagePool, error) { cache, _ := lru.New2Q(build.BlsSignatureCacheSize) mp := &MessagePool{ closer: make(chan struct{}), @@ -93,9 +108,18 @@ func NewMessagePool(sm *stmgr.StateManager, ps *pubsub.PubSub) *MessagePool { sm: sm, ps: ps, minGasPrice: types.NewInt(0), - maxTxPoolSize: 100000, + maxTxPoolSize: 5000, blsSigCache: cache, + changes: lps.New(50), + localMsgs: namespace.Wrap(ds, datastore.NewKey(localMsgsDs)), } + + if err := mp.loadLocal(); err != nil { + return nil, xerrors.Errorf("loading local messages: %w", err) + } + + go mp.repubLocal() + sm.ChainStore().SubscribeHeadChanges(func(rev, app []*types.TipSet) error { err := mp.HeadChange(rev, app) if err != nil { @@ -104,7 +128,7 @@ func NewMessagePool(sm *stmgr.StateManager, ps *pubsub.PubSub) *MessagePool { return err }) - return mp + return mp, nil } func (mp *MessagePool) Close() error { @@ -127,13 +151,13 @@ func (mp *MessagePool) repubLocal() { for _, msg := range msgs { msgb, err := msg.Serialize() if err != nil { - multierr.Append(errout, xerrors.Errorf("could not serialize: %w", err)) + errout = multierr.Append(errout, xerrors.Errorf("could not serialize: %w", err)) continue } err = mp.ps.Publish(msgTopic, msgb) if err != nil { - multierr.Append(errout, xerrors.Errorf("could not publish: %w", err)) + errout = multierr.Append(errout, xerrors.Errorf("could not publish: %w", err)) continue } } @@ -149,8 +173,14 @@ func (mp *MessagePool) repubLocal() { } -func (mp *MessagePool) addLocal(a address.Address) { - mp.localAddrs[a] = struct{}{} +func (mp *MessagePool) addLocal(m *types.SignedMessage, msgb []byte) error { + mp.localAddrs[m.Message.From] = struct{}{} + + if err := mp.localMsgs.Put(datastore.NewKey(string(m.Cid().Bytes())), msgb); err != nil { + return xerrors.Errorf("persisting local message: %w", err) + } + + return nil } func (mp *MessagePool) Push(m *types.SignedMessage) error { @@ -164,7 +194,10 @@ func (mp *MessagePool) Push(m *types.SignedMessage) error { } mp.lk.Lock() - mp.addLocal(m.Message.From) + if err := mp.addLocal(m, msgb); err != nil { + mp.lk.Unlock() + return err + } mp.lk.Unlock() return mp.ps.Publish(msgTopic, msgb) @@ -224,13 +257,25 @@ func (mp *MessagePool) addLocked(m *types.SignedMessage) error { return err } + if _, err := mp.sm.ChainStore().PutMessage(&m.Message); err != nil { + log.Warnf("mpooladd cs.PutMessage failed: %s", err) + return err + } + mset, ok := mp.pending[m.Message.From] if !ok { mset = newMsgSet() mp.pending[m.Message.From] = mset } - mset.add(m) + if err := mset.add(m); err != nil { + log.Error(err) + } + + mp.changes.Pub(api.MpoolUpdate{ + Type: api.MpoolAdd, + Message: m, + }, localUpdates) return nil } @@ -242,12 +287,23 @@ func (mp *MessagePool) GetNonce(addr address.Address) (uint64, error) { } func (mp *MessagePool) getNonceLocked(addr address.Address) (uint64, error) { + stateNonce, err := mp.getStateNonce(addr) // sanity check + if err != nil { + return 0, err + } + mset, ok := mp.pending[addr] if ok { + if stateNonce > mset.nextNonce { + log.Errorf("state nonce was larger than mset.nextNonce (%d > %d)", stateNonce, mset.nextNonce) + + return stateNonce, nil + } + return mset.nextNonce, nil } - return mp.getStateNonce(addr) + return stateNonce, nil } func (mp *MessagePool) getStateNonce(addr address.Address) (uint64, error) { @@ -290,7 +346,9 @@ func (mp *MessagePool) PushWithNonce(addr address.Address, cb func(uint64) (*typ if err := mp.addLocked(msg); err != nil { return nil, err } - mp.addLocal(msg.Message.From) + if err := mp.addLocal(msg, msgb); err != nil { + log.Errorf("addLocal failed: %+v", err) + } return msg, mp.ps.Publish(msgTopic, msgb) } @@ -304,13 +362,19 @@ func (mp *MessagePool) Remove(from address.Address, nonce uint64) { return } + if m, ok := mset.msgs[nonce]; ok { + mp.changes.Pub(api.MpoolUpdate{ + Type: api.MpoolRemove, + Message: m, + }, localUpdates) + } + // NB: This deletes any message with the given nonce. This makes sense // as two messages with the same sender cannot have the same nonce delete(mset.msgs, nonce) if len(mset.msgs) == 0 { - // FIXME: This is racy - //delete(mp.pending, from) + delete(mp.pending, from) } else { var max uint64 for nonce := range mset.msgs { @@ -318,6 +382,10 @@ func (mp *MessagePool) Remove(from address.Address, nonce uint64) { max = nonce } } + if max < nonce { + max = nonce // we could have not seen the removed message before + } + mset.nextNonce = max + 1 } } @@ -361,7 +429,7 @@ func (mp *MessagePool) HeadChange(revert []*types.TipSet, apply []*types.TipSet) } for _, msg := range smsgs { if err := mp.Add(msg); err != nil { - return err + log.Error(err) // TODO: probably lots of spam in multi-block tsets } } @@ -369,7 +437,7 @@ func (mp *MessagePool) HeadChange(revert []*types.TipSet, apply []*types.TipSet) smsg := mp.RecoverSig(msg) if smsg != nil { if err := mp.Add(smsg); err != nil { - return err + log.Error(err) // TODO: probably lots of spam in multi-block tsets } } else { log.Warnf("could not recover signature for bls message %s during a reorg revert", msg.Cid()) @@ -404,7 +472,7 @@ func (mp *MessagePool) RecoverSig(msg *types.Message) *types.SignedMessage { } sig, ok := val.(types.Signature) if !ok { - log.Warnf("value in signature cache was not a signature (got %T)", val) + log.Errorf("value in signature cache was not a signature (got %T)", val) return nil } @@ -413,3 +481,55 @@ func (mp *MessagePool) RecoverSig(msg *types.Message) *types.SignedMessage { Signature: sig, } } + +func (mp *MessagePool) Updates(ctx context.Context) (<-chan api.MpoolUpdate, error) { + out := make(chan api.MpoolUpdate, 20) + sub := mp.changes.Sub(localUpdates) + + go func() { + defer mp.changes.Unsub(sub, localIncoming) + + for { + select { + case u := <-sub: + select { + case out <- u.(api.MpoolUpdate): + case <-ctx.Done(): + return + } + case <-ctx.Done(): + return + } + } + }() + + return out, nil +} + +func (mp *MessagePool) loadLocal() error { + res, err := mp.localMsgs.Query(query.Query{}) + if err != nil { + return xerrors.Errorf("query local messages: %w", err) + } + + for r := range res.Next() { + if r.Error != nil { + return xerrors.Errorf("r.Error: %w", r.Error) + } + + var sm types.SignedMessage + if err := sm.UnmarshalCBOR(bytes.NewReader(r.Value)); err != nil { + return xerrors.Errorf("unmarshaling local message: %w", err) + } + + if err := mp.Add(&sm); err != nil { + if xerrors.Is(err, ErrNonceTooLow) { + continue // todo: drop the message from local cache (if above certain confidence threshold) + } + + return xerrors.Errorf("adding local message: %w", err) + } + } + + return nil +} diff --git a/chain/state/statetree.go b/chain/state/statetree.go index 55a812275..1237007a5 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -47,13 +47,11 @@ func LoadStateTree(cst *hamt.CborIpldStore, c cid.Cid) (*StateTree, error) { } func (st *StateTree) SetActor(addr address.Address, act *types.Actor) error { - if addr.Protocol() != address.ID { - iaddr, err := st.lookupID(addr) - if err != nil { - return xerrors.Errorf("ID lookup failed: %w", err) - } - addr = iaddr + iaddr, err := st.LookupID(addr) + if err != nil { + return xerrors.Errorf("ID lookup failed: %w", err) } + addr = iaddr cact, ok := st.actorcache[addr] if ok { @@ -67,7 +65,11 @@ func (st *StateTree) SetActor(addr address.Address, act *types.Actor) error { return st.root.Set(context.TODO(), string(addr.Bytes()), act) } -func (st *StateTree) lookupID(addr address.Address) (address.Address, error) { +func (st *StateTree) LookupID(addr address.Address) (address.Address, error) { + if addr.Protocol() == address.ID { + return addr, nil + } + act, err := st.GetActor(actors.InitAddress) if err != nil { return address.Undef, xerrors.Errorf("getting init actor: %w", err) @@ -86,16 +88,14 @@ func (st *StateTree) GetActor(addr address.Address) (*types.Actor, error) { return nil, fmt.Errorf("GetActor called on undefined address") } - if addr.Protocol() != address.ID { - iaddr, err := st.lookupID(addr) - if err != nil { - if xerrors.Is(err, hamt.ErrNotFound) { - return nil, xerrors.Errorf("resolution lookup failed (%s): %w", addr, types.ErrActorNotFound) - } - return nil, xerrors.Errorf("address resolution: %w", err) + iaddr, err := st.LookupID(addr) + if err != nil { + if xerrors.Is(err, hamt.ErrNotFound) { + return nil, xerrors.Errorf("resolution lookup failed (%s): %w", addr, types.ErrActorNotFound) } - addr = iaddr + return nil, xerrors.Errorf("address resolution: %w", err) } + addr = iaddr cact, ok := st.actorcache[addr] if ok { @@ -103,7 +103,7 @@ func (st *StateTree) GetActor(addr address.Address) (*types.Actor, error) { } var act types.Actor - err := st.root.Find(context.TODO(), string(addr.Bytes()), &act) + err = st.root.Find(context.TODO(), string(addr.Bytes()), &act) if err != nil { if err == hamt.ErrNotFound { return nil, types.ErrActorNotFound diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index a1a656d7f..c13a894a2 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -345,6 +345,29 @@ func (sm *StateManager) GetBlsPublicKey(ctx context.Context, addr address.Addres return pubk, nil } +func (sm *StateManager) GetReceipt(ctx context.Context, msg cid.Cid, ts *types.TipSet) (*types.MessageReceipt, error) { + m, err := sm.cs.GetCMessage(msg) + if err != nil { + return nil, fmt.Errorf("failed to load message: %w", err) + } + + r, err := sm.tipsetExecutedMessage(ts, msg, m.VMMessage()) + if err != nil { + return nil, err + } + + if r != nil { + return r, nil + } + + _, r, err = sm.searchBackForMsg(ctx, ts, m) + if err != nil { + return nil, fmt.Errorf("failed to look back through chain for message: %w", err) + } + + return r, nil +} + func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid) (*types.TipSet, *types.MessageReceipt, error) { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -369,7 +392,7 @@ func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid) (*type return nil, nil, fmt.Errorf("expected current head on SHC stream (got %s)", head[0].Type) } - r, err := sm.tipsetExecutedMessage(head[0].Val, mcid) + r, err := sm.tipsetExecutedMessage(head[0].Val, mcid, msg.VMMessage()) if err != nil { return nil, nil, err } @@ -404,7 +427,7 @@ func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid) (*type case store.HCRevert: continue case store.HCApply: - r, err := sm.tipsetExecutedMessage(val.Val, mcid) + r, err := sm.tipsetExecutedMessage(val.Val, mcid, msg.VMMessage()) if err != nil { return nil, nil, err } @@ -455,7 +478,7 @@ func (sm *StateManager) searchBackForMsg(ctx context.Context, from *types.TipSet return nil, nil, fmt.Errorf("failed to load tipset during msg wait searchback: %w", err) } - r, err := sm.tipsetExecutedMessage(ts, m.Cid()) + r, err := sm.tipsetExecutedMessage(ts, m.Cid(), m.VMMessage()) if err != nil { return nil, nil, fmt.Errorf("checking for message execution during lookback: %w", err) } @@ -468,7 +491,7 @@ func (sm *StateManager) searchBackForMsg(ctx context.Context, from *types.TipSet } } -func (sm *StateManager) tipsetExecutedMessage(ts *types.TipSet, msg cid.Cid) (*types.MessageReceipt, error) { +func (sm *StateManager) tipsetExecutedMessage(ts *types.TipSet, msg cid.Cid, vmm *types.Message) (*types.MessageReceipt, error) { // The genesis block did not execute any messages if ts.Height() == 0 { return nil, nil @@ -484,9 +507,24 @@ func (sm *StateManager) tipsetExecutedMessage(ts *types.TipSet, msg cid.Cid) (*t return nil, err } - for i, m := range cm { - if m.Cid() == msg { - return sm.cs.GetParentReceipt(ts.Blocks()[0], i) + for ii := range cm { + // iterate in reverse because we going backwards through the chain + i := len(cm) - ii - 1 + m := cm[i] + + if m.VMMessage().From == vmm.From { // cheaper to just check origin first + if m.VMMessage().Nonce == vmm.Nonce { + if m.Cid() == msg { + return sm.cs.GetParentReceipt(ts.Blocks()[0], i) + } + + // this should be that message + return nil, xerrors.Errorf("found message with equal nonce as the one we are looking for (F:%s n %d, TS: %s n%d)", + msg, vmm.Nonce, m.Cid(), m.VMMessage().Nonce) + } + if m.VMMessage().Nonce < vmm.Nonce { + return nil, nil // don't bother looking further + } } } diff --git a/chain/store/store.go b/chain/store/store.go index 5b047a38f..44bdefacf 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/vm" "go.opencensus.io/trace" + "go.uber.org/multierr" amt "github.com/filecoin-project/go-amt-ipld" "github.com/filecoin-project/lotus/chain/types" @@ -23,7 +24,6 @@ import ( hamt "github.com/ipfs/go-hamt-ipld" bstore "github.com/ipfs/go-ipfs-blockstore" logging "github.com/ipfs/go-log" - "github.com/pkg/errors" cbg "github.com/whyrusleeping/cbor-gen" pubsub "github.com/whyrusleeping/pubsub" "golang.org/x/xerrors" @@ -99,12 +99,12 @@ func (cs *ChainStore) Load() error { return nil } if err != nil { - return errors.Wrap(err, "failed to load chain state from datastore") + return xerrors.Errorf("failed to load chain state from datastore: %w", err) } var tscids []cid.Cid if err := json.Unmarshal(head, &tscids); err != nil { - return errors.Wrap(err, "failed to unmarshal stored chain head") + return xerrors.Errorf("failed to unmarshal stored chain head: %w", err) } ts, err := cs.LoadTipSet(tscids) @@ -120,11 +120,11 @@ func (cs *ChainStore) Load() error { func (cs *ChainStore) writeHead(ts *types.TipSet) error { data, err := json.Marshal(ts.Cids()) if err != nil { - return errors.Wrap(err, "failed to marshal tipset") + return xerrors.Errorf("failed to marshal tipset: %w", err) } if err := cs.ds.Put(chainHeadKey, data); err != nil { - return errors.Wrap(err, "failed to write chain head to datastore") + return xerrors.Errorf("failed to write chain head to datastore: %w", err) } return nil @@ -208,7 +208,7 @@ func (cs *ChainStore) PutTipSet(ctx context.Context, ts *types.TipSet) error { log.Debugf("expanded %s into %s\n", ts.Cids(), expanded.Cids()) if err := cs.MaybeTakeHeavierTipSet(ctx, expanded); err != nil { - return errors.Wrap(err, "MaybeTakeHeavierTipSet failed in PutTipSet") + return xerrors.Errorf("MaybeTakeHeavierTipSet failed in PutTipSet: %w", err) } return nil } @@ -428,17 +428,32 @@ func (cs *ChainStore) AddToTipSetTracker(b *types.BlockHeader) error { return nil } -func (cs *ChainStore) PersistBlockHeaders(b ...*types.BlockHeader) (err error) { +func (cs *ChainStore) PersistBlockHeaders(b ...*types.BlockHeader) error { sbs := make([]block.Block, len(b)) for i, header := range b { + var err error sbs[i], err = header.ToStorageBlock() if err != nil { return err } } - return cs.bs.PutMany(sbs) + batchSize := 256 + calls := len(b) / batchSize + + var err error + for i := 0; i <= calls; i++ { + start := batchSize * i + end := start + batchSize + if end > len(b) { + end = len(b) + } + + err = multierr.Append(err, cs.bs.PutMany(sbs[start:end])) + } + + return err } type storable interface { @@ -506,7 +521,7 @@ func (cs *ChainStore) AddBlock(ctx context.Context, b *types.BlockHeader) error } if err := cs.MaybeTakeHeavierTipSet(ctx, ts); err != nil { - return errors.Wrap(err, "MaybeTakeHeavierTipSet failed") + return xerrors.Errorf("MaybeTakeHeavierTipSet failed: %w", err) } return nil @@ -536,6 +551,9 @@ func (cs *ChainStore) GetCMessage(c cid.Cid) (ChainMsg, error) { if err == nil { return m, nil } + if err != bstore.ErrNotFound { + log.Warn("GetCMessage: unexpected error getting unsigned message: %s", err) + } return cs.GetSignedMessage(c) } @@ -666,12 +684,12 @@ func (cs *ChainStore) readMsgMetaCids(mmc cid.Cid) ([]cid.Cid, []cid.Cid, error) blscids, err := cs.readAMTCids(msgmeta.BlsMessages) if err != nil { - return nil, nil, errors.Wrap(err, "loading bls message cids for block") + return nil, nil, xerrors.Errorf("loading bls message cids for block: %w", err) } secpkcids, err := cs.readAMTCids(msgmeta.SecpkMessages) if err != nil { - return nil, nil, errors.Wrap(err, "loading secpk message cids for block") + return nil, nil, xerrors.Errorf("loading secpk message cids for block: %w", err) } cs.mmCache.Add(mmc, &mmCids{ @@ -690,12 +708,12 @@ func (cs *ChainStore) MessagesForBlock(b *types.BlockHeader) ([]*types.Message, blsmsgs, err := cs.LoadMessagesFromCids(blscids) if err != nil { - return nil, nil, errors.Wrap(err, "loading bls messages for block") + return nil, nil, xerrors.Errorf("loading bls messages for block: %w", err) } secpkmsgs, err := cs.LoadSignedMessagesFromCids(secpkcids) if err != nil { - return nil, nil, errors.Wrap(err, "loading secpk messages for block") + return nil, nil, xerrors.Errorf("loading secpk messages for block: %w", err) } return blsmsgs, secpkmsgs, nil @@ -705,7 +723,7 @@ func (cs *ChainStore) GetParentReceipt(b *types.BlockHeader, i int) (*types.Mess bs := amt.WrapBlockstore(cs.bs) a, err := amt.LoadAMT(bs, b.ParentMessageReceipts) if err != nil { - return nil, errors.Wrap(err, "amt load") + return nil, xerrors.Errorf("amt load: %w", err) } var r types.MessageReceipt @@ -721,7 +739,7 @@ func (cs *ChainStore) LoadMessagesFromCids(cids []cid.Cid) ([]*types.Message, er for i, c := range cids { m, err := cs.GetMessage(c) if err != nil { - return nil, errors.Wrapf(err, "failed to get message: (%s):%d", c, i) + return nil, xerrors.Errorf("failed to get message: (%s):%d: %w", err, c, i) } msgs = append(msgs, m) @@ -735,7 +753,7 @@ func (cs *ChainStore) LoadSignedMessagesFromCids(cids []cid.Cid) ([]*types.Signe for i, c := range cids { m, err := cs.GetSignedMessage(c) if err != nil { - return nil, errors.Wrapf(err, "failed to get message: (%s):%d", c, i) + return nil, xerrors.Errorf("failed to get message: (%s):%d: %w", err, c, i) } msgs = append(msgs, m) diff --git a/chain/sync.go b/chain/sync.go index 4ec8b153a..4b299f504 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -20,6 +20,7 @@ import ( logging "github.com/ipfs/go-log" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" + "github.com/whyrusleeping/pubsub" "go.opencensus.io/trace" "golang.org/x/xerrors" @@ -38,6 +39,8 @@ import ( var log = logging.Logger("chain") +var localIncoming = "incoming" + type Syncer struct { // The heaviest known tipset in the network. @@ -61,6 +64,8 @@ type Syncer struct { syncLock sync.Mutex syncmgr *SyncManager + + incoming *pubsub.PubSub } func NewSyncer(sm *stmgr.StateManager, bsync *blocksync.BlockSync, self peer.ID) (*Syncer, error) { @@ -81,6 +86,8 @@ func NewSyncer(sm *stmgr.StateManager, bsync *blocksync.BlockSync, self peer.ID) store: sm.ChainStore(), sm: sm, self: self, + + incoming: pubsub.New(50), } s.syncmgr = NewSyncManager(s.Sync) @@ -112,6 +119,8 @@ func (syncer *Syncer) InformNewHead(from peer.ID, fts *store.FullTipSet) { } } + syncer.incoming.Pub(fts.TipSet().Blocks(), localIncoming) + if from == syncer.self { // TODO: this is kindof a hack... log.Debug("got block from ourselves") @@ -142,6 +151,33 @@ func (syncer *Syncer) InformNewHead(from peer.ID, fts *store.FullTipSet) { syncer.syncmgr.SetPeerHead(ctx, from, fts.TipSet()) } +func (syncer *Syncer) IncomingBlocks(ctx context.Context) (<-chan *types.BlockHeader, error) { + sub := syncer.incoming.Sub(localIncoming) + out := make(chan *types.BlockHeader, 10) + + go func() { + defer syncer.incoming.Unsub(sub, localIncoming) + + for { + select { + case r := <-sub: + hs := r.([]*types.BlockHeader) + for _, h := range hs { + select { + case out <- h: + case <-ctx.Done(): + return + } + } + case <-ctx.Done(): + return + } + } + }() + + return out, nil +} + func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error { var bcids, scids []cbg.CBORMarshaler for _, m := range fblk.BlsMessages { @@ -396,6 +432,10 @@ func (syncer *Syncer) Sync(ctx context.Context, maybeHead *types.TipSet) error { ) } + if syncer.store.GetHeaviestTipSet().ParentWeight().GreaterThan(maybeHead.ParentWeight()) { + return nil + } + if syncer.Genesis.Equals(maybeHead) || syncer.store.GetHeaviestTipSet().Equals(maybeHead) { return nil } diff --git a/chain/types/bigint.go b/chain/types/bigint.go index 85d2181c8..266f2a8af 100644 --- a/chain/types/bigint.go +++ b/chain/types/bigint.go @@ -124,6 +124,28 @@ func (bi *BigInt) UnmarshalJSON(b []byte) error { return nil } +func (bi *BigInt) Scan(value interface{}) error { + switch value := value.(type) { + case string: + i, ok := big.NewInt(0).SetString(value, 10) + if !ok { + if value == "" { + return nil + } + return xerrors.Errorf("failed to parse bigint string: '%s'", value) + } + + bi.Int = i + + return nil + case int64: + bi.Int = big.NewInt(value) + return nil + default: + return xerrors.Errorf("non-string types unsupported: %T", value) + } +} + func (bi *BigInt) cborBytes() []byte { if bi.Int == nil { return []byte{} diff --git a/chain/types/message.go b/chain/types/message.go index d2bf27358..aee5300cf 100644 --- a/chain/types/message.go +++ b/chain/types/message.go @@ -77,3 +77,7 @@ func (m *Message) RequiredFunds() BigInt { func (m *Message) VMMessage() *Message { return m } + +func (m *Message) Equals(o *Message) bool { + return m.Cid() == o.Cid() +} diff --git a/chain/types/signature.go b/chain/types/signature.go index 8d7182e12..f3a61327b 100644 --- a/chain/types/signature.go +++ b/chain/types/signature.go @@ -56,7 +56,6 @@ func (s *Signature) TypeCode() int { case KTBLS: return IKTBLS default: - log.Errorf("called TypeCode on signature with unknown Type: %q", s.Type) return IKTUnknown } } diff --git a/chain/validation/message.go b/chain/validation/message.go index 99ffb9b64..f9728708f 100644 --- a/chain/validation/message.go +++ b/chain/validation/message.go @@ -3,11 +3,10 @@ package validation import ( "context" - "github.com/pkg/errors" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" + "golang.org/x/xerrors" "github.com/filecoin-project/chain-validation/pkg/chain" "github.com/filecoin-project/chain-validation/pkg/state" @@ -43,7 +42,7 @@ func (mf *MessageFactory) MakeMessage(from, to state.Address, method chain.Metho } if int(method) >= len(methods) { - return nil, errors.Errorf("No method name for method %v", method) + return nil, xerrors.Errorf("No method name for method %v", method) } methodId := methods[method] msg := &types.Message{ diff --git a/chain/validation/state.go b/chain/validation/state.go index 9886c43f7..8a51e442e 100644 --- a/chain/validation/state.go +++ b/chain/validation/state.go @@ -5,11 +5,10 @@ import ( "fmt" "math/rand" - "github.com/pkg/errors" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/vm" + "golang.org/x/xerrors" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" @@ -109,7 +108,7 @@ func (s *StateWrapper) SetActor(addr vstate.Address, code vstate.ActorCodeID, ba // The ID-based address is dropped here, but should be reported back to the caller. _, err = tree.RegisterNewAddress(addrInt, &actr.Actor) if err != nil { - return nil, nil, errors.Wrapf(err, "register new address for actor") + return nil, nil, xerrors.Errorf("register new address for actor: %w", err) } return actr, s.storage, s.flush(tree) } @@ -131,7 +130,7 @@ func (s *StateWrapper) SetSingletonActor(addr vstate.SingletonActorID, balance v return nil, nil, err } if err := tree.SetActor(actors.InitAddress, initact); err != nil { - return nil, nil, errors.Wrapf(err, "set init actor") + return nil, nil, xerrors.Errorf("set init actor: %w", err) } return &actorWrapper{*initact}, s.storage, s.flush(tree) @@ -141,7 +140,7 @@ func (s *StateWrapper) SetSingletonActor(addr vstate.SingletonActorID, balance v return nil, nil, err } if err := tree.SetActor(actors.StorageMarketAddress, smact); err != nil { - return nil, nil, errors.Wrapf(err, "set network storage market actor") + return nil, nil, xerrors.Errorf("set network storage market actor: %w", err) } return &actorWrapper{*smact}, s.storage, s.flush(tree) case actors.StoragePowerAddress: @@ -150,7 +149,7 @@ func (s *StateWrapper) SetSingletonActor(addr vstate.SingletonActorID, balance v return nil, nil, err } if err := tree.SetActor(actors.StoragePowerAddress, spact); err != nil { - return nil, nil, errors.Wrapf(err, "set network storage market actor") + return nil, nil, xerrors.Errorf("set network storage market actor: %w", err) } return &actorWrapper{*spact}, s.storage, s.flush(tree) case actors.NetworkAddress: @@ -160,7 +159,7 @@ func (s *StateWrapper) SetSingletonActor(addr vstate.SingletonActorID, balance v Head: vm.EmptyObjectCid, } if err := tree.SetActor(actors.NetworkAddress, ntwkact); err != nil { - return nil, nil, errors.Wrapf(err, "set network actor") + return nil, nil, xerrors.Errorf("set network actor: %w", err) } return &actorWrapper{*ntwkact}, s.storage, s.flush(tree) case actors.BurntFundsAddress: @@ -170,11 +169,11 @@ func (s *StateWrapper) SetSingletonActor(addr vstate.SingletonActorID, balance v Head: vm.EmptyObjectCid, } if err := tree.SetActor(actors.BurntFundsAddress, ntwkact); err != nil { - return nil, nil, errors.Wrapf(err, "set network actor") + return nil, nil, xerrors.Errorf("set network actor: %w", err) } return &actorWrapper{*ntwkact}, s.storage, s.flush(tree) default: - return nil, nil, errors.Errorf("%v is not a singleton actor address", addr) + return nil, nil, xerrors.Errorf("%v is not a singleton actor address", addr) } } diff --git a/cli/mpool.go b/cli/mpool.go index 23adfad68..a2b4324f6 100644 --- a/cli/mpool.go +++ b/cli/mpool.go @@ -4,7 +4,11 @@ import ( "encoding/json" "fmt" + "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" + + "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/types" ) var mpoolCmd = &cli.Command{ @@ -12,6 +16,8 @@ var mpoolCmd = &cli.Command{ Usage: "Manage message pool", Subcommands: []*cli.Command{ mpoolPending, + mpoolSub, + mpoolStat, }, } @@ -43,3 +49,107 @@ var mpoolPending = &cli.Command{ return nil }, } + +var mpoolSub = &cli.Command{ + Name: "sub", + Usage: "Subscibe to mpool changes", + Action: func(cctx *cli.Context) error { + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := ReqContext(cctx) + + sub, err := api.MpoolSub(ctx) + if err != nil { + return err + } + + for { + select { + case update := <-sub: + out, err := json.MarshalIndent(update, "", " ") + if err != nil { + return err + } + fmt.Println(string(out)) + case <-ctx.Done(): + return nil + } + } + }, +} + +type statBucket struct { + msgs map[uint64]*types.SignedMessage +} + +var mpoolStat = &cli.Command{ + Name: "stat", + Usage: "print mempool stats", + Action: func(cctx *cli.Context) error { + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := ReqContext(cctx) + + ts, err := api.ChainHead(ctx) + if err != nil { + return xerrors.Errorf("getting chain head: %w", err) + } + + msgs, err := api.MpoolPending(ctx, nil) + if err != nil { + return err + } + + buckets := map[address.Address]*statBucket{} + + for _, v := range msgs { + bkt, ok := buckets[v.Message.From] + if !ok { + bkt = &statBucket{ + msgs: map[uint64]*types.SignedMessage{}, + } + buckets[v.Message.From] = bkt + } + + bkt.msgs[v.Message.Nonce] = v + } + for a, bkt := range buckets { + act, err := api.StateGetActor(ctx, a, ts) + if err != nil { + return err + } + + cur := act.Nonce + for { + _, ok := bkt.msgs[cur] + if !ok { + break + } + cur++ + } + + past := 0 + future := 0 + for _, m := range bkt.msgs { + if m.Message.Nonce < act.Nonce { + past++ + } + if m.Message.Nonce > cur { + future++ + } + } + + fmt.Printf("%s, past: %d, cur: %d, future: %d\n", a, past, cur-act.Nonce, future) + } + + return nil + }, +} diff --git a/cli/state.go b/cli/state.go index 91a7f9fa1..24809f982 100644 --- a/cli/state.go +++ b/cli/state.go @@ -21,6 +21,7 @@ var stateCmd = &cli.Command{ stateListActorsCmd, stateListMinersCmd, stateGetActorCmd, + stateLookupIDCmd, }, } @@ -288,3 +289,35 @@ var stateGetActorCmd = &cli.Command{ return nil }, } + +var stateLookupIDCmd = &cli.Command{ + Name: "lookup", + Usage: "Find corresponding ID address", + Action: func(cctx *cli.Context) error { + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := ReqContext(cctx) + + if !cctx.Args().Present() { + return fmt.Errorf("must pass address of actor to get") + } + + addr, err := address.NewFromString(cctx.Args().First()) + if err != nil { + return err + } + + a, err := api.StateLookupID(ctx, addr, nil) + if err != nil { + return err + } + + fmt.Printf("%s\n", a) + + return nil + }, +} diff --git a/cli/sync.go b/cli/sync.go index b1b4efd0c..533a7bc9f 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -41,14 +41,20 @@ var syncStatusCmd = &cli.Command{ for i, ss := range state.ActiveSyncs { fmt.Printf("worker %d:\n", i) var base, target []cid.Cid + var heightDiff int64 if ss.Base != nil { base = ss.Base.Cids() + heightDiff = int64(ss.Base.Height()) } if ss.Target != nil { target = ss.Target.Cids() + heightDiff = int64(ss.Target.Height()) - heightDiff + } else { + heightDiff = 0 } fmt.Printf("\tBase:\t%s\n", base) fmt.Printf("\tTarget:\t%s\n", target) + fmt.Printf("\tHeight diff:\t%d\n", heightDiff) fmt.Printf("\tStage: %s\n", chain.SyncStageString(ss.Stage)) fmt.Printf("\tHeight: %d\n", ss.Height) } diff --git a/cmd/lotus-chainwatch/blockssub.go b/cmd/lotus-chainwatch/blockssub.go new file mode 100644 index 000000000..9c5957f67 --- /dev/null +++ b/cmd/lotus-chainwatch/blockssub.go @@ -0,0 +1,27 @@ +package main + +import ( + "context" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + aapi "github.com/filecoin-project/lotus/api" +) + +func subBlocks(ctx context.Context, api aapi.FullNode, st *storage) { + sub, err := api.SyncIncomingBlocks(ctx) + if err != nil { + log.Error(err) + return + } + + for bh := range sub { + err := st.storeHeaders(map[cid.Cid]*types.BlockHeader{ + bh.Cid(): bh, + }, false) + if err != nil { + log.Error(err) + } + } +} diff --git a/cmd/lotus-chainwatch/dot.go b/cmd/lotus-chainwatch/dot.go new file mode 100644 index 000000000..087b0753a --- /dev/null +++ b/cmd/lotus-chainwatch/dot.go @@ -0,0 +1,63 @@ +package main + +import ( + "fmt" + "hash/crc32" + "strconv" + + "github.com/ipfs/go-cid" + "gopkg.in/urfave/cli.v2" +) + +var dotCmd = &cli.Command{ + Name: "dot", + Usage: "generate dot graphs", + Action: func(cctx *cli.Context) error { + st, err := openStorage() + if err != nil { + return err + } + + minH, err := strconv.ParseInt(cctx.Args().Get(0), 10, 32) + tosee, err := strconv.ParseInt(cctx.Args().Get(1), 10, 32) + maxH := minH + tosee + + res, err := st.db.Query("select block, parent, b.miner, b.height from block_parents inner join blocks b on block_parents.block = b.cid where b.height > ? and b.height < ?", minH, maxH) + if err != nil { + return err + } + + fmt.Println("digraph D {") + + for res.Next() { + var block, parent, miner string + var height uint64 + if err := res.Scan(&block, &parent, &miner, &height); err != nil { + return err + } + + bc, err := cid.Parse(block) + if err != nil { + return err + } + + has := st.hasBlock(bc) + + col := crc32.Checksum([]byte(miner), crc32.MakeTable(crc32.Castagnoli))&0xc0c0c0c0 + 0x30303030 + + hasstr := "" + if !has { + hasstr = " UNSYNCED" + } + + fmt.Printf("%s [label = \"%s:%d%s\", fillcolor = \"#%06x\", style=filled, forcelabels=true]\n%s -> %s\n", block, miner, height, hasstr, col, block, parent) + } + if res.Err() != nil { + return res.Err() + } + + fmt.Println("}") + + return nil + }, +} diff --git a/cmd/lotus-chainwatch/main.go b/cmd/lotus-chainwatch/main.go new file mode 100644 index 000000000..c8ea2d863 --- /dev/null +++ b/cmd/lotus-chainwatch/main.go @@ -0,0 +1,99 @@ +package main + +import ( + "fmt" + "net/http" + "os" + + logging "github.com/ipfs/go-log" + "golang.org/x/xerrors" + "gopkg.in/urfave/cli.v2" + + "github.com/filecoin-project/lotus/build" + lcli "github.com/filecoin-project/lotus/cli" +) + +var log = logging.Logger("chainwatch") + +func main() { + logging.SetLogLevel("*", "INFO") + + log.Info("Starting chainwatch") + + local := []*cli.Command{ + runCmd, + dotCmd, + } + + app := &cli.App{ + Name: "lotus-chainwatch", + Usage: "Devnet token distribution utility", + Version: build.Version, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "repo", + EnvVars: []string{"LOTUS_PATH"}, + Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME + }, + }, + + Commands: local, + } + + if err := app.Run(os.Args); err != nil { + log.Warnf("%+v", err) + return + } +} + +var runCmd = &cli.Command{ + Name: "run", + Usage: "Start lotus chainwatch", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "front", + Value: "127.0.0.1:8418", + }, + }, + Action: func(cctx *cli.Context) error { + api, closer, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := lcli.ReqContext(cctx) + + v, err := api.Version(ctx) + if err != nil { + return err + } + + log.Info("Remote version: %s", v.Version) + + st, err := openStorage() + if err != nil { + return err + } + defer st.close() + + runSyncer(ctx, api, st) + go subMpool(ctx, api, st) + go subBlocks(ctx, api, st) + + h, err := newHandler(api, st) + if err != nil { + return xerrors.Errorf("handler setup: %w", err) + } + + http.Handle("/", h) + + fmt.Printf("Open http://%s\n", cctx.String("front")) + + go func() { + <-ctx.Done() + os.Exit(0) + }() + + return http.ListenAndServe(cctx.String("front"), nil) + }, +} diff --git a/cmd/lotus-chainwatch/mpool.go b/cmd/lotus-chainwatch/mpool.go new file mode 100644 index 000000000..ce9e39b84 --- /dev/null +++ b/cmd/lotus-chainwatch/mpool.go @@ -0,0 +1,38 @@ +package main + +import ( + "context" + + "github.com/ipfs/go-cid" + + aapi "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" +) + +func subMpool(ctx context.Context, api aapi.FullNode, st *storage) { + sub, err := api.MpoolSub(ctx) + if err != nil { + return + } + + for change := range sub { + if change.Type != aapi.MpoolAdd { + continue + } + + log.Info("mpool message") + + err := st.storeMessages(map[cid.Cid]*types.Message{ + change.Message.Message.Cid(): &change.Message.Message, + }) + if err != nil { + log.Error(err) + continue + } + + if err := st.storeMpoolInclusion(change.Message.Message.Cid()); err != nil { + log.Error(err) + continue + } + } +} diff --git a/cmd/lotus-chainwatch/site/index.html b/cmd/lotus-chainwatch/site/index.html new file mode 100644 index 000000000..1a1c2813a --- /dev/null +++ b/cmd/lotus-chainwatch/site/index.html @@ -0,0 +1,32 @@ + + + + Lotus ChainWatch + + + +
+
+
+ Lotus ChainWatch +
+
+
+
+ {{countCol "actors" "id"}} Actors; + {{countCol "miner_heads" "addr"}} Miners; + {{netPower "slashed_at = 0" | sizeStr}} Power + ({{netPower "" | sizeStr}} Total; + {{netPower "slashed_at > 0" | sizeStr}} Slashed) +
+
+ {{count "messages"}} Messages; {{count "actors"}} state changes +
+
+ {{count "id_address_map" "id != address"}} Keys; + E% FIL in wallets; F% FIL in miners; M% in market; %G Other actors; %H FIL it treasury +
+
+
+ + diff --git a/cmd/lotus-chainwatch/site/key.html b/cmd/lotus-chainwatch/site/key.html new file mode 100644 index 000000000..697838ed2 --- /dev/null +++ b/cmd/lotus-chainwatch/site/key.html @@ -0,0 +1,40 @@ + + + + Lotus ChainWatch + + + +{{$wallet := param "w"}} + +
+
+
+ Lotus ChainWatch - Wallet {{$wallet}} +
+
+
+
+ Balance: {{queryNum "select balance from actors inner join main.id_address_map m on m.address = ? where actors.id = m.id order by nonce desc limit 1" $wallet }} +
+
+ Messages: + + + {{ range messages "`from` = ? or `to` = ?" $wallet $wallet $wallet}} + + {{ if eq .From.String $wallet }} + + {{else}} + + {{end}} + + + + {{end}} +
DirPeerNonceValueBlockMpool Wait
To{{.To.String}}From{{.From.String}}{{.Nonce}}{{.Value}}
+
+
+
+ + diff --git a/cmd/lotus-chainwatch/site/keys.html b/cmd/lotus-chainwatch/site/keys.html new file mode 100644 index 000000000..adf96adb1 --- /dev/null +++ b/cmd/lotus-chainwatch/site/keys.html @@ -0,0 +1,23 @@ + + + + Lotus ChainWatch + + + +
+
+
+ Lotus ChainWatch - Wallets +
+
+
+
+ {{range strings "id_address_map" "address" "address != id"}} +
{{.}}
+ {{end}} +
+
+
+ + diff --git a/cmd/lotus-chainwatch/site/main.css b/cmd/lotus-chainwatch/site/main.css new file mode 100644 index 000000000..820625360 --- /dev/null +++ b/cmd/lotus-chainwatch/site/main.css @@ -0,0 +1,62 @@ +body { + font-family: 'monospace'; + background: #1f1f1f; + color: #f0f0f0; + padding: 0; + margin: 0; +} + +.Index { + width: 100vw; + height: 100vh; + background: #1a1a1a; + color: #f0f0f0; + font-family: monospace; + overflow: auto; + + display: grid; + grid-template-columns: auto 80vw auto; + grid-template-rows: 3em auto auto auto; + grid-template-areas: + "header header header header" + ". . . ." + ". main main ." + ". main main ." + ". main main ." + ". main main ." + ". main main ." + ". . . ."; +} + +.Index-header { + background: #2a2a2a; + grid-area: header; +} + +.Index-Index-header > div { + padding-left: 0.7em; + padding-top: 0.7em; +} + +.Index-nodes { + grid-area: main; + background: #2a2a2a; +} + +.Index-node { + margin: 5px; + padding: 15px; + background: #1f1f1f; +} + +a:link { + color: #50f020; +} + +a:visited { + color: #50f020; +} + +a:hover { + color: #30a00a; +} diff --git a/cmd/lotus-chainwatch/storage.go b/cmd/lotus-chainwatch/storage.go new file mode 100644 index 000000000..26f88640b --- /dev/null +++ b/cmd/lotus-chainwatch/storage.go @@ -0,0 +1,421 @@ +package main + +import ( + "database/sql" + "sync" + "time" + + "github.com/ipfs/go-cid" + _ "github.com/mattn/go-sqlite3" + + "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/types" +) + +type storage struct { + db *sql.DB + + headerLk sync.Mutex +} + +func openStorage() (*storage, error) { + db, err := sql.Open("sqlite3", "./chainwatch.db") + if err != nil { + return nil, err + } + + st := &storage{db: db} + + return st, st.setup() +} + +func (st *storage) setup() error { + tx, err := st.db.Begin() + if err != nil { + return err + } + _, err = tx.Exec(` +create table if not exists actors + ( + id text not null, + code text not null, + head text not null, + nonce int not null, + balance text not null, + stateroot text + constraint actors_blocks_stateroot_fk + references blocks (parentStateRoot), + constraint actors_pk + primary key (id, nonce, balance, stateroot) + ); + +create index if not exists actors_id_index + on actors (id); + +create table if not exists id_address_map +( + id text not null + constraint id_address_map_actors_id_fk + references actors (id), + address text not null, + constraint id_address_map_pk + primary key (id, address) +); + +create index if not exists id_address_map_address_index + on id_address_map (address); + +create index if not exists id_address_map_id_index + on id_address_map (id); + +create table if not exists messages +( + cid text not null + constraint messages_pk + primary key, + "from" text not null + constraint messages_id_address_map_from_fk + references id_address_map (address), + "to" text not null + constraint messages_id_address_map_to_fk + references id_address_map (address), + nonce int not null, + value text not null, + gasprice int not null, + gaslimit int not null, + method int, + params blob +); + +create unique index if not exists messages_cid_uindex + on messages (cid); + +create table if not exists blocks +( + cid text not null + constraint blocks_pk + primary key, + parentWeight numeric not null, + parentStateRoot text not null, + height int not null, + miner text not null + constraint blocks_id_address_map_miner_fk + references id_address_map (address), + timestamp int not null +); + +create unique index if not exists block_cid_uindex + on blocks (cid); + +create table if not exists blocks_synced +( + cid text not null + constraint blocks_synced_pk + primary key + constraint blocks_synced_blocks_cid_fk + references blocks +); + +create unique index if not exists blocks_synced_cid_uindex + on blocks_synced (cid); + +create table if not exists block_parents +( + block text not null + constraint block_parents_blocks_cid_fk + references blocks, + parent text not null + constraint block_parents_blocks_cid_fk_2 + references blocks +); + +create unique index if not exists block_parents_block_parent_uindex + on block_parents (block, parent); + +create unique index if not exists blocks_cid_uindex + on blocks (cid); + +create table if not exists block_messages +( + block text not null + constraint block_messages_blk_fk + references blocks (cid), + message text not null + constraint block_messages_msg_fk + references messages, + constraint block_messages_pk + primary key (block, message) +); + +create table if not exists mpool_messages +( + msg text not null + constraint mpool_messages_pk + primary key + constraint mpool_messages_messages_cid_fk + references messages, + add_ts int not null +); + +create unique index if not exists mpool_messages_msg_uindex + on mpool_messages (msg); + +create table if not exists miner_heads +( + head text not null + constraint miner_heads_actors_head_fk + references actors (head), + addr text not null + constraint miner_heads_actors_id_fk + references actors (id), + stateroot text not null + constraint miner_heads_blocks_stateroot_fk + references blocks (parentStateRoot), + sectorset text not null, + provingset text not null, + owner text not null, + worker text not null, + peerid text not null, + sectorsize int not null, + power text not null, + active int, + ppe int not null, + slashed_at int not null, + constraint miner_heads_id_address_map_owner_fk + foreign key (owner) references id_address_map (address), + constraint miner_heads_id_address_map_worker_fk + foreign key (worker) references id_address_map (address), + constraint miner_heads_pk + primary key (head, addr) +); + +`) + if err != nil { + return err + } + return tx.Commit() +} + +func (st *storage) hasBlock(bh cid.Cid) bool { + var exitsts bool + err := st.db.QueryRow(`select exists (select 1 FROM blocks_synced where cid=?)`, bh.String()).Scan(&exitsts) + if err != nil { + log.Error(err) + return false + } + return exitsts +} + +func (st *storage) storeActors(actors map[address.Address]map[types.Actor]cid.Cid) error { + tx, err := st.db.Begin() + if err != nil { + return err + } + + stmt, err := tx.Prepare(`insert into actors (id, code, head, nonce, balance, stateroot) values (?, ?, ?, ?, ?, ?) on conflict do nothing`) + if err != nil { + return err + } + defer stmt.Close() + for addr, acts := range actors { + for act, st := range acts { + if _, err := stmt.Exec(addr.String(), act.Code.String(), act.Head.String(), act.Nonce, act.Balance.String(), st.String()); err != nil { + return err + } + } + } + + return tx.Commit() +} + +func (st *storage) storeMiners(miners map[minerKey]*minerInfo) error { + tx, err := st.db.Begin() + if err != nil { + return err + } + + stmt, err := tx.Prepare(`insert into miner_heads (head, addr, stateroot, sectorset, provingset, owner, worker, peerid, sectorsize, power, active, ppe, slashed_at) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict do nothing`) + if err != nil { + return err + } + defer stmt.Close() + for k, i := range miners { + if _, err := stmt.Exec( + k.act.Head.String(), + k.addr.String(), + k.stateroot.String(), + i.state.Sectors.String(), + i.state.ProvingSet.String(), + i.info.Owner.String(), + i.info.Worker.String(), + i.info.PeerID.String(), + i.info.SectorSize, + i.state.Power.String(), + i.state.Active, + i.state.ProvingPeriodEnd, + i.state.SlashedAt, + ); err != nil { + return err + } + } + + return tx.Commit() +} + +func (st *storage) storeHeaders(bhs map[cid.Cid]*types.BlockHeader, sync bool) error { + st.headerLk.Lock() + defer st.headerLk.Unlock() + + tx, err := st.db.Begin() + if err != nil { + return err + } + + stmt, err := tx.Prepare(`insert into blocks (cid, parentWeight, parentStateRoot, height, miner, "timestamp") values (?, ?, ?, ?, ?, ?) on conflict do nothing`) + if err != nil { + return err + } + defer stmt.Close() + for _, bh := range bhs { + if _, err := stmt.Exec(bh.Cid().String(), bh.ParentWeight.String(), bh.ParentStateRoot.String(), bh.Height, bh.Miner.String(), bh.Timestamp); err != nil { + return err + } + } + + stmt2, err := tx.Prepare(`insert into block_parents (block, parent) values (?, ?) on conflict do nothing`) + if err != nil { + return err + } + defer stmt2.Close() + for _, bh := range bhs { + for _, parent := range bh.Parents { + if _, err := stmt2.Exec(bh.Cid().String(), parent.String()); err != nil { + return err + } + } + } + + if sync { + stmt, err := tx.Prepare(`insert into blocks_synced (cid) values (?) on conflict do nothing`) + if err != nil { + return err + } + defer stmt.Close() + for _, bh := range bhs { + if _, err := stmt.Exec(bh.Cid().String()); err != nil { + return err + } + } + } + + return tx.Commit() +} + +func (st *storage) storeMessages(msgs map[cid.Cid]*types.Message) error { + tx, err := st.db.Begin() + if err != nil { + return err + } + + stmt, err := tx.Prepare(`insert into messages (cid, "from", "to", nonce, "value", gasprice, gaslimit, method, params) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict do nothing`) + if err != nil { + return err + } + defer stmt.Close() + + for c, m := range msgs { + if _, err := stmt.Exec( + c.String(), + m.From.String(), + m.To.String(), + m.Nonce, + m.Value.String(), + m.GasPrice.String(), + m.GasLimit.String(), + m.Method, + m.Params, + ); err != nil { + return err + } + } + + return tx.Commit() +} + +func (st *storage) storeAddressMap(addrs map[address.Address]address.Address) error { + tx, err := st.db.Begin() + if err != nil { + return err + } + + stmt, err := tx.Prepare(`insert into id_address_map (id, address) VALUES (?, ?) on conflict do nothing`) + if err != nil { + return err + } + defer stmt.Close() + + for a, i := range addrs { + if i == address.Undef { + continue + } + if _, err := stmt.Exec( + i.String(), + a.String(), + ); err != nil { + return err + } + } + + return tx.Commit() +} + +func (st *storage) storeMsgInclusions(incls map[cid.Cid][]cid.Cid) error { + tx, err := st.db.Begin() + if err != nil { + return err + } + + stmt, err := tx.Prepare(`insert into block_messages (block, message) VALUES (?, ?) on conflict do nothing`) + if err != nil { + return err + } + defer stmt.Close() + + for b, msgs := range incls { + for _, msg := range msgs { + if _, err := stmt.Exec( + b.String(), + msg.String(), + ); err != nil { + return err + } + } + } + + return tx.Commit() +} + +func (st *storage) storeMpoolInclusion(msg cid.Cid) error { + tx, err := st.db.Begin() + if err != nil { + return err + } + + stmt, err := tx.Prepare(`insert into mpool_messages (msg, add_ts) VALUES (?, ?) on conflict do nothing`) + if err != nil { + return err + } + defer stmt.Close() + + if _, err := stmt.Exec( + msg.String(), + time.Now().Unix(), + ); err != nil { + return err + } + return tx.Commit() +} + +func (st *storage) close() error { + return st.db.Close() +} diff --git a/cmd/lotus-chainwatch/sync.go b/cmd/lotus-chainwatch/sync.go new file mode 100644 index 000000000..5c87ea04d --- /dev/null +++ b/cmd/lotus-chainwatch/sync.go @@ -0,0 +1,292 @@ +package main + +import ( + "bytes" + "container/list" + "context" + "sync" + + actors2 "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/address" + + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types" +) + +func runSyncer(ctx context.Context, api api.FullNode, st *storage) { + notifs, err := api.ChainNotify(ctx) + if err != nil { + panic(err) + } + go func() { + for notif := range notifs { + for _, change := range notif { + switch change.Type { + case store.HCCurrent: + fallthrough + case store.HCApply: + syncHead(ctx, api, st, change.Val) + case store.HCRevert: + log.Warnf("revert todo") + } + } + } + }() +} + +type minerKey struct { + addr address.Address + act types.Actor + stateroot cid.Cid +} + +type minerInfo struct { + state actors2.StorageMinerActorState + info actors2.MinerInfo +} + +func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipSet) { + addresses := map[address.Address]address.Address{} + actors := map[address.Address]map[types.Actor]cid.Cid{} + var alk sync.Mutex + + log.Infof("Getting headers / actors") + + toSync := map[cid.Cid]*types.BlockHeader{} + toVisit := list.New() + + for _, header := range ts.Blocks() { + toVisit.PushBack(header) + } + + for toVisit.Len() > 0 { + bh := toVisit.Remove(toVisit.Back()).(*types.BlockHeader) + + if _, seen := toSync[bh.Cid()]; seen || st.hasBlock(bh.Cid()) { + continue + } + + toSync[bh.Cid()] = bh + addresses[bh.Miner] = address.Undef + + if len(toSync)%500 == 10 { + log.Infof("todo: (%d) %s", len(toSync), bh.Cid()) + } + + if len(bh.Parents) == 0 { + continue + } + + pts, err := api.ChainGetTipSet(ctx, types.NewTipSetKey(bh.Parents...)) + if err != nil { + log.Error(err) + continue + } + + for _, header := range pts.Blocks() { + toVisit.PushBack(header) + } + } + + log.Infof("Syncing %d blocks", len(toSync)) + + log.Infof("Persisting actors") + + paDone := 0 + par(50, maparr(toSync), func(bh *types.BlockHeader) { + paDone++ + if paDone%100 == 0 { + log.Infof("pa: %d %d%%", paDone, (paDone*100)/len(toSync)) + } + + if len(bh.Parents) == 0 { // genesis case + ts, err := types.NewTipSet([]*types.BlockHeader{bh}) + aadrs, err := api.StateListActors(ctx, ts) + if err != nil { + log.Error(err) + return + } + + par(50, aadrs, func(addr address.Address) { + act, err := api.StateGetActor(ctx, addr, ts) + if err != nil { + log.Error(err) + return + } + alk.Lock() + _, ok := actors[addr] + if !ok { + actors[addr] = map[types.Actor]cid.Cid{} + } + actors[addr][*act] = bh.ParentStateRoot + addresses[addr] = address.Undef + alk.Unlock() + }) + + return + } + + pts, err := api.ChainGetTipSet(ctx, types.NewTipSetKey(bh.Parents...)) + if err != nil { + log.Error(err) + return + } + + changes, err := api.StateChangedActors(ctx, pts.ParentState(), bh.ParentStateRoot) + if err != nil { + log.Error(err) + return + } + + for a, act := range changes { + addr, err := address.NewFromString(a) + if err != nil { + log.Error(err) + return + } + + alk.Lock() + _, ok := actors[addr] + if !ok { + actors[addr] = map[types.Actor]cid.Cid{} + } + actors[addr][act] = bh.ParentStateRoot + addresses[addr] = address.Undef + alk.Unlock() + } + }) + + if err := st.storeActors(actors); err != nil { + log.Error(err) + return + } + + log.Infof("Persisting miners") + + miners := map[minerKey]*minerInfo{} + + for addr, m := range actors { + for actor, c := range m { + if actor.Code != actors2.StorageMinerCodeCid { + continue + } + + miners[minerKey{ + addr: addr, + act: actor, + stateroot: c, + }] = &minerInfo{} + } + } + + par(50, kvmaparr(miners), func(it func() (minerKey, *minerInfo)) { + k, info := it() + + astb, err := api.ChainReadObj(ctx, k.act.Head) + if err != nil { + log.Error(err) + return + } + + if err := info.state.UnmarshalCBOR(bytes.NewReader(astb)); err != nil { + log.Error(err) + return + } + + ib, err := api.ChainReadObj(ctx, info.state.Info) + if err != nil { + log.Error(err) + return + } + + if err := info.info.UnmarshalCBOR(bytes.NewReader(ib)); err != nil { + log.Error(err) + return + } + }) + + if err := st.storeMiners(miners); err != nil { + log.Error(err) + return + } + + log.Infof("Persisting headers") + if err := st.storeHeaders(toSync, true); err != nil { + log.Error(err) + return + } + + log.Infof("Getting messages") + + msgs, incls := fetchMessages(ctx, api, toSync) + + if err := st.storeMessages(msgs); err != nil { + log.Error(err) + return + } + + if err := st.storeMsgInclusions(incls); err != nil { + log.Error(err) + return + } + + log.Infof("Resolving addresses") + + for _, message := range msgs { + addresses[message.To] = address.Undef + addresses[message.From] = address.Undef + } + + par(50, kmaparr(addresses), func(addr address.Address) { + raddr, err := api.StateLookupID(ctx, addr, nil) + if err != nil { + log.Warn(err) + return + } + alk.Lock() + addresses[addr] = raddr + alk.Unlock() + }) + + if err := st.storeAddressMap(addresses); err != nil { + log.Error(err) + return + } + + log.Infof("Sync done") +} + +func fetchMessages(ctx context.Context, api api.FullNode, toSync map[cid.Cid]*types.BlockHeader) (map[cid.Cid]*types.Message, map[cid.Cid][]cid.Cid) { + var lk sync.Mutex + messages := map[cid.Cid]*types.Message{} + inclusions := map[cid.Cid][]cid.Cid{} // block -> msgs + + par(50, maparr(toSync), func(header *types.BlockHeader) { + msgs, err := api.ChainGetBlockMessages(ctx, header.Cid()) + if err != nil { + log.Error(err) + return + } + + vmm := make([]*types.Message, 0, len(msgs.Cids)) + for _, m := range msgs.BlsMessages { + vmm = append(vmm, m) + } + + for _, m := range msgs.SecpkMessages { + vmm = append(vmm, &m.Message) + } + + lk.Lock() + for _, message := range vmm { + messages[message.Cid()] = message + inclusions[header.Cid()] = append(inclusions[header.Cid()], message.Cid()) + } + lk.Unlock() + }) + + return messages, inclusions +} diff --git a/cmd/lotus-chainwatch/templates.go b/cmd/lotus-chainwatch/templates.go new file mode 100644 index 000000000..02e85d6b0 --- /dev/null +++ b/cmd/lotus-chainwatch/templates.go @@ -0,0 +1,253 @@ +package main + +import ( + "fmt" + "html/template" + "net/http" + "os" + "path/filepath" + + rice "github.com/GeertJohan/go.rice" + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" +) + +type handler struct { + api api.FullNode + st *storage + site *rice.Box + assets http.Handler + + templates map[string]*template.Template +} + +func newHandler(api api.FullNode, st *storage) (*handler, error) { + h := &handler{ + api: api, + st: st, + site: rice.MustFindBox("site"), + + templates: map[string]*template.Template{}, + } + h.assets = http.FileServer(h.site.HTTPBox()) + + funcs := template.FuncMap{ + "count": h.count, + "countCol": h.countCol, + "sum": h.sum, + "netPower": h.netPower, + "queryNum": h.queryNum, + "sizeStr": sizeStr, + "strings": h.strings, + "messages": h.messages, + + "param": func(string) string { return "" }, // replaced in request handler + } + + base := template.New("") + + base.Funcs(funcs) + + return h, h.site.Walk("", func(path string, info os.FileInfo, err error) error { + if filepath.Ext(path) != ".html" { + return nil + } + if err != nil { + return err + } + log.Info(path) + + h.templates["/"+path], err = base.New(path).Parse(h.site.MustString(path)) + return err + }) +} + +func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h, err := newHandler(h.api, h.st) // for faster dev + if err != nil { + log.Error(err) + return + } + + p := r.URL.Path + if p == "/" { + p = "/index.html" + } + + t, ok := h.templates[p] + if !ok { + h.assets.ServeHTTP(w, r) + return + } + + t, err = t.Clone() + if err != nil { + log.Error(err) + return + } + + t.Funcs(map[string]interface{}{ + "param": r.FormValue, + }) + + if err := t.Execute(w, nil); err != nil { + log.Errorf("%+v", err) + return + } + + log.Info(r.URL.Path) +} + +// Template funcs + +func (h *handler) count(table string, filters ...string) (int, error) { + // explicitly not caring about sql injection too much, this doesn't take user input + + filts := "" + if len(filters) > 0 { + filts = " where " + for _, filter := range filters { + filts += filter + " and " + } + filts = filts[:len(filts)-5] + } + + var c int + err := h.st.db.QueryRow("select count(1) from " + table + filts).Scan(&c) + if err != nil { + return 0, err + } + + return c, nil +} + +func (h *handler) countCol(table string, col string, filters ...string) (int, error) { + // explicitly not caring about sql injection too much, this doesn't take user input + + filts := "" + if len(filters) > 0 { + filts = " where " + for _, filter := range filters { + filts += filter + " and " + } + filts = filts[:len(filts)-5] + } + + var c int + err := h.st.db.QueryRow("select count(distinct " + col + ") from " + table + filts).Scan(&c) + if err != nil { + return 0, err + } + + return c, nil +} + +func (h *handler) sum(table string, col string) (types.BigInt, error) { + return h.queryNum("select sum(cast(" + col + " as bigint)) from " + table) +} + +func (h *handler) netPower(slashFilt string) (types.BigInt, error) { + if slashFilt != "" { + slashFilt = " where " + slashFilt + } + return h.queryNum(`select sum(power) from ( + select miner_heads.power, miner_heads.slashed_at, max(height) from miner_heads + inner join blocks b on miner_heads.stateroot = b.parentStateRoot + group by miner_heads.addr)` + slashFilt) +} + +func (h *handler) queryNum(q string, p ...interface{}) (types.BigInt, error) { + // explicitly not caring about sql injection too much, this doesn't take user input + + var c string + err := h.st.db.QueryRow(q, p...).Scan(&c) + if err != nil { + log.Error("qnum ", q, p, err) + return types.NewInt(0), err + } + + i := types.NewInt(0) + _, ok := i.SetString(c, 10) + if !ok { + return types.NewInt(0), xerrors.New("num parse error: " + c) + } + return i, nil +} + +var units = []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB"} + +func sizeStr(size types.BigInt) string { + size = types.BigMul(size, types.NewInt(100)) + i := 0 + for types.BigCmp(size, types.NewInt(102400)) >= 0 && i < len(units)-1 { + size = types.BigDiv(size, types.NewInt(1024)) + i++ + } + return fmt.Sprintf("%s.%s %s", types.BigDiv(size, types.NewInt(100)), types.BigMod(size, types.NewInt(100)), units[i]) +} + +func (h *handler) strings(table string, col string, filter string, args ...interface{}) (out []string, err error) { + if len(filter) > 0 { + filter = " where " + filter + } + log.Info("strings qstr ", "select "+col+" from "+table+filter) + rws, err := h.st.db.Query("select "+col+" from "+table+filter, args...) + if err != nil { + return nil, err + } + for rws.Next() { + var r string + if err := rws.Scan(&r); err != nil { + return nil, err + } + out = append(out, r) + } + + return +} + +func (h *handler) messages(filter string, args ...interface{}) (out []types.Message, err error) { + if len(filter) > 0 { + filter = " where " + filter + } + + rws, err := h.st.db.Query("select * from messages "+filter, args...) + if err != nil { + return nil, err + } + for rws.Next() { + var r types.Message + var cs string + + if err := rws.Scan( + &cs, + &r.From, + &r.To, + &r.Nonce, + &r.Value, + &r.GasPrice, + &r.GasLimit, + &r.Method, + &r.Params, + ); err != nil { + return nil, err + } + + c, err := cid.Parse(cs) + if err != nil { + return nil, err + } + if c != r.Cid() { + log.Warn("msg cid doesn't match") + } + + out = append(out, r) + } + + return +} + +var _ http.Handler = &handler{} diff --git a/cmd/lotus-chainwatch/utils.go b/cmd/lotus-chainwatch/utils.go new file mode 100644 index 000000000..a5c821683 --- /dev/null +++ b/cmd/lotus-chainwatch/utils.go @@ -0,0 +1,85 @@ +package main + +import ( + "reflect" + "sync" +) + +func maparr(in interface{}) interface{} { + rin := reflect.ValueOf(in) + rout := reflect.MakeSlice(reflect.SliceOf(rin.Type().Elem()), rin.Len(), rin.Len()) + var i int + + it := rin.MapRange() + for it.Next() { + rout.Index(i).Set(it.Value()) + i++ + } + + return rout.Interface() +} + +func kmaparr(in interface{}) interface{} { + rin := reflect.ValueOf(in) + rout := reflect.MakeSlice(reflect.SliceOf(rin.Type().Key()), rin.Len(), rin.Len()) + var i int + + it := rin.MapRange() + for it.Next() { + rout.Index(i).Set(it.Key()) + i++ + } + + return rout.Interface() +} + +// map[k]v => []func() (k, v) +func kvmaparr(in interface{}) interface{} { + rin := reflect.ValueOf(in) + + t := reflect.FuncOf([]reflect.Type{}, []reflect.Type{ + rin.Type().Key(), + rin.Type().Elem(), + }, false) + + rout := reflect.MakeSlice(reflect.SliceOf(t), rin.Len(), rin.Len()) + var i int + + it := rin.MapRange() + for it.Next() { + k := it.Key() + v := it.Value() + + rout.Index(i).Set(reflect.MakeFunc(t, func(args []reflect.Value) (results []reflect.Value) { + return []reflect.Value{k, v} + })) + i++ + } + + return rout.Interface() +} + +func par(concurrency int, arr interface{}, f interface{}) { + throttle := make(chan struct{}, concurrency) + var wg sync.WaitGroup + + varr := reflect.ValueOf(arr) + l := varr.Len() + + rf := reflect.ValueOf(f) + + wg.Add(l) + for i := 0; i < l; i++ { + throttle <- struct{}{} + + go func(i int) { + defer wg.Done() + defer func() { + <-throttle + }() + rf.Call([]reflect.Value{varr.Index(i)}) + }(i) + } + + wg.Wait() +} diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 9a3944417..735a0d21d 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -56,6 +56,10 @@ var runCmd = &cli.Command{ return err } + if v.APIVersion != build.APIVersion { + return xerrors.Errorf("lotus-daemon API version doesn't match: local: ", api.Version{APIVersion: build.APIVersion}) + } + storageRepoPath := cctx.String(FlagStorageRepo) r, err := repo.NewFS(storageRepoPath) if err != nil { diff --git a/go.mod b/go.mod index d3e0ea533..37661cdc8 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/GeertJohan/go.rice v1.0.0 github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect - github.com/btcsuite/btcd v0.0.0-20190807005414-4063feeff79a // indirect github.com/fatih/color v1.7.0 // indirect github.com/filecoin-project/chain-validation v0.0.0-20191106200742-11986803c0f7 github.com/filecoin-project/filecoin-ffi v0.0.0-00010101000000-000000000000 @@ -18,7 +17,7 @@ require ( github.com/go-ole/go-ole v1.2.4 // indirect github.com/google/go-cmp v0.3.1 // indirect github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect - github.com/gorilla/websocket v1.4.0 + github.com/gorilla/websocket v1.4.1 github.com/hashicorp/go-multierror v1.0.0 github.com/hashicorp/golang-lru v0.5.3 github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e @@ -47,12 +46,11 @@ require ( github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb github.com/ipld/go-ipld-prime v0.0.2-0.20191025154717-8dff1cbec43b github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 - github.com/libp2p/go-eventbus v0.1.0 // indirect github.com/libp2p/go-libp2p v0.3.0 github.com/libp2p/go-libp2p-circuit v0.1.1 github.com/libp2p/go-libp2p-connmgr v0.1.0 - github.com/libp2p/go-libp2p-core v0.2.2 - github.com/libp2p/go-libp2p-discovery v0.1.0 + github.com/libp2p/go-libp2p-core v0.2.4 + github.com/libp2p/go-libp2p-discovery v0.2.0 github.com/libp2p/go-libp2p-host v0.1.0 github.com/libp2p/go-libp2p-kad-dht v0.1.1 github.com/libp2p/go-libp2p-mplex v0.2.1 @@ -60,30 +58,30 @@ require ( github.com/libp2p/go-libp2p-peerstore v0.1.3 github.com/libp2p/go-libp2p-pnet v0.1.0 github.com/libp2p/go-libp2p-protocol v0.1.0 - github.com/libp2p/go-libp2p-pubsub v0.1.0 + github.com/libp2p/go-libp2p-pubsub v0.2.3 github.com/libp2p/go-libp2p-quic-transport v0.1.1 github.com/libp2p/go-libp2p-record v0.1.1 github.com/libp2p/go-libp2p-routing-helpers v0.1.0 github.com/libp2p/go-libp2p-secio v0.2.0 - github.com/libp2p/go-libp2p-swarm v0.2.1 // indirect github.com/libp2p/go-libp2p-tls v0.1.0 github.com/libp2p/go-libp2p-yamux v0.2.1 github.com/libp2p/go-maddr-filter v0.0.5 + github.com/libp2p/go-ws-transport v0.1.2 // indirect github.com/mattn/go-isatty v0.0.9 // indirect github.com/mattn/go-runewidth v0.0.4 // indirect + github.com/mattn/go-sqlite3 v1.12.0 github.com/miekg/dns v1.1.16 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 - github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 + github.com/minio/sha256-simd v0.1.1 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-base32 v0.0.3 - github.com/multiformats/go-multiaddr v0.0.4 - github.com/multiformats/go-multiaddr-dns v0.0.3 - github.com/multiformats/go-multiaddr-net v0.0.1 + github.com/multiformats/go-multiaddr v0.1.1 + github.com/multiformats/go-multiaddr-dns v0.2.0 + github.com/multiformats/go-multiaddr-net v0.1.0 github.com/multiformats/go-multihash v0.0.9 github.com/onsi/ginkgo v1.9.0 // indirect github.com/onsi/gomega v1.6.0 // indirect github.com/opentracing/opentracing-go v1.1.0 - github.com/pkg/errors v0.8.1 github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a github.com/smartystreets/assertions v1.0.1 // indirect github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect @@ -92,16 +90,15 @@ require ( github.com/whyrusleeping/cbor-gen v0.0.0-20191116002219-891f55cd449d github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d - go.opencensus.io v0.22.0 + go.opencensus.io v0.22.1 go.uber.org/dig v1.7.0 // indirect go.uber.org/fx v1.9.0 go.uber.org/goleak v0.10.0 // indirect go.uber.org/multierr v1.1.0 go.uber.org/zap v1.10.0 go4.org v0.0.0-20190313082347-94abd6928b1d // indirect - golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 // indirect golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 - golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 + golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 google.golang.org/api v0.9.0 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 diff --git a/go.sum b/go.sum index dc802c681..c376e7bba 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= -github.com/btcsuite/btcd v0.0.0-20190807005414-4063feeff79a h1:We35J+0yvVFrEXbtViYUW8H/wNOhqjIF3PsrW4yYmGw= -github.com/btcsuite/btcd v0.0.0-20190807005414-4063feeff79a/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= +github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 h1:A/EVblehb75cUgXA5njHPn0kLAsykn6mJGz7rnmW5W0= +github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= @@ -98,7 +98,12 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -126,6 +131,8 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE= @@ -264,6 +271,7 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -297,6 +305,8 @@ github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3Pt github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro= github.com/libp2p/go-libp2p-blankhost v0.1.3 h1:0KycuXvPDhmehw0ASsg+s1o3IfXgCUDqfzAl94KEBOg= github.com/libp2p/go-libp2p-blankhost v0.1.3/go.mod h1:KML1//wiKR8vuuJO0y3LUd1uLv+tlkGTAr3jC0S5cLg= +github.com/libp2p/go-libp2p-blankhost v0.1.4 h1:I96SWjR4rK9irDHcHq3XHN6hawCRTPUADzkJacgZLvk= +github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU= github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= github.com/libp2p/go-libp2p-circuit v0.1.1 h1:eopfG9fAg6rEHWQO1TSrLosXDgYbbbu/RTva/tBANus= github.com/libp2p/go-libp2p-circuit v0.1.1/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= @@ -310,11 +320,15 @@ github.com/libp2p/go-libp2p-core v0.0.6/go.mod h1:0d9xmaYAVY5qmbp/fcgxHT3ZJsLjYe github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv3j7yRXjO77xSI= github.com/libp2p/go-libp2p-core v0.2.2 h1:Sv1ggdoMx9c7v7FOFkR7agraHCnAgqYsXrU1ARSRUMs= github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0= +github.com/libp2p/go-libp2p-core v0.2.4 h1:Et6ykkTwI6PU44tr8qUF9k43vP0aduMNniShAbUJJw8= +github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE= github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= github.com/libp2p/go-libp2p-discovery v0.1.0 h1:j+R6cokKcGbnZLf4kcNwpx6mDEUPF3N6SrqMymQhmvs= github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g= +github.com/libp2p/go-libp2p-discovery v0.2.0 h1:1p3YSOq7VsgaL+xVHPi8XAmtGyas6D2J6rWBEfz/aiY= +github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= github.com/libp2p/go-libp2p-host v0.0.1/go.mod h1:qWd+H1yuU0m5CwzAkvbSjqKairayEHdR5MMl7Cwa7Go= github.com/libp2p/go-libp2p-host v0.1.0 h1:OZwENiFm6JOK3YR5PZJxkXlJE8a5u8g4YvAUrEV2MjM= github.com/libp2p/go-libp2p-host v0.1.0/go.mod h1:5+fWuLbDn8OxoxPN3CV0vsLe1hAKScSMbT84qRfxum8= @@ -347,8 +361,8 @@ github.com/libp2p/go-libp2p-pnet v0.1.0/go.mod h1:ZkyZw3d0ZFOex71halXRihWf9WH/j3 github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= github.com/libp2p/go-libp2p-protocol v0.1.0 h1:HdqhEyhg0ToCaxgMhnOmUO8snQtt/kQlcjVk3UoJU3c= github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= -github.com/libp2p/go-libp2p-pubsub v0.1.0 h1:SmQeMa7IUv5vadh0fYgYsafWCBA1sCy5d/68kIYqGcU= -github.com/libp2p/go-libp2p-pubsub v0.1.0/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= +github.com/libp2p/go-libp2p-pubsub v0.2.3 h1:qJRnRnM7Z4xnHb4i6EBb3DKQXRPgtFWlKP4AmfJudLQ= +github.com/libp2p/go-libp2p-pubsub v0.2.3/go.mod h1:Jscj3fk23R5mCrOwb625xjVs5ZEyTZcx/OlTwMDqU+g= github.com/libp2p/go-libp2p-quic-transport v0.1.1 h1:MFMJzvsxIEDEVKzO89BnB/FgvMj9WI4GDGUW2ArDPUA= github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= @@ -364,8 +378,8 @@ github.com/libp2p/go-libp2p-secio v0.2.0 h1:ywzZBsWEEz2KNTn5RtzauEDq5RFEefPsttXY github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4= github.com/libp2p/go-libp2p-swarm v0.2.0/go.mod h1:x07b4zkMFo2EvgPV2bMTlNmdQc8i+74Jjio7xGvsTgU= -github.com/libp2p/go-libp2p-swarm v0.2.1 h1:9A8oQqPIZvbaRyrjViHeDYS7fE7fNtP7BRWdJrBHbe8= -github.com/libp2p/go-libp2p-swarm v0.2.1/go.mod h1:x07b4zkMFo2EvgPV2bMTlNmdQc8i+74Jjio7xGvsTgU= +github.com/libp2p/go-libp2p-swarm v0.2.2 h1:T4hUpgEs2r371PweU3DuH7EOmBIdTBCwWs+FLcgx3bQ= +github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaTNyBcHImCxRpPKU= github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.4/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= @@ -393,6 +407,8 @@ github.com/libp2p/go-nat v0.0.3 h1:l6fKV+p0Xa354EqQOQP+d8CivdLM4kl5GxC1hSc/UeI= github.com/libp2p/go-nat v0.0.3/go.mod h1:88nUEt0k0JD45Bk93NIwDqjlhiOwOoV36GchpcVc1yI= github.com/libp2p/go-openssl v0.0.2 h1:9pP2d3Ubaxkv7ZisLjx9BFwgOGnQdQYnfcH29HNY3ls= github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0= +github.com/libp2p/go-openssl v0.0.3 h1:wjlG7HvQkt4Fq4cfH33Ivpwp0omaElYEi9z26qaIkIk= +github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FWpFLw= github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= github.com/libp2p/go-reuseport-transport v0.0.2 h1:WglMwyXyBu61CMkjCCtnmqNqnjib0GIEjMiHTwR/KN4= @@ -402,9 +418,13 @@ github.com/libp2p/go-stream-muxer-multistream v0.2.0 h1:714bRJ4Zy9mdhyTLJ+ZKiROm github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc= github.com/libp2p/go-tcp-transport v0.1.0 h1:IGhowvEqyMFknOar4FWCKSWE0zL36UFKQtiRQD60/8o= github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc= +github.com/libp2p/go-tcp-transport v0.1.1 h1:yGlqURmqgNA2fvzjSgZNlHcsd/IulAnKM8Ncu+vlqnw= +github.com/libp2p/go-tcp-transport v0.1.1/go.mod h1:3HzGvLbx6etZjnFlERyakbaYPdfjg2pWP97dFZworkY= github.com/libp2p/go-testutil v0.1.0/go.mod h1:81b2n5HypcVyrCg/MJx4Wgfp/VHojytjVe/gLzZ2Ehc= github.com/libp2p/go-ws-transport v0.1.0 h1:F+0OvvdmPTDsVc4AjPHjV7L7Pk1B7D5QwtDcKE2oag4= github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw7yT74kj3raBFuo= +github.com/libp2p/go-ws-transport v0.1.2 h1:VnxQcLfSGtqupqPpBNu8fUiCv+IN1RJ2BcVqQEM+z8E= +github.com/libp2p/go-ws-transport v0.1.2/go.mod h1:dsh2Ld8F+XNmzpkaAijmg5Is+e9l6/1tK/6VFOdN69Y= github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.2.3 h1:xX8A36vpXb59frIzWFdEgptLMsOANMFq2K7fPRlunYI= github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= @@ -427,6 +447,8 @@ github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-sqlite3 v1.12.0 h1:u/x3mp++qUxvYfulZ4HKOvVO0JWhk7HtE8lWhbGz/Do= +github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -439,6 +461,8 @@ github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+ github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -452,18 +476,26 @@ github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lg github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.0.4 h1:WgMSI84/eRLdbptXMkMWDXPjPq7SPLIgGUVm2eroyU4= github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= +github.com/multiformats/go-multiaddr v0.1.0/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= +github.com/multiformats/go-multiaddr v0.1.1 h1:rVAztJYMhCQ7vEFr8FvxW3mS+HF2eY/oPbOMeS0ZDnE= +github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= -github.com/multiformats/go-multiaddr-dns v0.0.3 h1:P19q/k9jwmtgh+qXFkKfgFM7rCg/9l5AVqh7VNxSXhs= -github.com/multiformats/go-multiaddr-dns v0.0.3/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= +github.com/multiformats/go-multiaddr-dns v0.2.0 h1:YWJoIDwLePniH7OU5hBnDZV6SWuvJqJ0YtN6pLeH9zA= +github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= github.com/multiformats/go-multiaddr-fmt v0.0.1 h1:5YjeOIzbX8OTKVaN72aOzGIYW7PnrZrnkDyOfAWRSMA= github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q= +github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= +github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multiaddr-net v0.0.1 h1:76O59E3FavvHqNg7jvzWzsPSW5JSi/ek0E4eiDVbg9g= github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d1tnvM7ihcFwHMSstNAVUU= +github.com/multiformats/go-multiaddr-net v0.1.0 h1:ZepO8Ezwovd+7b5XPPDhQhayk1yt0AJpzQBpq9fejx4= +github.com/multiformats/go-multiaddr-net v0.1.0/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= +github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.9 h1:aoijQXYYl7Xtb2pUUP68R+ys1TlnlR3eX6wmozr0Hp4= github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= @@ -580,8 +612,8 @@ github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7V github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50= +go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/dig v1.7.0 h1:E5/L92iQTNJTjfgJF2KgU+/JpMaiuvK2DHLBj0+kSZk= @@ -610,8 +642,8 @@ golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 h1:Gv7RPwsi3eZ2Fgewe3CBsuOebPwO27PoXzRpJPsvSSM= -golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -631,7 +663,6 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -682,6 +713,7 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -692,6 +724,8 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= diff --git a/lib/padreader/padreader_test.go b/lib/padreader/padreader_test.go index 551393b4e..712317e11 100644 --- a/lib/padreader/padreader_test.go +++ b/lib/padreader/padreader_test.go @@ -1,8 +1,9 @@ package padreader import ( - "gotest.tools/assert" "testing" + + "gotest.tools/assert" ) func TestComputePaddedSize(t *testing.T) { diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index ea6e43977..fae3db3b9 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -154,14 +154,14 @@ func (sb *SectorBuilder) AcquireSectorId() (uint64, error) { } func (sb *SectorBuilder) AddPiece(pieceSize uint64, sectorId uint64, file io.Reader, existingPieceSizes []uint64) (PublicPieceInfo, error) { + ret := sb.RateLimit() + defer ret() + f, werr, err := toReadableFile(file, int64(pieceSize)) if err != nil { return PublicPieceInfo{}, err } - ret := sb.RateLimit() - defer ret() - stagedFile, err := sb.stagedSectorFile(sectorId) if err != nil { return PublicPieceInfo{}, err diff --git a/lotuspond/api.go b/lotuspond/api.go index 2bd87c0b9..ef742b1a2 100644 --- a/lotuspond/api.go +++ b/lotuspond/api.go @@ -2,14 +2,15 @@ package main import ( "crypto/rand" - "github.com/filecoin-project/lotus/lib/jsonrpc" - "github.com/filecoin-project/lotus/node/repo" "io" "io/ioutil" "os" "sync" "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/lib/jsonrpc" + "github.com/filecoin-project/lotus/node/repo" ) type NodeState int diff --git a/lotuspond/outmux.go b/lotuspond/outmux.go index 713338d97..13b8899c0 100644 --- a/lotuspond/outmux.go +++ b/lotuspond/outmux.go @@ -3,12 +3,13 @@ package main import ( "bufio" "fmt" - "github.com/gorilla/websocket" - "github.com/opentracing/opentracing-go/log" "io" "net/http" "strings" "sync" + + "github.com/gorilla/websocket" + "github.com/opentracing/opentracing-go/log" ) type outmux struct { diff --git a/miner/miner.go b/miner/miner.go index aea344cc7..6da666002 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -13,7 +13,6 @@ import ( "github.com/filecoin-project/lotus/chain/types" logging "github.com/ipfs/go-log" - "github.com/pkg/errors" "go.opencensus.io/trace" "golang.org/x/xerrors" ) @@ -242,7 +241,7 @@ func (m *Miner) mineOne(ctx context.Context, addr address.Address, base *MiningB log.Debugw("attempting to mine a block", "tipset", types.LogCids(base.ts.Cids())) ticket, err := m.computeTicket(ctx, addr, base) if err != nil { - return nil, errors.Wrap(err, "scratching ticket failed") + return nil, xerrors.Errorf("scratching ticket failed: %w", err) } win, proof, err := gen.IsRoundWinner(ctx, base.ts, int64(base.ts.Height()+base.nullRounds+1), addr, m.epp, m.api) @@ -257,7 +256,7 @@ func (m *Miner) mineOne(ctx context.Context, addr address.Address, base *MiningB b, err := m.createBlock(base, addr, ticket, proof) if err != nil { - return nil, errors.Wrap(err, "failed to create block") + return nil, xerrors.Errorf("failed to create block: %w", err) } log.Infow("mined new block", "cid", b.Cid()) @@ -313,7 +312,7 @@ func (m *Miner) createBlock(base *MiningBase, addr address.Address, ticket *type pending, err := m.api.MpoolPending(context.TODO(), base.ts) if err != nil { - return nil, errors.Wrapf(err, "failed to get pending messages") + return nil, xerrors.Errorf("failed to get pending messages: %w", err) } msgs, err := selectMessages(context.TODO(), m.api.StateGetActor, base, pending) @@ -363,7 +362,7 @@ func selectMessages(ctx context.Context, al actorLookup, base *MiningBase, msgs } if msg.Message.Nonce < inclNonces[from] { - log.Warnf("message in mempool has already used nonce (%d < %d) %s", msg.Message.Nonce, inclNonces[from], msg.Cid()) + log.Warnf("message in mempool has already used nonce (%d < %d), from %s, to %s, %s", msg.Message.Nonce, inclNonces[from], msg.Message.From, msg.Message.To, msg.Cid()) continue } diff --git a/node/builder.go b/node/builder.go index 3512354ec..412730fab 100644 --- a/node/builder.go +++ b/node/builder.go @@ -311,6 +311,9 @@ func ConfigFullNode(c interface{}) Option { return Options( ConfigCommon(&cfg.Common), Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)), + If(cfg.Metrics.PubsubTracing, + Override(new(*pubsub.PubSub), lp2p.GossipSub(lp2p.PubsubTracer())), + ), ) } diff --git a/node/config/def.go b/node/config/def.go index d704b428b..9391708a6 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -41,7 +41,8 @@ type Libp2p struct { // // Full Node type Metrics struct { - Nickname string + Nickname string + PubsubTracing bool } // // Storage Miner diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index 66cdf873b..db318a083 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -6,6 +6,7 @@ import ( "go.uber.org/fx" "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" @@ -53,3 +54,7 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message) (*t func (a *MpoolAPI) MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) { return a.Mpool.GetNonce(addr) } + +func (a *MpoolAPI) MpoolSub(ctx context.Context) (<-chan api.MpoolUpdate, error) { + return a.Mpool.Updates(ctx) +} diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 844220475..a51276d6c 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -154,6 +154,15 @@ func (a *StateAPI) StateGetActor(ctx context.Context, actor address.Address, ts return state.GetActor(actor) } +func (a *StateAPI) StateLookupID(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) { + state, err := a.stateForTs(ctx, ts) + if err != nil { + return address.Undef, err + } + + return state.LookupID(addr) +} + func (a *StateAPI) StateReadState(ctx context.Context, act *types.Actor, ts *types.TipSet) (*api.ActorState, error) { state, err := a.stateForTs(ctx, ts) if err != nil { @@ -209,6 +218,10 @@ func (a *StateAPI) StateWaitMsg(ctx context.Context, msg cid.Cid) (*api.MsgWait, }, nil } +func (a *StateAPI) StateGetReceipt(ctx context.Context, msg cid.Cid, ts *types.TipSet) (*types.MessageReceipt, error) { + return a.StateManager.GetReceipt(ctx, msg, ts) +} + func (a *StateAPI) StateListMiners(ctx context.Context, ts *types.TipSet) ([]address.Address, error) { var state actors.StoragePowerState if _, err := a.StateManager.LoadActorState(ctx, actors.StoragePowerAddress, &state, ts); err != nil { @@ -294,3 +307,54 @@ func (a *StateAPI) StateMarketDeals(ctx context.Context, ts *types.TipSet) (map[ func (a *StateAPI) StateMarketStorageDeal(ctx context.Context, dealId uint64, ts *types.TipSet) (*actors.OnChainDeal, error) { return stmgr.GetStorageDeal(ctx, a.StateManager, dealId, ts) } + +func (a *StateAPI) StateChangedActors(ctx context.Context, old cid.Cid, new cid.Cid) (map[string]types.Actor, error) { + cst := hamt.CSTFromBstore(a.Chain.Blockstore()) + + nh, err := hamt.LoadNode(ctx, cst, new) + if err != nil { + return nil, err + } + + oh, err := hamt.LoadNode(ctx, cst, old) + if err != nil { + return nil, err + } + + out := map[string]types.Actor{} + + err = nh.ForEach(ctx, func(k string, nval interface{}) error { + ncval := nval.(*cbg.Deferred) + var act types.Actor + + var ocval cbg.Deferred + + switch err := oh.Find(ctx, k, &ocval); err { + case nil: + if bytes.Equal(ocval.Raw, ncval.Raw) { + return nil // not changed + } + fallthrough + case hamt.ErrNotFound: + if err := act.UnmarshalCBOR(bytes.NewReader(ncval.Raw)); err != nil { + return err + } + + addr, err := address.NewFromBytes([]byte(k)) + if err != nil { + return xerrors.Errorf("address in state tree was not valid: %w", err) + } + + out[addr.String()] = act + default: + return err + } + + return nil + }) + if err != nil { + return nil, err + } + + return out, nil +} diff --git a/node/impl/full/sync.go b/node/impl/full/sync.go index b06d51da4..c84957ad7 100644 --- a/node/impl/full/sync.go +++ b/node/impl/full/sync.go @@ -54,7 +54,7 @@ func (a *SyncAPI) SyncSubmitBlock(ctx context.Context, blk *types.BlockMsg) erro } if err := a.Syncer.ValidateMsgMeta(fb); err != nil { - xerrors.Errorf("provided messages did not match block: %w", err) + return xerrors.Errorf("provided messages did not match block: %w", err) } ts, err := types.NewTipSet([]*types.BlockHeader{blk.Header}) @@ -73,3 +73,7 @@ func (a *SyncAPI) SyncSubmitBlock(ctx context.Context, blk *types.BlockMsg) erro // TODO: anything else to do here? return a.PubSub.Publish("/fil/blocks", b) } + +func (a *SyncAPI) SyncIncomingBlocks(ctx context.Context) (<-chan *types.BlockHeader, error) { + return a.Syncer.IncomingBlocks(ctx) +} diff --git a/node/modules/chain.go b/node/modules/chain.go index 25f71c84a..b6fb29c8c 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -41,14 +41,17 @@ func ChainExchange(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, rt return exch } -func MessagePool(lc fx.Lifecycle, sm *stmgr.StateManager, ps *pubsub.PubSub) *chain.MessagePool { - mp := chain.NewMessagePool(sm, ps) +func MessagePool(lc fx.Lifecycle, sm *stmgr.StateManager, ps *pubsub.PubSub, ds dtypes.MetadataDS) (*chain.MessagePool, error) { + mp, err := chain.NewMessagePool(sm, ps, ds) + if err != nil { + return nil, xerrors.Errorf("constructing mpool: %w", err) + } lc.Append(fx.Hook{ OnStop: func(_ context.Context) error { return mp.Close() }, }) - return mp + return mp, nil } func ChainBlockstore(r repo.LockedRepo) (dtypes.ChainBlockstore, error) { diff --git a/node/modules/core.go b/node/modules/core.go index afc1b682a..5eab490eb 100644 --- a/node/modules/core.go +++ b/node/modules/core.go @@ -3,6 +3,9 @@ package modules import ( "context" "crypto/rand" + "io" + "io/ioutil" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" @@ -14,8 +17,6 @@ import ( "github.com/libp2p/go-libp2p-core/peerstore" record "github.com/libp2p/go-libp2p-record" "golang.org/x/xerrors" - "io" - "io/ioutil" ) var log = logging.Logger("modules") diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go index 3795666c4..657c714f1 100644 --- a/node/modules/lp2p/pubsub.go +++ b/node/modules/lp2p/pubsub.go @@ -1,21 +1,45 @@ package lp2p import ( + "context" + host "github.com/libp2p/go-libp2p-core/host" + peer "github.com/libp2p/go-libp2p-core/peer" pubsub "github.com/libp2p/go-libp2p-pubsub" + ma "github.com/multiformats/go-multiaddr" "go.uber.org/fx" "github.com/filecoin-project/lotus/node/modules/helpers" ) -func FloodSub(pubsubOptions ...pubsub.Option) interface{} { - return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host) (service *pubsub.PubSub, err error) { - return pubsub.NewFloodSub(helpers.LifecycleCtx(mctx, lc), host, pubsubOptions...) +type PubsubOpt func(host.Host) pubsub.Option + +func PubsubTracer() PubsubOpt { + return func(host host.Host) pubsub.Option { + pi, err := peer.AddrInfoFromP2pAddr(ma.StringCast("/ip4/147.75.67.199/tcp/4001/p2p/QmTd6UvR47vUidRNZ1ZKXHrAFhqTJAD27rKL9XYghEKgKX")) + if err != nil { + panic(err) + } + + tr, err := pubsub.NewRemoteTracer(context.TODO(), host, *pi) + if err != nil { + panic(err) + } + + return pubsub.WithEventTracer(tr) } } -func GossipSub(pubsubOptions ...pubsub.Option) interface{} { +func GossipSub(pubsubOptions ...PubsubOpt) interface{} { return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host) (service *pubsub.PubSub, err error) { - return pubsub.NewGossipSub(helpers.LifecycleCtx(mctx, lc), host, pubsubOptions...) + return pubsub.NewGossipSub(helpers.LifecycleCtx(mctx, lc), host, paresOpts(host, pubsubOptions)...) } } + +func paresOpts(host host.Host, in []PubsubOpt) []pubsub.Option { + out := make([]pubsub.Option, len(in)) + for k, v := range in { + out[k] = v(host) + } + return out +} diff --git a/node/options.go b/node/options.go index 9c8a79710..75a57cb9b 100644 --- a/node/options.go +++ b/node/options.go @@ -40,6 +40,12 @@ func ApplyIf(check func(s *Settings) bool, opts ...Option) Option { } } +func If(b bool, opts ...Option) Option { + return ApplyIf(func(s *Settings) bool { + return b + }, opts...) +} + // Override option changes constructor for a given type func Override(typ, constructor interface{}) Option { return func(s *Settings) error { diff --git a/storage/miner.go b/storage/miner.go index 3721be65c..206bbbe4d 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -2,6 +2,7 @@ package storage import ( "context" + "errors" "sync" "github.com/ipfs/go-cid" @@ -9,7 +10,7 @@ import ( "github.com/ipfs/go-datastore/namespace" logging "github.com/ipfs/go-log" "github.com/libp2p/go-libp2p-core/host" - "github.com/pkg/errors" + "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/address" @@ -49,9 +50,6 @@ type Miner struct { } type storageMinerApi interface { - // I think I want this... but this is tricky - //ReadState(ctx context.Context, addr address.Address) (????, error) - // Call a read only method on actors (no interaction with the chain required) StateCall(ctx context.Context, msg *types.Message, ts *types.TipSet) (*types.MessageReceipt, error) StateMinerWorker(context.Context, address.Address, *types.TipSet) (address.Address, error) @@ -59,7 +57,9 @@ type storageMinerApi interface { StateMinerSectors(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error) StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error) StateMinerSectorSize(context.Context, address.Address, *types.TipSet) (uint64, error) - StateWaitMsg(context.Context, cid.Cid) (*api.MsgWait, error) + StateWaitMsg(context.Context, cid.Cid) (*api.MsgWait, error) // TODO: removeme eventually + StateGetActor(ctx context.Context, actor address.Address, ts *types.TipSet) (*types.Actor, error) + StateGetReceipt(context.Context, cid.Cid, *types.TipSet) (*types.MessageReceipt, error) MpoolPushMessage(context.Context, *types.Message) (*types.SignedMessage, error) @@ -94,7 +94,7 @@ func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datasto func (m *Miner) Run(ctx context.Context) error { if err := m.runPreflightChecks(ctx); err != nil { - return errors.Wrap(err, "miner preflight checks failed") + return xerrors.Errorf("miner preflight checks failed: %w", err) } m.events = events.NewEvents(ctx, m.api) @@ -124,7 +124,7 @@ func (m *Miner) runPreflightChecks(ctx context.Context) error { has, err := m.api.WalletHas(ctx, worker) if err != nil { - return errors.Wrap(err, "failed to check wallet for worker key") + return xerrors.Errorf("failed to check wallet for worker key: %w", err) } if !has { diff --git a/storage/post.go b/storage/post.go index 072b9615c..628f7b94c 100644 --- a/storage/post.go +++ b/storage/post.go @@ -5,7 +5,6 @@ import ( ffi "github.com/filecoin-project/filecoin-ffi" "time" - "github.com/ipfs/go-cid" "go.opencensus.io/trace" "golang.org/x/xerrors" @@ -16,6 +15,8 @@ import ( "github.com/filecoin-project/lotus/lib/sectorbuilder" ) +const postMsgTimeout = 20 + func (m *Miner) beginPosting(ctx context.Context) { ts, err := m.api.ChainHead(context.TODO()) if err != nil { @@ -23,14 +24,14 @@ func (m *Miner) beginPosting(ctx context.Context) { return } - ppe, err := m.api.StateMinerProvingPeriodEnd(ctx, m.maddr, ts) + sppe, err := m.api.StateMinerProvingPeriodEnd(ctx, m.maddr, ts) if err != nil { - log.Errorf("failed to get proving period end for miner: %s", err) + log.Errorf("failed to get proving period end for miner (ts h: %d): %s", ts.Height(), err) return } - if ppe == 0 { - log.Errorf("Proving period end == 0") + if sppe == 0 { + log.Warn("Not proving yet") return } @@ -43,12 +44,16 @@ func (m *Miner) beginPosting(ctx context.Context) { // height needs to be +1, because otherwise we'd be trying to schedule PoSt // at current block height - ppe, _ = actors.ProvingPeriodEnd(ppe, ts.Height()+1) + ppe, _ := actors.ProvingPeriodEnd(sppe, ts.Height()+1) m.schedPost = ppe m.postLk.Unlock() - log.Infof("Scheduling post at height %d", ppe-build.PoStChallangeTime) + if build.PoStChallangeTime > ppe { + ppe = build.PoStChallangeTime + } + + log.Infof("Scheduling post at height %d (begin ts: %d, statePPE: %d)", ppe-build.PoStChallangeTime, ts.Height(), sppe) err = m.events.ChainAt(m.computePost(m.schedPost), func(ctx context.Context, ts *types.TipSet) error { // Revert // TODO: Cancel post log.Errorf("TODO: Cancel PoSt, re-run") @@ -114,7 +119,7 @@ type post struct { proof []byte // commit - smsg cid.Cid + smsg *types.SignedMessage } func (p *post) doPost(ctx context.Context) error { @@ -122,19 +127,19 @@ func (p *post) doPost(ctx context.Context) error { defer span.End() if err := p.preparePost(ctx); err != nil { - return err + return xerrors.Errorf("prepare: %w", err) } if err := p.runPost(ctx); err != nil { - return err + return xerrors.Errorf("run: %w", err) } if err := p.commitPost(ctx); err != nil { - return err + return xerrors.Errorf("commit: %w", err) } if err := p.waitCommit(ctx); err != nil { - return err + return xerrors.Errorf("wait: %w", err) } return nil @@ -147,7 +152,7 @@ func (p *post) preparePost(ctx context.Context) error { sset, err := p.m.api.StateMinerProvingSet(ctx, p.m.maddr, p.ts) if err != nil { - return xerrors.Errorf("failed to get proving set for miner: %w", err) + return xerrors.Errorf("failed to get proving set for miner (tsH: %d): %w", p.ts.Height(), err) } if len(sset) == 0 { log.Warn("empty proving set! (ts.H: %d)", p.ts.Height()) @@ -207,7 +212,7 @@ func (p *post) runPost(ctx context.Context) error { return nil } -func (p *post) commitPost(ctx context.Context) error { +func (p *post) commitPost(ctx context.Context) (err error) { ctx, span := trace.StartSpan(ctx, "storage.commitPost") defer span.End() @@ -235,12 +240,11 @@ func (p *post) commitPost(ctx context.Context) error { log.Info("mpush") - smsg, err := p.m.api.MpoolPushMessage(ctx, msg) - if err != nil { - return xerrors.Errorf("pushing message to mpool: %w", err) - } - p.smsg = smsg.Cid() - */ + p.smsg, err = p.m.api.MpoolPushMessage(ctx, msg) + if err != nil { + return xerrors.Errorf("pushing message to mpool: %w", err) + } + */ return nil } @@ -249,18 +253,24 @@ func (p *post) waitCommit(ctx context.Context) error { ctx, span := trace.StartSpan(ctx, "storage.waitPost") defer span.End() - log.Infof("Waiting for post %s to appear on chain", p.smsg) + log.Infof("Waiting for post %s to appear on chain", p.smsg.Cid()) - // make sure it succeeds... - rec, err := p.m.api.StateWaitMsg(ctx, p.smsg) + err := p.m.events.CalledMsg(ctx, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (more bool, err error) { + if rec.ExitCode != 0 { + log.Warnf("SubmitPoSt EXIT: %d", rec.ExitCode) + } + + log.Infof("Post made it on chain! (height=%d)", ts.Height()) + + return false, nil + }, func(ctx context.Context, ts *types.TipSet) error { + log.Warn("post message reverted") + return nil + }, 3, postMsgTimeout, p.smsg) if err != nil { - return err + return xerrors.Errorf("waiting for post to appear on chain: %w", err) } - if rec.Receipt.ExitCode != 0 { - log.Warnf("SubmitPoSt EXIT: %d", rec.Receipt.ExitCode) - // TODO: Do something - } - log.Infof("Post made it on chain! (height=%d)", rec.TipSet.Height()) + return nil } @@ -279,19 +289,13 @@ func (m *Miner) computePost(ppe uint64) func(ctx context.Context, ts *types.TipS ppe: ppe, ts: ts, }).doPost(ctx) + + m.scheduleNextPost(ppe + build.ProvingPeriodDuration) + if err != nil { return xerrors.Errorf("doPost: %w", err) } - m.scheduleNextPost(ppe + build.ProvingPeriodDuration) return nil } } - -func sectorIdList(si []*api.ChainSectorInfo) []uint64 { - out := make([]uint64, len(si)) - for i, s := range si { - out[i] = s.SectorID - } - return out -} diff --git a/storage/sealing.go b/storage/sealing.go index abb831676..3e3c576f5 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -124,8 +124,27 @@ func (t *SectorInfo) rspco() sectorbuilder.RawSealPreCommitOutput { return out } -func (m *Miner) sectorStateLoop(ctx context.Context) { - // TODO: restore state +func (m *Miner) sectorStateLoop(ctx context.Context) error { + toRestart, err := m.ListSectors() + if err != nil { + return err + } + + go func() { + for _, si := range toRestart { + select { + case m.sectorUpdated <- sectorUpdate{ + newState: si.State, + id: si.SectorID, + err: nil, + mut: nil, + }: + case <-ctx.Done(): + log.Warn("didn't restart processing for all sectors: ", ctx.Err()) + return + } + } + }() go func() { defer log.Warn("quitting deal provider loop") @@ -142,6 +161,8 @@ func (m *Miner) sectorStateLoop(ctx context.Context) { } } }() + + return nil } func (m *Miner) onSectorIncoming(sector *SectorInfo) { diff --git a/storage/sector_states.go b/storage/sector_states.go index ec0810d26..ed5666285 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -3,7 +3,6 @@ package storage import ( "context" - "github.com/pkg/errors" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" @@ -206,7 +205,7 @@ func (m *Miner) committing(ctx context.Context, sector SectorInfo) (func(*Sector smsg, err := m.api.MpoolPushMessage(ctx, msg) if err != nil { - log.Error(errors.Wrap(err, "pushing message to mpool")) + log.Error(xerrors.Errorf("pushing message to mpool: %w", err)) } // TODO: Separate state before this wait, so we persist message cid? @@ -217,7 +216,7 @@ func (m *Miner) committing(ctx context.Context, sector SectorInfo) (func(*Sector } if mw.Receipt.ExitCode != 0 { - log.Errorf("UNHANDLED: submitting sector proof failed (t:%x; s:%x(%d); p:%x)", sector.Ticket.TicketBytes, sector.Seed.TicketBytes, sector.Seed.BlockHeight, params.Proof) + log.Errorf("UNHANDLED: submitting sector proof failed (exit=%d, msg=%s) (t:%x; s:%x(%d); p:%x)", mw.Receipt.ExitCode, smsg.Cid(), sector.Ticket.TicketBytes, sector.Seed.TicketBytes, sector.Seed.BlockHeight, params.Proof) return nil, xerrors.New("UNHANDLED: submitting sector proof failed") } diff --git a/storage/sectorblocks/blockstore.go b/storage/sectorblocks/blockstore.go index e28e750d0..afa0fcbdf 100644 --- a/storage/sectorblocks/blockstore.go +++ b/storage/sectorblocks/blockstore.go @@ -2,7 +2,8 @@ package sectorblocks import ( "context" - "github.com/ipfs/go-block-format" + + blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" blockstore "github.com/ipfs/go-ipfs-blockstore" )