From 97489f8bf9d70130a9899089a91f25ffc9e2b823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 13 Aug 2020 09:40:19 +0200 Subject: [PATCH 01/36] config: Set lower default max commit maxfee --- node/config/def.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/config/def.go b/node/config/def.go index 4fd5a5bd2..2e1243e5b 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -154,8 +154,8 @@ func DefaultStorageMiner() *StorageMiner { }, Fees: MinerFeeConfig{ - MaxPreCommitGasFee: types.FIL(types.FromFil(1)), - MaxCommitGasFee: types.FIL(types.FromFil(1)), + MaxPreCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), // 0.05 + MaxCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), MaxWindowPoStGasFee: types.FIL(types.FromFil(50)), }, From 066bac0607cf283609f7e162774d6e968c4e845b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 13 Aug 2020 10:51:13 +0200 Subject: [PATCH 02/36] miner: Add retrieval deal list to miner allinfo --- cmd/lotus-storage-miner/info_all.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/lotus-storage-miner/info_all.go b/cmd/lotus-storage-miner/info_all.go index 4b81bf37c..e951a6505 100644 --- a/cmd/lotus-storage-miner/info_all.go +++ b/cmd/lotus-storage-miner/info_all.go @@ -89,6 +89,11 @@ var infoAllCmd = &cli.Command{ return err } + fmt.Println("\n#: Retrieval Deals") + if err := retrievalDealsListCmd.Action(cctx); err != nil { + return err + } + fmt.Println("\n#: Sector List") if err := sectorsListCmd.Action(cctx); err != nil { return err From 1555984785221281bc89c6170dc1a58feb582152 Mon Sep 17 00:00:00 2001 From: yaohcn Date: Thu, 13 Aug 2020 17:31:18 +0800 Subject: [PATCH 03/36] change to RLock --- extern/sector-storage/stats.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extern/sector-storage/stats.go b/extern/sector-storage/stats.go index ee88898a4..be7cab73b 100644 --- a/extern/sector-storage/stats.go +++ b/extern/sector-storage/stats.go @@ -3,8 +3,8 @@ package sectorstorage import "github.com/filecoin-project/sector-storage/storiface" func (m *Manager) WorkerStats() map[uint64]storiface.WorkerStats { - m.sched.workersLk.Lock() - defer m.sched.workersLk.Unlock() + m.sched.workersLk.RLock() + defer m.sched.workersLk.RUnlock() out := map[uint64]storiface.WorkerStats{} @@ -22,8 +22,8 @@ func (m *Manager) WorkerStats() map[uint64]storiface.WorkerStats { } func (m *Manager) WorkerJobs() map[uint64][]storiface.WorkerJob { - m.sched.workersLk.Lock() - defer m.sched.workersLk.Unlock() + m.sched.workersLk.RLock() + defer m.sched.workersLk.RUnlock() out := map[uint64][]storiface.WorkerJob{} From 632fd36205b4be59417bc152bcc0fc6f305c2b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 13 Aug 2020 12:17:24 +0200 Subject: [PATCH 04/36] sealing sched: Fix deadlock in worker watcher --- extern/sector-storage/sched_watch.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/extern/sector-storage/sched_watch.go b/extern/sector-storage/sched_watch.go index d93cf1af3..2dd9875d7 100644 --- a/extern/sector-storage/sched_watch.go +++ b/extern/sector-storage/sched_watch.go @@ -87,11 +87,14 @@ func (sh *scheduler) runWorkerWatcher() { } log.Warnf("worker %d dropped", wid) - select { - case sh.workerClosing <- wid: - case <-sh.closing: - return - } + // send in a goroutine to avoid a deadlock between workerClosing / watchClosing + go func() { + select { + case sh.workerClosing <- wid: + case <-sh.closing: + return + } + }() } } } From 3f2a62b5e2e7f527a4a7ee39ea3e41fbf4a0c7b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 13 Aug 2020 13:18:14 +0200 Subject: [PATCH 05/36] cli: net reachability command --- api/api_common.go | 7 +++++++ api/apistruct/struct.go | 6 ++++++ cli/net.go | 26 ++++++++++++++++++++++++++ node/impl/common/common.go | 19 +++++++++++++++++++ 4 files changed, 58 insertions(+) diff --git a/api/api_common.go b/api/api_common.go index aa63e9815..0c20d0ad9 100644 --- a/api/api_common.go +++ b/api/api_common.go @@ -6,6 +6,7 @@ import ( "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" + ma "github.com/multiformats/go-multiaddr" "github.com/filecoin-project/go-jsonrpc/auth" @@ -28,6 +29,7 @@ type Common interface { NetDisconnect(context.Context, peer.ID) error NetFindPeer(context.Context, peer.ID) (peer.AddrInfo, error) NetPubsubScores(context.Context) ([]PubsubScore, error) + NetAutoNatStatus(context.Context) (NatInfo, error) // MethodGroup: Common @@ -65,3 +67,8 @@ type Version struct { func (v Version) String() string { return fmt.Sprintf("%s+api%s", v.Version, v.APIVersion.String()) } + +type NatInfo struct { + Reachability network.Reachability + PublicAddr ma.Multiaddr +} diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 5be189c98..49a0c2afc 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -49,6 +49,7 @@ type CommonStruct struct { NetDisconnect func(context.Context, peer.ID) error `perm:"write"` NetFindPeer func(context.Context, peer.ID) (peer.AddrInfo, error) `perm:"read"` NetPubsubScores func(context.Context) ([]api.PubsubScore, error) `perm:"read"` + NetAutoNatStatus func(context.Context) (api.NatInfo, error) `perm:"read"` ID func(context.Context) (peer.ID, error) `perm:"read"` Version func(context.Context) (api.Version, error) `perm:"read"` @@ -331,6 +332,7 @@ func (c *CommonStruct) AuthNew(ctx context.Context, perms []auth.Permission) ([] func (c *CommonStruct) NetPubsubScores(ctx context.Context) ([]api.PubsubScore, error) { return c.Internal.NetPubsubScores(ctx) } + func (c *CommonStruct) NetConnectedness(ctx context.Context, pid peer.ID) (network.Connectedness, error) { return c.Internal.NetConnectedness(ctx, pid) } @@ -355,6 +357,10 @@ func (c *CommonStruct) NetFindPeer(ctx context.Context, p peer.ID) (peer.AddrInf return c.Internal.NetFindPeer(ctx, p) } +func (c *CommonStruct) NetAutoNatStatus(ctx context.Context) (api.NatInfo, error) { + return c.Internal.NetAutoNatStatus(ctx) +} + // ID implements API.ID func (c *CommonStruct) ID(ctx context.Context) (peer.ID, error) { return c.Internal.ID(ctx) diff --git a/cli/net.go b/cli/net.go index 2e35f552f..e960a7234 100644 --- a/cli/net.go +++ b/cli/net.go @@ -24,6 +24,7 @@ var netCmd = &cli.Command{ NetId, netFindPeer, netScores, + netReachability, }, } @@ -202,3 +203,28 @@ var netFindPeer = &cli.Command{ return nil }, } + +var netReachability = &cli.Command{ + Name: "reachability", + Usage: "Print information about reachability from the internet", + Action: func(cctx *cli.Context) error { + api, closer, err := GetAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := ReqContext(cctx) + + i, err := api.NetAutoNatStatus(ctx) + if err != nil { + return err + } + + fmt.Println("AutoNAT status: ", i.Reachability.String()) + if i.PublicAddr != nil { + fmt.Println("Public address: ", i.PublicAddr.String()) + } + return nil + }, +} diff --git a/node/impl/common/common.go b/node/impl/common/common.go index 1d2695b6e..89078dd39 100644 --- a/node/impl/common/common.go +++ b/node/impl/common/common.go @@ -2,6 +2,7 @@ package common import ( "context" + basichost "github.com/libp2p/go-libp2p/p2p/host/basic" "sort" "strings" @@ -28,6 +29,7 @@ type CommonAPI struct { fx.In APISecret *dtypes.APIAlg + RawHost lp2p.RawHost Host host.Host Router lp2p.BaseIpfsRouting Sk *dtypes.ScoreKeeper @@ -113,6 +115,23 @@ func (a *CommonAPI) NetFindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo, return a.Router.FindPeer(ctx, p) } +func (a *CommonAPI) NetAutoNatStatus(ctx context.Context) (i api.NatInfo, err error) { + autonat := a.RawHost.(*basichost.BasicHost).AutoNat + + var maddr ma.Multiaddr + if autonat.Status() == network.ReachabilityPublic { + maddr, err = autonat.PublicAddr() + if err != nil { + return api.NatInfo{}, err + } + } + + return api.NatInfo{ + Reachability: autonat.Status(), + PublicAddr: maddr, + }, nil +} + func (a *CommonAPI) ID(context.Context) (peer.ID, error) { return a.Host.ID(), nil } From 069c39fc1281a402d614503980b292bf5677e544 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 13 Aug 2020 13:44:03 +0200 Subject: [PATCH 06/36] Add reachability info to miner info all --- cli/net.go | 4 ++-- cmd/lotus-storage-miner/info_all.go | 5 +++++ node/impl/common/common.go | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cli/net.go b/cli/net.go index e960a7234..c3928a5b8 100644 --- a/cli/net.go +++ b/cli/net.go @@ -24,7 +24,7 @@ var netCmd = &cli.Command{ NetId, netFindPeer, netScores, - netReachability, + NetReachability, }, } @@ -204,7 +204,7 @@ var netFindPeer = &cli.Command{ }, } -var netReachability = &cli.Command{ +var NetReachability = &cli.Command{ Name: "reachability", Usage: "Print information about reachability from the internet", Action: func(cctx *cli.Context) error { diff --git a/cmd/lotus-storage-miner/info_all.go b/cmd/lotus-storage-miner/info_all.go index 4b81bf37c..be9377816 100644 --- a/cmd/lotus-storage-miner/info_all.go +++ b/cmd/lotus-storage-miner/info_all.go @@ -63,6 +63,11 @@ var infoAllCmd = &cli.Command{ return err } + fmt.Println("\n#: Reachability") + if err := lcli.NetReachability.Action(cctx); err != nil { + return err + } + // Very Verbose info fmt.Println("\n#: Peers") if err := lcli.NetPeers.Action(cctx); err != nil { diff --git a/node/impl/common/common.go b/node/impl/common/common.go index 89078dd39..73abea8fb 100644 --- a/node/impl/common/common.go +++ b/node/impl/common/common.go @@ -2,7 +2,6 @@ package common import ( "context" - basichost "github.com/libp2p/go-libp2p/p2p/host/basic" "sort" "strings" @@ -13,6 +12,7 @@ import ( "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" swarm "github.com/libp2p/go-libp2p-swarm" + basichost "github.com/libp2p/go-libp2p/p2p/host/basic" ma "github.com/multiformats/go-multiaddr" "go.uber.org/fx" "golang.org/x/xerrors" From 3b23ab952bccf79033373d4ee8ba3108ba67d2e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 13 Aug 2020 13:44:43 +0200 Subject: [PATCH 07/36] gofmt --- chain/events/state/diff_adt.go | 5 ++--- chain/events/state/diff_adt_test.go | 8 +++----- chain/events/state/predicates.go | 13 +++++-------- cli/net.go | 4 ++-- cmd/lotus-chainwatch/processor/common_actors.go | 2 +- 5 files changed, 13 insertions(+), 19 deletions(-) diff --git a/chain/events/state/diff_adt.go b/chain/events/state/diff_adt.go index 9e66eede3..1b921dd9e 100644 --- a/chain/events/state/diff_adt.go +++ b/chain/events/state/diff_adt.go @@ -41,7 +41,7 @@ func DiffAdtArray(preArr, curArr *adt.Array, out AdtArrayDiff) error { } // no modification - if !bytes.Equal(prevVal.Raw,curVal.Raw) { + if !bytes.Equal(prevVal.Raw, curVal.Raw) { if err := out.Modify(uint64(i), prevVal, curVal); err != nil { return err } @@ -95,13 +95,12 @@ func DiffAdtMap(preMap, curMap *adt.Map, out AdtMapDiff) error { } // no modification - if !bytes.Equal(prevVal.Raw,curVal.Raw) { + if !bytes.Equal(prevVal.Raw, curVal.Raw) { if err := out.Modify(key, prevVal, curVal); err != nil { return err } } - return curMap.Delete(k) }); err != nil { return err diff --git a/chain/events/state/diff_adt_test.go b/chain/events/state/diff_adt_test.go index a430d2de7..56a03bf33 100644 --- a/chain/events/state/diff_adt_test.go +++ b/chain/events/state/diff_adt_test.go @@ -71,7 +71,6 @@ func TestDiffAdtArray(t *testing.T) { assert.EqualValues(t, []byte{1}, changes.Removed[1].val) } - func TestDiffAdtMap(t *testing.T) { ctxstoreA := newContextStore() ctxstoreB := newContextStore() @@ -128,14 +127,13 @@ func TestDiffAdtMap(t *testing.T) { } type TestDiffMap struct { - Added []adtMapDiffResult + Added []adtMapDiffResult Modified []TestAdtMapDiffModified - Removed []adtMapDiffResult + Removed []adtMapDiffResult } var _ AdtMapDiff = &TestDiffMap{} - func (t *TestDiffMap) AsKey(key string) (adt.Keyer, error) { k, err := adt.ParseUIntKey(key) if err != nil { @@ -218,7 +216,7 @@ type adtMapDiffResult struct { type TestAdtMapDiffModified struct { From adtMapDiffResult - To adtMapDiffResult + To adtMapDiffResult } type adtArrayDiffResult struct { diff --git a/chain/events/state/predicates.go b/chain/events/state/predicates.go index 544ff7b14..bf85f1f1a 100644 --- a/chain/events/state/predicates.go +++ b/chain/events/state/predicates.go @@ -649,22 +649,20 @@ type AddressPair struct { } type InitActorAddressChanges struct { - Added []AddressPair + Added []AddressPair Modified []AddressChange - Removed []AddressPair + Removed []AddressPair } type AddressChange struct { From AddressPair - To AddressPair + To AddressPair } type DiffInitActorStateFunc func(ctx context.Context, oldState *init_.State, newState *init_.State) (changed bool, user UserData, err error) - - func (i *InitActorAddressChanges) AsKey(key string) (adt.Keyer, error) { - addr , err := address.NewFromBytes([]byte(key)) + addr, err := address.NewFromBytes([]byte(key)) if err != nil { return nil, err } @@ -720,7 +718,7 @@ func (i *InitActorAddressChanges) Modify(key string, from, to *typegen.Deferred) ID: fromIDAddr, PK: pkAddr, }, - To: AddressPair{ + To: AddressPair{ ID: toIDAddr, PK: pkAddr, }, @@ -786,4 +784,3 @@ func (sp *StatePredicates) OnAddressMapChange() DiffInitActorStateFunc { return true, addressChanges, nil } } - diff --git a/cli/net.go b/cli/net.go index c3928a5b8..7a2474bf4 100644 --- a/cli/net.go +++ b/cli/net.go @@ -205,8 +205,8 @@ var netFindPeer = &cli.Command{ } var NetReachability = &cli.Command{ - Name: "reachability", - Usage: "Print information about reachability from the internet", + Name: "reachability", + Usage: "Print information about reachability from the internet", Action: func(cctx *cli.Context) error { api, closer, err := GetAPI(cctx) if err != nil { diff --git a/cmd/lotus-chainwatch/processor/common_actors.go b/cmd/lotus-chainwatch/processor/common_actors.go index b2e86ddc2..3fbdbf170 100644 --- a/cmd/lotus-chainwatch/processor/common_actors.go +++ b/cmd/lotus-chainwatch/processor/common_actors.go @@ -176,7 +176,7 @@ func (p Processor) storeActorAddresses(ctx context.Context, actors map[cid.Cid]A for _, updates := range addressesToUpdate { if _, err := updateTx.Exec( fmt.Sprintf("update id_address_map set id=%s, address=%s where id=%s and address=%s", updates.New.ID, updates.New.PK, updates.Old.ID, updates.Old.PK), - ); err != nil { + ); err != nil { return err } } From 9a39bb4e78430c0e76dcfaa849dd77197366e37b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 13 Aug 2020 14:03:59 +0200 Subject: [PATCH 08/36] api: Remove unused StatePledgeCollateral --- api/api_full.go | 1 - api/apistruct/struct.go | 5 ----- cli/state.go | 28 ---------------------------- cmd/lotus-storage-miner/init.go | 8 ++------ node/impl/full/state.go | 31 ------------------------------- 5 files changed, 2 insertions(+), 71 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index f2585b927..55e958850 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -319,7 +319,6 @@ type FullNode interface { StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*SectorExpiration, error) // StateSectorPartition finds deadline/partition with the specified sector StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*SectorLocation, error) - StatePledgeCollateral(context.Context, types.TipSetKey) (types.BigInt, error) // StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed StateSearchMsg(context.Context, cid.Cid) (*MsgLookup, error) // StateWaitMsg looks back in the chain for a message. If not found, it blocks until the diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 49a0c2afc..4ecea5bde 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -164,7 +164,6 @@ type FullNodeStruct struct { StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` - StatePledgeCollateral func(context.Context, types.TipSetKey) (types.BigInt, error) `perm:"read"` StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"` StateSearchMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"` StateListMiners func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` @@ -738,10 +737,6 @@ func (c *FullNodeStruct) StateReadState(ctx context.Context, addr address.Addres return c.Internal.StateReadState(ctx, addr, tsk) } -func (c *FullNodeStruct) StatePledgeCollateral(ctx context.Context, tsk types.TipSetKey) (types.BigInt, error) { - return c.Internal.StatePledgeCollateral(ctx, tsk) -} - func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid, confidence uint64) (*api.MsgLookup, error) { return c.Internal.StateWaitMsg(ctx, msgc, confidence) } diff --git a/cli/state.go b/cli/state.go index 71bf236c7..57b1409c2 100644 --- a/cli/state.go +++ b/cli/state.go @@ -52,7 +52,6 @@ var stateCmd = &cli.Command{ statePowerCmd, stateSectorsCmd, stateActiveSectorsCmd, - statePledgeCollateralCmd, stateListActorsCmd, stateListMinersCmd, stateCircSupplyCmd, @@ -386,33 +385,6 @@ var stateReplaySetCmd = &cli.Command{ }, } -var statePledgeCollateralCmd = &cli.Command{ - Name: "pledge-collateral", - Usage: "Get minimum miner pledge collateral", - Action: func(cctx *cli.Context) error { - api, closer, err := GetFullNodeAPI(cctx) - if err != nil { - return err - } - defer closer() - - ctx := ReqContext(cctx) - - ts, err := LoadTipSet(ctx, cctx, api) - if err != nil { - return err - } - - coll, err := api.StatePledgeCollateral(ctx, ts.Key()) - if err != nil { - return err - } - - fmt.Println(types.FIL(coll)) - return nil - }, -} - var stateGetDealSetCmd = &cli.Command{ Name: "get-deal", Usage: "View on-chain deal info", diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 73956430b..6e21893db 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -7,6 +7,7 @@ import ( "encoding/binary" "encoding/json" "fmt" + "github.com/filecoin-project/specs-actors/actors/abi/big" "io/ioutil" "os" "path/filepath" @@ -614,11 +615,6 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, return address.Undef, err } - collateral, err := api.StatePledgeCollateral(ctx, types.EmptyTSK) - if err != nil { - return address.Undef, err - } - spt, err := ffiwrapper.SealProofTypeFromSectorSize(abi.SectorSize(ssize)) if err != nil { return address.Undef, err @@ -637,7 +633,7 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, createStorageMinerMsg := &types.Message{ To: builtin.StoragePowerActorAddr, From: owner, - Value: types.BigAdd(collateral, types.BigDiv(collateral, types.NewInt(100))), + Value: big.Zero(), Method: builtin.MethodsPower.CreateMiner, Params: params, diff --git a/node/impl/full/state.go b/node/impl/full/state.go index b94c3ab53..4f08d324d 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -258,37 +258,6 @@ func (a *StateAPI) StateMinerPower(ctx context.Context, addr address.Address, ts }, nil } -func (a *StateAPI) StatePledgeCollateral(ctx context.Context, tsk types.TipSetKey) (types.BigInt, error) { - /*ts, err := a.Chain.GetTipSetFromKey(tsk) - if err != nil { - return types.EmptyInt, xerrors.Errorf("loading tipset %s: %w", tsk, err) - } - - param, err := actors.SerializeParams(&actors.PledgeCollateralParams{Size: types.NewInt(0)}) - if err != nil { - return types.NewInt(0), err - } - - ret, aerr := a.StateManager.Call(ctx, &types.Message{ - From: actors.StoragePowerAddress, - To: actors.StoragePowerAddress, - Method: actors.SPAMethods.PledgeCollateralForSize, - - Params: param, - }, ts) - if aerr != nil { - return types.NewInt(0), xerrors.Errorf("failed to get miner worker addr: %w", err) - } - - if ret.MsgRct.ExitCode != 0 { - return types.NewInt(0), xerrors.Errorf("failed to get miner worker addr (exit code %d)", ret.MsgRct.ExitCode) - } - - return types.BigFromBytes(ret.Return), nil*/ - log.Error("TODO StatePledgeCollateral") - return big.Zero(), nil -} - func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { From a366522c8de8a43c758e40d4c6e3d4f9d60b0d09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 13 Aug 2020 14:04:45 +0200 Subject: [PATCH 09/36] Fix lotus-fountain build --- cmd/lotus-fountain/main.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go index 84538e61b..4f84760c0 100644 --- a/cmd/lotus-fountain/main.go +++ b/cmd/lotus-fountain/main.go @@ -23,6 +23,7 @@ import ( "github.com/filecoin-project/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/abi/big" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/power" @@ -339,13 +340,6 @@ func (h *handler) mkminer(w http.ResponseWriter, r *http.Request) { return } - collateral, err := h.api.StatePledgeCollateral(r.Context(), types.EmptyTSK) - if err != nil { - w.WriteHeader(400) - w.Write([]byte(err.Error())) - return - } - smsg, err := h.api.MpoolPushMessage(h.ctx, &types.Message{ Value: types.BigInt(h.sendPerRequest), From: h.from, @@ -380,7 +374,7 @@ func (h *handler) mkminer(w http.ResponseWriter, r *http.Request) { createStorageMinerMsg := &types.Message{ To: builtin.StoragePowerActorAddr, From: h.from, - Value: types.BigAdd(collateral, types.BigDiv(collateral, types.NewInt(100))), + Value: big.Zero(), Method: builtin.MethodsPower.CreateMiner, Params: params, From d25f386bb57582cb85f63a7d98480ee73fee1879 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 12 Aug 2020 19:50:35 +0300 Subject: [PATCH 10/36] mixin the previous chain's effective performance to capture effect of dependencies Signed-off-by: Jakub Sztandera --- chain/messagepool/selection.go | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/chain/messagepool/selection.go b/chain/messagepool/selection.go index 7bfaaf811..a70cb7021 100644 --- a/chain/messagepool/selection.go +++ b/chain/messagepool/selection.go @@ -27,6 +27,7 @@ type msgChain struct { gasLimit int64 gasPerf float64 effPerf float64 + bp float64 valid bool merged bool next *msgChain @@ -810,18 +811,13 @@ func (mc *msgChain) Trim(gasLimit int64, mp *MessagePool, baseFee types.BigInt, mc.gasReward = new(big.Int).Sub(mc.gasReward, gasReward) mc.gasLimit -= mc.msgs[i].Message.GasLimit if mc.gasLimit > 0 { - bp := 1.0 - if mc.gasPerf != 0 { // prevent div by 0 - bp = mc.effPerf / mc.gasPerf - } - mc.gasPerf = mp.getGasPerf(mc.gasReward, mc.gasLimit) - - if mc.effPerf != 0 { // keep effPerf 0 if it is 0 - mc.effPerf = bp * mc.gasPerf + if mc.bp != 0 { + mc.setEffPerf() } } else { mc.gasPerf = 0 + mc.effPerf = 0 } i-- } @@ -849,7 +845,17 @@ func (mc *msgChain) Invalidate() { } func (mc *msgChain) SetEffectivePerf(bp float64) { - mc.effPerf = mc.gasPerf * bp + mc.bp = bp + mc.setEffPerf() +} + +func (mc *msgChain) setEffPerf() { + effPerf := mc.gasPerf * mc.bp + if mc.prev != nil { + effPerf = (effPerf*float64(mc.gasLimit) + mc.prev.effPerf*float64(mc.prev.gasLimit)) / float64(mc.gasLimit+mc.prev.gasLimit) + } + mc.effPerf = effPerf + } func (mc *msgChain) SetNullEffectivePerf() { From f35555964d6a3771147c9072f928b3ca8cad26fe Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 13 Aug 2020 02:23:52 +0200 Subject: [PATCH 11/36] Better "optimal selection Signed-off-by: Jakub Sztandera --- chain/messagepool/block_proba.go | 33 +++++++++++++++--------- chain/messagepool/selection.go | 40 +++++++++++++++++++---------- chain/messagepool/selection_test.go | 35 ++++++++++++------------- 3 files changed, 64 insertions(+), 44 deletions(-) diff --git a/chain/messagepool/block_proba.go b/chain/messagepool/block_proba.go index 403507352..7304bd3ce 100644 --- a/chain/messagepool/block_proba.go +++ b/chain/messagepool/block_proba.go @@ -1,20 +1,29 @@ package messagepool -import "math" +import ( + "math" + "sync" +) + +var noWinnersProbCache []float64 +var noWinnersProbOnce sync.Once func noWinnersProb() []float64 { - poissPdf := func(x float64) float64 { - const Mu = 5 - lg, _ := math.Lgamma(x + 1) - result := math.Exp((math.Log(Mu) * x) - lg - Mu) - return result - } + noWinnersProbOnce.Do(func() { + poissPdf := func(x float64) float64 { + const Mu = 5 + lg, _ := math.Lgamma(x + 1) + result := math.Exp((math.Log(Mu) * x) - lg - Mu) + return result + } - out := make([]float64, 0, MaxBlocks) - for i := 0; i < MaxBlocks; i++ { - out = append(out, poissPdf(float64(i))) - } - return out + out := make([]float64, 0, MaxBlocks) + for i := 0; i < MaxBlocks; i++ { + out = append(out, poissPdf(float64(i))) + } + noWinnersProbCache = out + }) + return noWinnersProbCache } func binomialCoefficient(n, k float64) float64 { diff --git a/chain/messagepool/selection.go b/chain/messagepool/selection.go index a70cb7021..e38daf150 100644 --- a/chain/messagepool/selection.go +++ b/chain/messagepool/selection.go @@ -22,16 +22,17 @@ var bigBlockGasLimit = big.NewInt(build.BlockGasLimit) const MaxBlocks = 15 type msgChain struct { - msgs []*types.SignedMessage - gasReward *big.Int - gasLimit int64 - gasPerf float64 - effPerf float64 - bp float64 - valid bool - merged bool - next *msgChain - prev *msgChain + msgs []*types.SignedMessage + gasReward *big.Int + gasLimit int64 + gasPerf float64 + effPerf float64 + bp float64 + parentOffset float64 + valid bool + merged bool + next *msgChain + prev *msgChain } func (mp *MessagePool) SelectMessages(ts *types.TipSet, tq float64) ([]*types.SignedMessage, error) { @@ -153,7 +154,7 @@ func (mp *MessagePool) selectMessagesOptimal(curTs, ts *types.TipSet, tq float64 last := len(chains) for i, chain := range chains { // did we run out of performing chains? - if chain.gasPerf < 0 { + if chain.effPerf < 0 { break } @@ -176,12 +177,22 @@ func (mp *MessagePool) selectMessagesOptimal(curTs, ts *types.TipSet, tq float64 for i := len(chainDeps) - 1; i >= 0; i-- { curChain := chainDeps[i] curChain.merged = true + if next := curChain.next; next != nil { + next.effPerf += next.parentOffset + } result = append(result, curChain.msgs...) } chain.merged = true + if next := chain.next; next != nil { + next.effPerf += next.parentOffset + } result = append(result, chain.msgs...) gasLimit -= chainGasLimit + + sort.Slice(chains[i+1:], func(i, j int) bool { + return chains[i].BeforeEffective(chains[j]) + }) continue } @@ -852,7 +863,9 @@ func (mc *msgChain) SetEffectivePerf(bp float64) { func (mc *msgChain) setEffPerf() { effPerf := mc.gasPerf * mc.bp if mc.prev != nil { - effPerf = (effPerf*float64(mc.gasLimit) + mc.prev.effPerf*float64(mc.prev.gasLimit)) / float64(mc.gasLimit+mc.prev.gasLimit) + effPerfWithParent := (effPerf*float64(mc.gasLimit) + mc.prev.effPerf*float64(mc.prev.gasLimit)) / float64(mc.gasLimit+mc.prev.gasLimit) + mc.parentOffset = effPerf - effPerfWithParent + effPerf = effPerfWithParent } mc.effPerf = effPerf @@ -867,7 +880,8 @@ func (mc *msgChain) SetNullEffectivePerf() { } func (mc *msgChain) BeforeEffective(other *msgChain) bool { - return mc.effPerf > other.effPerf || + // moved merged chains to the front so we can discard them earlier + return (mc.merged && !other.merged) || mc.effPerf > other.effPerf || (mc.effPerf == other.effPerf && mc.gasPerf > other.gasPerf) || (mc.effPerf == other.effPerf && mc.gasPerf == other.gasPerf && mc.gasReward.Cmp(other.gasReward) > 0) } diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index fb2e4cc5f..28909b78f 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -919,20 +919,20 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) { t.Fatal(err) } - // 2. optimal selection - minersRand := rng.Float64() - winerProba := noWinnersProb() - i := 0 - for ; i < MaxBlocks && minersRand > 0; i++ { - minersRand -= winerProba[i] - } - nMiners := i - if nMiners == 0 { - nMiners = 1 - } - logging.SetLogLevel("messagepool", "error") - for i := 0; i < 1; i++ { + for i := 0; i < 50; i++ { + // 2. optimal selection + minersRand := rng.Float64() + winerProba := noWinnersProb() + i := 0 + for ; i < MaxBlocks && minersRand > 0; i++ { + minersRand -= winerProba[i] + } + nMiners := i + if nMiners == 0 { + nMiners = 1 + } + optMsgs := make(map[cid.Cid]*types.SignedMessage) for j := 0; j < nMiners; j++ { tq := rng.Float64() @@ -946,7 +946,7 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) { } t.Logf("nMiners: %d", nMiners) - t.Logf("greedy capacity %d, optimal capacity %d (x%.1f)", len(greedyMsgs), + t.Logf("greedy capacity %d, optimal capacity %d (x %.1f )", len(greedyMsgs), len(optMsgs), float64(len(optMsgs))/float64(len(greedyMsgs))) if len(greedyMsgs) > len(optMsgs) { t.Fatal("greedy capacity higher than optimal capacity; wtf") @@ -965,18 +965,15 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) { nMinersBig := big.NewInt(int64(nMiners)) greedyAvgReward, _ := new(big.Rat).SetFrac(greedyReward, nMinersBig).Float64() optimalAvgReward, _ := new(big.Rat).SetFrac(optReward, nMinersBig).Float64() - t.Logf("greedy reward: %.0f, optimal reward: %.0f (x%.1f)", greedyAvgReward, + t.Logf("greedy reward: %.0f, optimal reward: %.0f (x %.1f )", greedyAvgReward, optimalAvgReward, optimalAvgReward/greedyAvgReward) - if greedyReward.Cmp(optReward) > 0 { - t.Fatal("greedy reward raw higher than optimal reward; booh") - } } logging.SetLogLevel("messagepool", "info") } func TestCompetitiveMessageSelection(t *testing.T) { - seeds := []int64{1947, 1976, 2020, 2100, 10000} + seeds := []int64{1947, 1976, 2020, 2100, 10000, 143324, 432432, 131, 32, 45} for _, seed := range seeds { t.Log("running competitve message selection with seed", seed) testCompetitiveMessageSelection(t, rand.New(rand.NewSource(seed))) From 7bbf1c7db2f5c6c2284f3911b1cf7ce3f2a41776 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 13 Aug 2020 10:28:56 +0300 Subject: [PATCH 12/36] don't mix negative performing chains with their parent, add some comments Signed-off-by: Jakub Sztandera --- chain/messagepool/selection.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/chain/messagepool/selection.go b/chain/messagepool/selection.go index e38daf150..d1295afd4 100644 --- a/chain/messagepool/selection.go +++ b/chain/messagepool/selection.go @@ -177,6 +177,7 @@ func (mp *MessagePool) selectMessagesOptimal(curTs, ts *types.TipSet, tq float64 for i := len(chainDeps) - 1; i >= 0; i-- { curChain := chainDeps[i] curChain.merged = true + // adjust the next chain for the parent, which is being merged if next := curChain.next; next != nil { next.effPerf += next.parentOffset } @@ -190,6 +191,7 @@ func (mp *MessagePool) selectMessagesOptimal(curTs, ts *types.TipSet, tq float64 result = append(result, chain.msgs...) gasLimit -= chainGasLimit + // resort to account for already merged chains and effective performance adjustments sort.Slice(chains[i+1:], func(i, j int) bool { return chains[i].BeforeEffective(chains[j]) }) @@ -862,7 +864,7 @@ func (mc *msgChain) SetEffectivePerf(bp float64) { func (mc *msgChain) setEffPerf() { effPerf := mc.gasPerf * mc.bp - if mc.prev != nil { + if effPerf > 0 && mc.prev != nil { effPerfWithParent := (effPerf*float64(mc.gasLimit) + mc.prev.effPerf*float64(mc.prev.gasLimit)) / float64(mc.gasLimit+mc.prev.gasLimit) mc.parentOffset = effPerf - effPerfWithParent effPerf = effPerfWithParent @@ -880,7 +882,7 @@ func (mc *msgChain) SetNullEffectivePerf() { } func (mc *msgChain) BeforeEffective(other *msgChain) bool { - // moved merged chains to the front so we can discard them earlier + // move merged chains to the front so we can discard them earlier return (mc.merged && !other.merged) || mc.effPerf > other.effPerf || (mc.effPerf == other.effPerf && mc.gasPerf > other.gasPerf) || (mc.effPerf == other.effPerf && mc.gasPerf == other.gasPerf && mc.gasReward.Cmp(other.gasReward) > 0) From 5e8ae7498a014c32cc89a2f003a6fc329a87c2d6 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 13 Aug 2020 10:33:20 +0300 Subject: [PATCH 13/36] only adjust next chains if they have positve perf Signed-off-by: Jakub Sztandera --- chain/messagepool/selection.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/messagepool/selection.go b/chain/messagepool/selection.go index d1295afd4..43154a4b9 100644 --- a/chain/messagepool/selection.go +++ b/chain/messagepool/selection.go @@ -178,14 +178,14 @@ func (mp *MessagePool) selectMessagesOptimal(curTs, ts *types.TipSet, tq float64 curChain := chainDeps[i] curChain.merged = true // adjust the next chain for the parent, which is being merged - if next := curChain.next; next != nil { + if next := curChain.next; next != nil && next.effPerf > 0 { next.effPerf += next.parentOffset } result = append(result, curChain.msgs...) } chain.merged = true - if next := chain.next; next != nil { + if next := chain.next; next != nil && next.effPerf > 0 { next.effPerf += next.parentOffset } result = append(result, chain.msgs...) From 0f5598e59b80d8fc9b12fdcaf0f9b0a94260ed17 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 13 Aug 2020 11:00:42 +0300 Subject: [PATCH 14/36] compute average boost in tests Signed-off-by: Jakub Sztandera --- chain/messagepool/selection_test.go | 35 ++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index 28909b78f..607fb1db3 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -861,7 +861,7 @@ func TestOptimalMessageSelection3(t *testing.T) { } } -func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) { +func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) (float64, float64) { // in this test we use 100 actors and send 10 blocks of messages. // actors send with an exponentially decreasing premium. // a number of miners select with varying ticket quality and we compare the @@ -913,13 +913,16 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) { mustAdd(t, mp, m) } + logging.SetLogLevel("messagepool", "error") + // 1. greedy selection greedyMsgs, err := mp.selectMessagesGreedy(ts, ts) if err != nil { t.Fatal(err) } - logging.SetLogLevel("messagepool", "error") + capacityBoost := 0.0 + rewardBoost := 0.0 for i := 0; i < 50; i++ { // 2. optimal selection minersRand := rng.Float64() @@ -945,9 +948,12 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) { } } + boost := float64(len(optMsgs)) / float64(len(greedyMsgs)) + capacityBoost += boost + t.Logf("nMiners: %d", nMiners) t.Logf("greedy capacity %d, optimal capacity %d (x %.1f )", len(greedyMsgs), - len(optMsgs), float64(len(optMsgs))/float64(len(greedyMsgs))) + len(optMsgs), boost) if len(greedyMsgs) > len(optMsgs) { t.Fatal("greedy capacity higher than optimal capacity; wtf") } @@ -965,17 +971,36 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) { nMinersBig := big.NewInt(int64(nMiners)) greedyAvgReward, _ := new(big.Rat).SetFrac(greedyReward, nMinersBig).Float64() optimalAvgReward, _ := new(big.Rat).SetFrac(optReward, nMinersBig).Float64() + + boost = optimalAvgReward / greedyAvgReward + rewardBoost += boost t.Logf("greedy reward: %.0f, optimal reward: %.0f (x %.1f )", greedyAvgReward, - optimalAvgReward, optimalAvgReward/greedyAvgReward) + optimalAvgReward, boost) } + + capacityBoost /= 50 + rewardBoost /= 50 + t.Logf("Average capacity boost: %f", capacityBoost) + t.Logf("Average reward boost: %f", rewardBoost) + logging.SetLogLevel("messagepool", "info") + + return capacityBoost, rewardBoost } func TestCompetitiveMessageSelection(t *testing.T) { + var capacityBoost, rewardBoost float64 seeds := []int64{1947, 1976, 2020, 2100, 10000, 143324, 432432, 131, 32, 45} for _, seed := range seeds { t.Log("running competitve message selection with seed", seed) - testCompetitiveMessageSelection(t, rand.New(rand.NewSource(seed))) + cb, rb := testCompetitiveMessageSelection(t, rand.New(rand.NewSource(seed))) + capacityBoost += cb + rewardBoost += rb } + + capacityBoost /= float64(len(seeds)) + rewardBoost /= float64(len(seeds)) + t.Logf("Average capacity boost across all seeds: %f", capacityBoost) + t.Logf("Average reward boost across all seeds: %f", rewardBoost) } From 1aa8b6cab21fbddd96afe3d878c98d63003e42d1 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 13 Aug 2020 12:52:15 +0300 Subject: [PATCH 15/36] make OptimalSelection2 test pass Signed-off-by: Jakub Sztandera --- chain/messagepool/selection_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index 607fb1db3..20fdf8af3 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -755,9 +755,9 @@ func TestOptimalMessageSelection2(t *testing.T) { nMessages := int(5 * build.BlockGasLimit / gasLimit) for i := 0; i < nMessages; i++ { bias := (nMessages - i) / 3 - m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(10000+i%3+bias)) + m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(200000+i%3+bias)) mustAdd(t, mp, m) - m = makeTestMessage(w2, a2, a1, uint64(i), gasLimit, uint64(1+i%3+bias)) + m = makeTestMessage(w2, a2, a1, uint64(i), gasLimit, uint64(190000+i%3+bias)) mustAdd(t, mp, m) } From f3eec33becb2c2de7239b6238ad9be9a0713667e Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 13 Aug 2020 13:04:36 +0300 Subject: [PATCH 16/36] make OptimalSelection3 test pass Signed-off-by: Jakub Sztandera --- chain/messagepool/selection_test.go | 31 ++++++++++++++++------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index 20fdf8af3..8843cfe61 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -824,8 +824,8 @@ func TestOptimalMessageSelection3(t *testing.T) { nMessages := int(build.BlockGasLimit/gasLimit) + 1 for i := 0; i < nMessages; i++ { for j := 0; j < nActors; j++ { - bias := (nActors-j)*nMessages + (nMessages+2-i)/(3*nActors) + i%3 - m := makeTestMessage(wallets[j], actors[j], actors[j%nActors], uint64(i), gasLimit, uint64(1+bias)) + premium := 500000 + 20000*(nActors-j) + (nMessages+2-i)/(3*nActors) + i%3 + m := makeTestMessage(wallets[j], actors[j], actors[j%nActors], uint64(i), gasLimit, uint64(premium)) mustAdd(t, mp, m) } } @@ -840,24 +840,27 @@ func TestOptimalMessageSelection3(t *testing.T) { t.Fatalf("expected %d messages, but got %d", expectedMsgs, len(msgs)) } - nextNonce := uint64(0) - a := actors[len(actors)/2-1] - for _, m := range msgs { - if m.Message.From != a { - who := 0 - for i, a := range actors { - if a == m.Message.From { - who = i - break - } + whoIs := func(a address.Address) int { + for i, aa := range actors { + if a == aa { + return i } - t.Fatalf("expected message from last actor, but got from %d instead", who) + } + return -1 + } + + nonces := make([]uint64, nActors) + for _, m := range msgs { + who := whoIs(m.Message.From) + if who < 3 { + t.Fatalf("got message from %dth actor", who) } + nextNonce := nonces[who] if m.Message.Nonce != nextNonce { t.Fatalf("expected nonce %d but got %d", nextNonce, m.Message.Nonce) } - nextNonce++ + nonces[who]++ } } From 09f5a64ce03ea8e8d3c99cbdaa1a297acd4290f5 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 13 Aug 2020 13:43:40 +0300 Subject: [PATCH 17/36] use gasPerf for negative performance check it's the same in actually because effPerf<0 iff gasPerf<0, but we should be consistent Signed-off-by: Jakub Sztandera --- chain/messagepool/selection.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/messagepool/selection.go b/chain/messagepool/selection.go index 43154a4b9..10ccf6b99 100644 --- a/chain/messagepool/selection.go +++ b/chain/messagepool/selection.go @@ -154,7 +154,7 @@ func (mp *MessagePool) selectMessagesOptimal(curTs, ts *types.TipSet, tq float64 last := len(chains) for i, chain := range chains { // did we run out of performing chains? - if chain.effPerf < 0 { + if chain.gasPerf < 0 { break } From 0daed04c07688cc1af4faf752e0976310e69c84e Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 13 Aug 2020 14:00:55 +0200 Subject: [PATCH 18/36] Don't run the test 50 times Signed-off-by: Jakub Sztandera --- chain/messagepool/selection_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index 8843cfe61..0d82c09d8 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -926,7 +926,8 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) (float64, flo capacityBoost := 0.0 rewardBoost := 0.0 - for i := 0; i < 50; i++ { + const runs = 1 + for i := 0; i < runs; i++ { // 2. optimal selection minersRand := rng.Float64() winerProba := noWinnersProb() @@ -982,8 +983,8 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) (float64, flo } - capacityBoost /= 50 - rewardBoost /= 50 + capacityBoost /= runs + rewardBoost /= runs t.Logf("Average capacity boost: %f", capacityBoost) t.Logf("Average reward boost: %f", rewardBoost) From 47582c0bb384e92d6aac3cedfb54b73ced0f6af6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 13 Aug 2020 14:16:29 +0200 Subject: [PATCH 19/36] mpool: Fix capGasFee math again --- node/impl/full/mpool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index f77085d04..64befa738 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -114,7 +114,7 @@ func capGasFee(msg *types.Message, maxFee abi.TokenAmount) { return } - gl := types.BigMul(msg.GasPremium, types.NewInt(uint64(msg.GasLimit))) + gl := types.NewInt(uint64(msg.GasLimit)) totalFee := types.BigMul(msg.GasFeeCap, gl) minerFee := types.BigMul(msg.GasPremium, gl) From 536b2ad9d73f0f9a263c39a73860179a8c61ac4c Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 13 Aug 2020 10:14:11 -0400 Subject: [PATCH 20/36] fix: paych - bug with allocating lane --- paychmgr/paych_test.go | 106 +++++++++++++++++++++++++++++++++-------- paychmgr/state.go | 14 ++---- 2 files changed, 90 insertions(+), 30 deletions(-) diff --git a/paychmgr/paych_test.go b/paychmgr/paych_test.go index 56aa9ebb2..126e73702 100644 --- a/paychmgr/paych_test.go +++ b/paychmgr/paych_test.go @@ -507,6 +507,15 @@ func TestAddVoucherNextLane(t *testing.T) { require.NoError(t, err) require.EqualValues(t, ci.NextLane, 3) + // Allocate a lane (should be lane 3) + lane, err := mgr.AllocateLane(ch) + require.NoError(t, err) + require.EqualValues(t, lane, 3) + + ci, err = mgr.GetChannelInfo(ch) + require.NoError(t, err) + require.EqualValues(t, ci.NextLane, 4) + // Add a voucher in lane 1 voucherLane = uint64(1) sv = testCreateVoucher(t, ch, voucherLane, nonce, voucherAmount, fromKeyPrivate) @@ -515,17 +524,89 @@ func TestAddVoucherNextLane(t *testing.T) { ci, err = mgr.GetChannelInfo(ch) require.NoError(t, err) - require.EqualValues(t, ci.NextLane, 3) + require.EqualValues(t, ci.NextLane, 4) - // Add a voucher in lane 5 - voucherLane = uint64(5) + // Add a voucher in lane 7 + voucherLane = uint64(7) sv = testCreateVoucher(t, ch, voucherLane, nonce, voucherAmount, fromKeyPrivate) _, err = mgr.AddVoucher(ctx, ch, sv, nil, minDelta) require.NoError(t, err) ci, err = mgr.GetChannelInfo(ch) require.NoError(t, err) - require.EqualValues(t, ci.NextLane, 6) + require.EqualValues(t, ci.NextLane, 8) +} + +func TestAllocateLane(t *testing.T) { + ctx := context.Background() + + // Set up a manager with a single payment channel + mgr, ch, _ := testSetupMgrWithChannel(ctx, t) + + // First lane should be 0 + lane, err := mgr.AllocateLane(ch) + require.NoError(t, err) + require.EqualValues(t, lane, 0) + + // Next lane should be 1 + lane, err = mgr.AllocateLane(ch) + require.NoError(t, err) + require.EqualValues(t, lane, 1) +} + +func TestAllocateLaneWithExistingLaneState(t *testing.T) { + ctx := context.Background() + + _, fromKeyPublic := testGenerateKeyPair(t) + + ch := tutils.NewIDAddr(t, 100) + from := tutils.NewSECP256K1Addr(t, string(fromKeyPublic)) + to := tutils.NewSECP256K1Addr(t, "secpTo") + fromAcct := tutils.NewActorAddr(t, "fromAct") + toAcct := tutils.NewActorAddr(t, "toAct") + + mock := newMockManagerAPI() + mock.setAccountState(fromAcct, account.State{Address: from}) + mock.setAccountState(toAcct, account.State{Address: to}) + + store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) + + actorBalance := big.NewInt(10) + toSend := big.NewInt(1) + laneStates := map[uint64]paych.LaneState{ + 2: { + Nonce: 1, + Redeemed: big.NewInt(4), + }, + } + + act := &types.Actor{ + Code: builtin.AccountActorCodeID, + Head: cid.Cid{}, + Nonce: 0, + Balance: actorBalance, + } + + lsCid, err := mock.storeLaneStates(laneStates) + require.NoError(t, err) + mock.setPaychState(ch, act, paych.State{ + From: fromAcct, + To: toAcct, + ToSend: toSend, + SettlingAt: abi.ChainEpoch(0), + MinSettleHeight: abi.ChainEpoch(0), + LaneStates: lsCid, + }) + + mgr, err := newManager(store, mock) + require.NoError(t, err) + + err = mgr.TrackInboundChannel(ctx, ch) + require.NoError(t, err) + + lane, err := mgr.AllocateLane(ch) + require.NoError(t, err) + require.EqualValues(t, 3, lane) } func TestAddVoucherProof(t *testing.T) { @@ -575,23 +656,6 @@ func TestAddVoucherProof(t *testing.T) { require.Len(t, ci.Vouchers[0].Proof, 1) } -func TestAllocateLane(t *testing.T) { - ctx := context.Background() - - // Set up a manager with a single payment channel - mgr, ch, _ := testSetupMgrWithChannel(ctx, t) - - // First lane should be 0 - lane, err := mgr.AllocateLane(ch) - require.NoError(t, err) - require.EqualValues(t, lane, 0) - - // Next lane should be 1 - lane, err = mgr.AllocateLane(ch) - require.NoError(t, err) - require.EqualValues(t, lane, 1) -} - func TestNextNonceForLane(t *testing.T) { ctx := context.Background() diff --git a/paychmgr/state.go b/paychmgr/state.go index 015479913..db1b8ded2 100644 --- a/paychmgr/state.go +++ b/paychmgr/state.go @@ -2,7 +2,6 @@ package paychmgr import ( "context" - "errors" "github.com/filecoin-project/specs-actors/actors/util/adt" @@ -78,18 +77,15 @@ func (ca *stateAccessor) nextLaneFromState(ctx context.Context, st *paych.State) return 0, nil } - nextID := int64(0) - stopErr := errors.New("stop") + maxID := int64(0) if err := laneStates.ForEach(nil, func(i int64) error { - if nextID < i { - // We've found a hole. Stop here. - return stopErr + if i > maxID { + maxID = i } - nextID++ return nil - }); err != nil && err != stopErr { + }); err != nil { return 0, err } - return uint64(nextID), nil + return uint64(maxID + 1), nil } From dd619e85ff2f88282c8bb973f084159d39682e49 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 13 Aug 2020 16:48:29 +0200 Subject: [PATCH 21/36] Add total gas limit to stats Signed-off-by: Jakub Sztandera --- tools/stats/metrics.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index 15c28039d..728570c02 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -16,6 +16,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/util/adt" + "golang.org/x/xerrors" "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" @@ -135,6 +136,7 @@ func RecordTipsetPoints(ctx context.Context, api api.FullNode, pl *PointList, ti p = NewPoint("chain.basefee", baseFeeFloat) pl.AddPoint(p) + totalGasLimit := int64(0) for _, blockheader := range tipset.Blocks() { bs, err := blockheader.Serialize() if err != nil { @@ -146,7 +148,20 @@ func RecordTipsetPoints(ctx context.Context, api api.FullNode, pl *PointList, ti p = NewPoint("chain.blockheader_size", len(bs)) pl.AddPoint(p) + + msgs, err := api.ChainGetBlockMessages(ctx, blockheader.Cid()) + if err != nil { + return xerrors.Errorf("ChainGetBlockMessages failed: %w", msgs) + } + for _, m := range msgs.BlsMessages { + totalGasLimit += m.GasLimit + } + for _, m := range msgs.SecpkMessages { + totalGasLimit += m.Message.GasLimit + } } + p = NewPoint("chain.gas_limit_total", totalGasLimit) + pl.AddPoint(p) return nil } From 3198d2d1e51c39813de9ad42e2ec2b890e42df71 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 13 Aug 2020 16:55:12 +0200 Subject: [PATCH 22/36] Add total gas used Signed-off-by: Jakub Sztandera --- tools/stats/metrics.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index 728570c02..ab7b876bc 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -105,7 +105,8 @@ func InfluxNewBatch() (client.BatchPoints, error) { } func NewPoint(name string, value interface{}) models.Point { - pt, _ := models.NewPoint(name, models.Tags{}, map[string]interface{}{"value": value}, build.Clock.Now()) + pt, _ := models.NewPoint(name, models.Tags{}, + map[string]interface{}{"value": value}, build.Clock.Now().UTC()) return pt } @@ -301,8 +302,16 @@ func RecordTipsetMessagesPoints(ctx context.Context, api api.FullNode, pl *Point msgn := make(map[msgTag][]cid.Cid) + totalGasUsed := int64(0) + for _, r := range recp { + totalGasUsed += r.GasUsed + } + p := NewPoint("chain.gas_used_total", totalGasUsed) + pl.AddPoint(p) + for i, msg := range msgs { // FIXME: use float so this doesn't overflow + // FIXME: this doesn't work as time points get overriden p := NewPoint("chain.message_gaspremium", msg.Message.GasPremium.Int64()) pl.AddPoint(p) p = NewPoint("chain.message_gasfeecap", msg.Message.GasFeeCap.Int64()) From 2eff03fa1f747b77e7f8003b364b043fae0ac046 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 13 Aug 2020 19:49:07 +0300 Subject: [PATCH 23/36] small fixes in optimal selection 1. Remove noop updates from the previous dependencies 2. Update subsequent dependencies, which was the intention --- chain/messagepool/selection.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/chain/messagepool/selection.go b/chain/messagepool/selection.go index 10ccf6b99..aeb80d53d 100644 --- a/chain/messagepool/selection.go +++ b/chain/messagepool/selection.go @@ -177,16 +177,16 @@ func (mp *MessagePool) selectMessagesOptimal(curTs, ts *types.TipSet, tq float64 for i := len(chainDeps) - 1; i >= 0; i-- { curChain := chainDeps[i] curChain.merged = true - // adjust the next chain for the parent, which is being merged - if next := curChain.next; next != nil && next.effPerf > 0 { - next.effPerf += next.parentOffset - } result = append(result, curChain.msgs...) } chain.merged = true + // adjust the effective pefromance for all subsequent chains if next := chain.next; next != nil && next.effPerf > 0 { next.effPerf += next.parentOffset + for next = next.next; next != nil && next.effPerf > 0; next = next.next { + next.setEffPerf() + } } result = append(result, chain.msgs...) gasLimit -= chainGasLimit From ff7e441c82fa0cec55c57ab8bc1172b9cac4c492 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 13 Aug 2020 18:57:06 +0200 Subject: [PATCH 24/36] Add noise to GasPremium calculation to help out message selection Signed-off-by: Jakub Sztandera --- node/impl/full/gas.go | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index 49f07b271..9f7d8907d 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -3,6 +3,7 @@ package full import ( "context" "math" + "math/rand" "sort" "github.com/filecoin-project/lotus/build" @@ -26,7 +27,7 @@ type GasAPI struct { Mpool *messagepool.MessagePool } -const MinGasPremium = 10e3 +const MinGasPremium = 100e3 const MaxSpendOnFeeDenom = 100 func (a *GasAPI) GasEstimateFeeCap(ctx context.Context, msg *types.Message, maxqueueblks int64, @@ -111,6 +112,7 @@ func (a *GasAPI) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, at := build.BlockGasTarget * int64(blocks) / 2 prev := big.Zero() + premium := big.Zero() for _, price := range prices { at -= price.limit if at > 0 { @@ -122,17 +124,26 @@ func (a *GasAPI) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, return types.BigAdd(price.price, big.NewInt(1)), nil } - return types.BigAdd(big.Div(types.BigAdd(price.price, prev), types.NewInt(2)), big.NewInt(1)), nil + premium = types.BigAdd(big.Div(types.BigAdd(price.price, prev), types.NewInt(2)), big.NewInt(1)) } - switch nblocksincl { - case 1: - return types.NewInt(2 * MinGasPremium), nil - case 2: - return types.NewInt(1.5 * MinGasPremium), nil - default: - return types.NewInt(MinGasPremium), nil + if types.BigCmp(premium, big.Zero()) == 0 { + switch nblocksincl { + case 1: + premium = types.NewInt(2 * MinGasPremium) + case 2: + premium = types.NewInt(1.5 * MinGasPremium) + default: + premium = types.NewInt(MinGasPremium) + } } + + const precision = 32 + // mean 1, stddev 0.005 => 95% within +-1% + noise := 1 + rand.NormFloat64()*0.005 + premium = types.BigMul(premium, types.NewInt(uint64(noise*(1< Date: Thu, 13 Aug 2020 20:02:08 +0300 Subject: [PATCH 25/36] make selection2 test less touchy --- chain/messagepool/selection_test.go | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index 0d82c09d8..ff93577c0 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -771,16 +771,26 @@ func TestOptimalMessageSelection2(t *testing.T) { t.Fatalf("expected %d messages, but got %d", expectedMsgs, len(msgs)) } - nextNonce := uint64(0) + var nFrom1, nFrom2 int + var nextNonce1, nextNonce2 uint64 for _, m := range msgs { - if m.Message.From != a2 { - t.Fatal("expected message from a2") + if m.Message.From == a1 { + if m.Message.Nonce != nextNonce1 { + t.Fatalf("expected nonce %d but got %d", nextNonce1, m.Message.Nonce) + } + nextNonce1++ + nFrom1++ + } else { + if m.Message.Nonce != nextNonce2 { + t.Fatalf("expected nonce %d but got %d", nextNonce2, m.Message.Nonce) + } + nextNonce2++ + nFrom2++ } + } - if m.Message.Nonce != nextNonce { - t.Fatalf("expected nonce %d but got %d", nextNonce, m.Message.Nonce) - } - nextNonce++ + if nFrom1 > nFrom2 { + t.Fatalf("expected more messages from a2 than a1; nFrom1=%d nFrom2=%d", nFrom1, nFrom2) } } From a6d79b26e449143be17866526ac1fc61e535e916 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 13 Aug 2020 19:27:31 +0200 Subject: [PATCH 26/36] Add comments Signed-off-by: Jakub Sztandera --- node/impl/full/gas.go | 1 + 1 file changed, 1 insertion(+) diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index 9f7d8907d..d501af2f7 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -138,6 +138,7 @@ func (a *GasAPI) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, } } + // add some noise to normalize behaviour of message selection const precision = 32 // mean 1, stddev 0.005 => 95% within +-1% noise := 1 + rand.NormFloat64()*0.005 From 55f88b46cfe4e35041a24342fd6ac4b23e0c2627 Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 13 Aug 2020 11:14:58 -0700 Subject: [PATCH 27/36] fix: best attempt at fixing the id_address_map --- .../processor/common_actors.go | 77 +++++++++---------- 1 file changed, 38 insertions(+), 39 deletions(-) diff --git a/cmd/lotus-chainwatch/processor/common_actors.go b/cmd/lotus-chainwatch/processor/common_actors.go index 3fbdbf170..5264a48c8 100644 --- a/cmd/lotus-chainwatch/processor/common_actors.go +++ b/cmd/lotus-chainwatch/processor/common_actors.go @@ -1,8 +1,8 @@ package processor import ( + "bytes" "context" - "fmt" "time" "golang.org/x/sync/errgroup" @@ -12,7 +12,12 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/events/state" + "github.com/filecoin-project/lotus/chain/types" + cw_util "github.com/filecoin-project/lotus/cmd/lotus-chainwatch/util" "github.com/filecoin-project/specs-actors/actors/builtin" + _init "github.com/filecoin-project/specs-actors/actors/builtin/init" + "github.com/filecoin-project/specs-actors/actors/util/adt" + typegen "github.com/whyrusleeping/cbor-gen" ) func (p *Processor) setupCommonActors() error { @@ -141,49 +146,41 @@ func (p Processor) storeActorAddresses(ctx context.Context, actors map[cid.Cid]A addressToID[builtin.StorageMarketActorAddr] = builtin.StorageMarketActorAddr addressToID[builtin.VerifiedRegistryActorAddr] = builtin.VerifiedRegistryActorAddr addressToID[builtin.BurntFundsActorAddr] = builtin.BurntFundsActorAddr - - addressesToUpdate := []UpdateAddresses{} - - pred := state.NewStatePredicates(p.node) - for _, act := range actors[builtin.InitActorCodeID] { - for _, info := range act { - changed, val, err := pred.OnInitActorChange(pred.OnAddressMapChange())(ctx, info.parentTsKey, info.tsKey) - if err != nil { - return err - } - if !changed { - continue - } - changes := val.(*state.InitActorAddressChanges) - for _, add := range changes.Added { - addressToID[add.PK] = add.ID - } - // we'll need to update any addresses that were modified, this indicates a reorg. - for _, mod := range changes.Modified { - addressesToUpdate = append(addressesToUpdate, UpdateAddresses{ - Old: mod.From, - New: mod.To, - }) - } - } - } - - updateTx, err := p.db.Begin() + initActor, err := p.node.StateGetActor(ctx, builtin.InitActorAddr, types.EmptyTSK) if err != nil { return err } - for _, updates := range addressesToUpdate { - if _, err := updateTx.Exec( - fmt.Sprintf("update id_address_map set id=%s, address=%s where id=%s and address=%s", updates.New.ID, updates.New.PK, updates.Old.ID, updates.Old.PK), - ); err != nil { - return err - } - } - if err := updateTx.Commit(); err != nil { + initActorRaw, err := p.node.ChainReadObj(ctx, initActor.Head) + if err != nil { return err } + var initActorState _init.State + if err := initActorState.UnmarshalCBOR(bytes.NewReader(initActorRaw)); err != nil { + return err + } + ctxStore := cw_util.NewAPIIpldStore(ctx, p.node) + addrMap, err := adt.AsMap(ctxStore, initActorState.AddressMap) + if err != nil { + return err + } + // gross.. + var actorID typegen.CborInt + if err := addrMap.ForEach(&actorID, func(key string) error { + longAddr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + shortAddr, err := address.NewIDAddress(uint64(actorID)) + if err != nil { + return err + } + addressToID[longAddr] = shortAddr + return nil + }); err != nil { + return err + } tx, err := p.db.Begin() if err != nil { return err @@ -215,8 +212,10 @@ create temp table iam (like id_address_map excluding constraints) on commit drop return err } - if _, err := tx.Exec(`insert into id_address_map select * from iam on conflict (id) do nothing`); err != nil { - return xerrors.Errorf("actor put: %w", err) + // HACK until chain watch can handle reorgs we need to update this table when ID -> PubKey mappings change + if _, err := tx.Exec(`insert into id_address_map select * from iam on conflict (id) do update set address = EXCLUDED.address`); err != nil { + log.Warnw("Failed to update id_address_map table, this is a known issue") + return nil } return tx.Commit() From f27a1227874cf8f82085d7a07f6d81f1c0485b77 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 13 Aug 2020 20:22:11 +0200 Subject: [PATCH 28/36] Use Zipf distribution for messages Signed-off-by: Jakub Sztandera --- chain/messagepool/selection_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index ff93577c0..9e1e7f489 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -2,7 +2,6 @@ package messagepool import ( "context" - "math" "math/big" "math/rand" "testing" @@ -916,13 +915,14 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) (float64, flo nMessages := 10 * int(build.BlockGasLimit/gasLimit) t.Log("nMessages", nMessages) nonces := make([]uint64, nActors) + zipf := rand.NewZipf(rng, 1.001, 1, 40000) for i := 0; i < nMessages; i++ { from := rng.Intn(nActors) to := rng.Intn(nActors) - premium := 20000*math.Exp(-3.*rand.Float64()) + 5000 nonce := nonces[from] nonces[from]++ - m := makeTestMessage(wallets[from], actors[from], actors[to], uint64(nonce), gasLimit, uint64(premium)) + premium := zipf.Uint64() + 10000 + m := makeTestMessage(wallets[from], actors[from], actors[to], uint64(nonce), gasLimit, premium) mustAdd(t, mp, m) } @@ -936,7 +936,7 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) (float64, flo capacityBoost := 0.0 rewardBoost := 0.0 - const runs = 1 + const runs = 50 for i := 0; i < runs; i++ { // 2. optimal selection minersRand := rng.Float64() From 50d4c2e0aca50fdbbc93049bb2faf5ab0f22bf55 Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 13 Aug 2020 14:10:05 -0700 Subject: [PATCH 29/36] fix: error checking for actor not found --- cmd/lotus-chainwatch/processor/miner.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/lotus-chainwatch/processor/miner.go b/cmd/lotus-chainwatch/processor/miner.go index a5f18a6a6..6d60f401d 100644 --- a/cmd/lotus-chainwatch/processor/miner.go +++ b/cmd/lotus-chainwatch/processor/miner.go @@ -319,7 +319,7 @@ func (p *Processor) storeMinerPreCommitInfo(ctx context.Context, miners []minerA changes, err := p.getMinerPreCommitChanges(ctx, m) if err != nil { - if strings.Contains(err.Error(), "address not found") { + if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) { continue } else { return err @@ -439,7 +439,7 @@ func (p *Processor) storeMinerSectorInfo(ctx context.Context, miners []minerActo for _, m := range miners { changes, err := p.getMinerSectorChanges(ctx, m) if err != nil { - if strings.Contains(err.Error(), "address not found") { + if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) { continue } else { return err @@ -518,7 +518,7 @@ func (p *Processor) getMinerPartitionsDifferences(ctx context.Context, miners [] m := m grp.Go(func() error { if err := p.diffMinerPartitions(ctx, m, events); err != nil { - if strings.Contains(err.Error(), "address not found") { + if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) { return nil } return err @@ -873,7 +873,7 @@ func (p *Processor) storeMinersActorInfoState(ctx context.Context, miners []mine for _, m := range miners { mi, err := p.node.StateMinerInfo(ctx, m.common.addr, m.common.tsKey) if err != nil { - if strings.Contains(err.Error(), "address not found") { + if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) { continue } else { return err From eb2879ca2244eef2d9d663e629534456c264bfd6 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 13 Aug 2020 17:08:46 -0700 Subject: [PATCH 30/36] allow client to specify provider collateral --- api/api_full.go | 17 +++++++++-------- cli/client.go | 30 ++++++++++++++++++++++-------- node/impl/client/client.go | 3 +-- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 55e958850..fe61afa26 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -609,14 +609,15 @@ type MethodCall struct { } type StartDealParams struct { - Data *storagemarket.DataRef - Wallet address.Address - Miner address.Address - EpochPrice types.BigInt - MinBlocksDuration uint64 - DealStartEpoch abi.ChainEpoch - FastRetrieval bool - VerifiedDeal bool + Data *storagemarket.DataRef + Wallet address.Address + Miner address.Address + EpochPrice types.BigInt + MinBlocksDuration uint64 + ProviderCollateral big.Int + DealStartEpoch abi.ChainEpoch + FastRetrieval bool + VerifiedDeal bool } type IpldObject struct { diff --git a/cli/client.go b/cli/client.go index 0e70a6faa..968146cc4 100644 --- a/cli/client.go +++ b/cli/client.go @@ -311,6 +311,10 @@ var clientDealCmd = &cli.Command{ Usage: "indicate that the deal counts towards verified client total", Value: false, }, + &cli.StringFlag{ + Name: "provider-collateral", + Usage: "specify the requested provider collateral the miner should put up", + }, &CidBaseFlag, }, Action: func(cctx *cli.Context) error { @@ -351,6 +355,15 @@ var clientDealCmd = &cli.Command{ return err } + var provCol big.Int + if pcs := cctx.String("provider-collateral"); pc != "" { + pc, err := big.FromString(pc) + if err != nil { + return fmt.Errorf("failed to parse provider-collateral: %w", err) + } + provCol = pc + } + if abi.ChainEpoch(dur) < build.MinDealDuration { return xerrors.Errorf("minimum deal duration is %d blocks", build.MinDealDuration) } @@ -415,14 +428,15 @@ var clientDealCmd = &cli.Command{ } proposal, err := api.ClientStartDeal(ctx, &lapi.StartDealParams{ - Data: ref, - Wallet: a, - Miner: miner, - EpochPrice: types.BigInt(price), - MinBlocksDuration: uint64(dur), - DealStartEpoch: abi.ChainEpoch(cctx.Int64("start-epoch")), - FastRetrieval: cctx.Bool("fast-retrieval"), - VerifiedDeal: isVerified, + Data: ref, + Wallet: a, + Miner: miner, + EpochPrice: types.BigInt(price), + MinBlocksDuration: uint64(dur), + DealStartEpoch: abi.ChainEpoch(cctx.Int64("start-epoch")), + FastRetrieval: cctx.Bool("fast-retrieval"), + VerifiedDeal: isVerified, + ProviderCollateral: provCol, }) if err != nil { return err diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 404487859..df9febe75 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -36,7 +36,6 @@ import ( "github.com/filecoin-project/go-multistore" "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/filecoin-project/specs-actors/actors/abi/big" "github.com/filecoin-project/specs-actors/actors/builtin/miner" marketevents "github.com/filecoin-project/lotus/markets/loggers" @@ -157,7 +156,7 @@ func (a *API) ClientStartDeal(ctx context.Context, params *api.StartDealParams) StartEpoch: dealStart, EndEpoch: calcDealExpiration(params.MinBlocksDuration, md, dealStart), Price: params.EpochPrice, - Collateral: big.Zero(), + Collateral: params.ProviderCollateral, Rt: rt, FastRetrieval: params.FastRetrieval, VerifiedDeal: params.VerifiedDeal, From b49276ef4768086421c7538a5a4ad5b44acad4ee Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 13 Aug 2020 21:54:39 -0400 Subject: [PATCH 31/36] Double provider collateral estimates in the client atadpter --- markets/storageadapter/client.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index f7627fa7a..31a63fb65 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -5,6 +5,7 @@ package storageadapter import ( "bytes" "context" + "github.com/filecoin-project/specs-actors/actors/abi/big" "golang.org/x/xerrors" @@ -230,7 +231,7 @@ func (c *ClientNodeAdapter) DealProviderCollateralBounds(ctx context.Context, si return abi.TokenAmount{}, abi.TokenAmount{}, err } - return bounds.Min, bounds.Max, nil + return big.Mul(bounds.Min, big.NewInt(2)), bounds.Max, nil } func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider address.Address, dealId abi.DealID, cb storagemarket.DealSectorCommittedCallback) error { From c2a24b7b4ce6116d204dfd6d6296380e7f27e032 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 13 Aug 2020 22:13:31 -0400 Subject: [PATCH 32/36] Update markets --- go.mod | 2 +- go.sum | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7a6d84fe5..13b154c26 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.6.1 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.5.5 + github.com/filecoin-project/go-fil-markets v0.5.6-0.20200814021159-7be996ed8ccb github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 diff --git a/go.sum b/go.sum index 95304002d..808632267 100644 --- a/go.sum +++ b/go.sum @@ -240,8 +240,8 @@ github.com/filecoin-project/go-data-transfer v0.6.1 h1:EA6X8fSiBRNVVwKm5pA7+njZn github.com/filecoin-project/go-data-transfer v0.6.1/go.mod h1:uRYBRKVBVM12CSusBtVrzDHkVw/3DKZpkxKJVP1Ydas= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.5.5 h1:6bDUXXWgMfhHLdp6KJJGfffOwM4lA7I8yJ9bSFLgnbQ= -github.com/filecoin-project/go-fil-markets v0.5.5/go.mod h1:9Sbm+N/WW2QpcmeDgEcQo7BJMPDbDpfHOvsYS9kT7zs= +github.com/filecoin-project/go-fil-markets v0.5.6-0.20200814021159-7be996ed8ccb h1:eCLqJb1tmhMCWUFAfJuSyyv/qLrqiAhICLjhUcbi4x8= +github.com/filecoin-project/go-fil-markets v0.5.6-0.20200814021159-7be996ed8ccb/go.mod h1:SJApXAKr5jyGpbzDEOhvemui0pih7hhT8r2MXJxCP1E= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM= github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0DzzQwqsL0XarpnI= @@ -254,6 +254,8 @@ github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261/g github.com/filecoin-project/go-statemachine v0.0.0-20200714194326-a77c3ae20989/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statemachine v0.0.0-20200730031800-c3336614d2a7 h1:KAF3WM/xSnl6G6RHX8vDJthg4+e4PSgBh72//6c6Qvc= github.com/filecoin-project/go-statemachine v0.0.0-20200730031800-c3336614d2a7/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= +github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370 h1:Jbburj7Ih2iaJ/o5Q9A+EAeTabME6YII7FLi9SKUf5c= +github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= From 2d90bcff3c8fecce01ddd274ba6f04bc2430a564 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 13 Aug 2020 22:14:45 -0400 Subject: [PATCH 33/36] jeromy is a bad software engineer --- cli/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/client.go b/cli/client.go index 968146cc4..41f0f8f8f 100644 --- a/cli/client.go +++ b/cli/client.go @@ -356,8 +356,8 @@ var clientDealCmd = &cli.Command{ } var provCol big.Int - if pcs := cctx.String("provider-collateral"); pc != "" { - pc, err := big.FromString(pc) + if pcs := cctx.String("provider-collateral"); pcs != "" { + pc, err := big.FromString(pcs) if err != nil { return fmt.Errorf("failed to parse provider-collateral: %w", err) } From 5ca410ae99c03de53bf80c3c9d298929a9ff62bd Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 13 Aug 2020 22:21:20 -0400 Subject: [PATCH 34/36] extract overestimation factor into a constant --- markets/storageadapter/client.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 31a63fb65..f621c3838 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -225,13 +225,15 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor return res.IDs[dealIdx], nil } +const clientOverestimation = 2 + func (c *ClientNodeAdapter) DealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, isVerified bool) (abi.TokenAmount, abi.TokenAmount, error) { bounds, err := c.StateDealProviderCollateralBounds(ctx, size, isVerified, types.EmptyTSK) if err != nil { return abi.TokenAmount{}, abi.TokenAmount{}, err } - return big.Mul(bounds.Min, big.NewInt(2)), bounds.Max, nil + return big.Mul(bounds.Min, big.NewInt(clientOverestimation)), bounds.Max, nil } func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider address.Address, dealId abi.DealID, cb storagemarket.DealSectorCommittedCallback) error { From b844f70a3c91965e66bb9ea5a4cfdf242ecbc944 Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 14 Aug 2020 10:09:25 +0300 Subject: [PATCH 35/36] make competitve selection test parameteric on distribution function, add test for Zipf distribution --- chain/messagepool/selection_test.go | 50 +++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index 9e1e7f489..d125eb4d9 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -2,6 +2,7 @@ package messagepool import ( "context" + "math" "math/big" "math/rand" "testing" @@ -873,9 +874,9 @@ func TestOptimalMessageSelection3(t *testing.T) { } } -func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) (float64, float64) { - // in this test we use 100 actors and send 10 blocks of messages. - // actors send with an exponentially decreasing premium. +func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand, getPremium func() uint64) (float64, float64) { + // in this test we use 300 actors and send 10 blocks of messages. + // actors send with an randomly distributed premium dictated by the getPremium function. // a number of miners select with varying ticket quality and we compare the // capacity and rewards of greedy selection -vs- optimal selection @@ -915,13 +916,12 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) (float64, flo nMessages := 10 * int(build.BlockGasLimit/gasLimit) t.Log("nMessages", nMessages) nonces := make([]uint64, nActors) - zipf := rand.NewZipf(rng, 1.001, 1, 40000) for i := 0; i < nMessages; i++ { from := rng.Intn(nActors) to := rng.Intn(nActors) nonce := nonces[from] nonces[from]++ - premium := zipf.Uint64() + 10000 + premium := getPremium() m := makeTestMessage(wallets[from], actors[from], actors[to], uint64(nonce), gasLimit, premium) mustAdd(t, mp, m) } @@ -936,7 +936,7 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) (float64, flo capacityBoost := 0.0 rewardBoost := 0.0 - const runs = 50 + const runs = 1 for i := 0; i < runs; i++ { // 2. optimal selection minersRand := rng.Float64() @@ -1003,12 +1003,44 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) (float64, flo return capacityBoost, rewardBoost } -func TestCompetitiveMessageSelection(t *testing.T) { +func makeExpPremiumDistribution(rng *rand.Rand) func() uint64 { + return func() uint64 { + premium := 20000*math.Exp(-3.*rng.Float64()) + 5000 + return uint64(premium) + } +} + +func makeZipfPremiumDistribution(rng *rand.Rand) func() uint64 { + zipf := rand.NewZipf(rng, 1.001, 1, 40000) + return func() uint64 { + return zipf.Uint64() + 10000 + } +} + +func TestCompetitiveMessageSelectionExp(t *testing.T) { var capacityBoost, rewardBoost float64 seeds := []int64{1947, 1976, 2020, 2100, 10000, 143324, 432432, 131, 32, 45} for _, seed := range seeds { - t.Log("running competitve message selection with seed", seed) - cb, rb := testCompetitiveMessageSelection(t, rand.New(rand.NewSource(seed))) + t.Log("running competitive message selection with Exponential premium distribution and seed", seed) + rng := rand.New(rand.NewSource(seed)) + cb, rb := testCompetitiveMessageSelection(t, rng, makeExpPremiumDistribution(rng)) + capacityBoost += cb + rewardBoost += rb + } + + capacityBoost /= float64(len(seeds)) + rewardBoost /= float64(len(seeds)) + t.Logf("Average capacity boost across all seeds: %f", capacityBoost) + t.Logf("Average reward boost across all seeds: %f", rewardBoost) +} + +func TestCompetitiveMessageSelectionZipf(t *testing.T) { + var capacityBoost, rewardBoost float64 + seeds := []int64{1947, 1976, 2020, 2100, 10000, 143324, 432432, 131, 32, 45} + for _, seed := range seeds { + t.Log("running competitive message selection with Zipf premium distribution and seed", seed) + rng := rand.New(rand.NewSource(seed)) + cb, rb := testCompetitiveMessageSelection(t, rng, makeZipfPremiumDistribution(rng)) capacityBoost += cb rewardBoost += rb } From a5d2ec0df8bd2ed25fdd5ab5549cf86aa2ff14ca Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 14 Aug 2020 10:12:55 +0300 Subject: [PATCH 36/36] appease linter --- chain/messagepool/selection_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index d125eb4d9..29a613256 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -922,7 +922,7 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand, getPremium fu nonce := nonces[from] nonces[from]++ premium := getPremium() - m := makeTestMessage(wallets[from], actors[from], actors[to], uint64(nonce), gasLimit, premium) + m := makeTestMessage(wallets[from], actors[from], actors[to], nonce, gasLimit, premium) mustAdd(t, mp, m) }