cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge remote-tracking branch 'origin/master' into feat/stateless-offline-dealflow

Browse Source
This commit is contained in:
Peter Rabbitson
2021-04-05 16:06:44 +02:00
parent 7fddbb528d eede19fb0b
commit b79be2a2c2
135 changed files with 5102 additions and 2274 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node/impl/client/client.go
+124 -66
View File
@@ -61,6 +61,7 @@ import (
"github.com/filecoin-project/lotus/node/impl/paych"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/filecoin-project/lotus/node/repo/retrievalstoremgr"
)
var DefaultHashFunction = uint64(mh.BLAKE2B_MIN + 31)
@@ -81,6 +82,7 @@ type API struct {
Chain *store.ChainStore
Imports dtypes.ClientImportMgr
Mds dtypes.ClientMultiDstore
CombinedBstore dtypes.ClientBlockstore // TODO: try to remove
RetrievalStoreMgr dtypes.ClientRetrievalStoreManager
@@ -369,10 +371,14 @@ func (a *API) newDealInfo(ctx context.Context, v storagemarket.ClientDeal) api.D
// be not found if it's no longer active
if err == nil {
ch := api.NewDataTransferChannel(a.Host.ID(), state)
ch.Stages = state.Stages()
transferCh = &ch
}
}
return a.newDealInfoWithTransfer(transferCh, v)
di := a.newDealInfoWithTransfer(transferCh, v)
di.DealStages = v.DealStages
return di
}
func (a *API) newDealInfoWithTransfer(transferCh *api.DataTransferChannel, v storagemarket.ClientDeal) api.DealInfo {
@@ -577,6 +583,29 @@ func (a *API) ClientListImports(ctx context.Context) ([]api.Import, error) {
return out, nil
}
func (a *API) ClientCancelRetrievalDeal(ctx context.Context, dealID retrievalmarket.DealID) error {
cerr := make(chan error)
go func() {
err := a.Retrieval.CancelDeal(dealID)
select {
case cerr <- err:
case <-ctx.Done():
}
}()
select {
case err := <-cerr:
if err != nil {
return xerrors.Errorf("failed to cancel retrieval deal: %w", err)
}
return nil
case <-ctx.Done():
return xerrors.Errorf("context timeout while canceling retrieval deal: %w", ctx.Err())
}
}
func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error {
events := make(chan marketevents.RetrievalEvent)
go a.clientRetrieve(ctx, order, ref, events)
@@ -657,86 +686,102 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
}
}
if order.MinerPeer.ID == "" {
mi, err := a.StateMinerInfo(ctx, order.Miner, types.EmptyTSK)
if err != nil {
finish(err)
var store retrievalstoremgr.RetrievalStore
if order.LocalStore == nil {
if order.MinerPeer == nil || order.MinerPeer.ID == "" {
mi, err := a.StateMinerInfo(ctx, order.Miner, types.EmptyTSK)
if err != nil {
finish(err)
return
}
order.MinerPeer = &retrievalmarket.RetrievalPeer{
ID: *mi.PeerId,
Address: order.Miner,
}
}
if order.Size == 0 {
finish(xerrors.Errorf("cannot make retrieval deal for zero bytes"))
return
}
order.MinerPeer = retrievalmarket.RetrievalPeer{
ID: *mi.PeerId,
Address: order.Miner,
/*id, st, err := a.imgr().NewStore()
if err != nil {
return err
}
}
if err := a.imgr().AddLabel(id, "source", "retrieval"); err != nil {
return err
}*/
if order.Size == 0 {
finish(xerrors.Errorf("cannot make retrieval deal for zero bytes"))
return
}
ppb := types.BigDiv(order.Total, types.NewInt(order.Size))
/*id, st, err := a.imgr().NewStore()
if err != nil {
return err
}
if err := a.imgr().AddLabel(id, "source", "retrieval"); err != nil {
return err
}*/
params, err := rm.NewParamsV1(ppb, order.PaymentInterval, order.PaymentIntervalIncrease, shared.AllSelector(), order.Piece, order.UnsealPrice)
if err != nil {
finish(xerrors.Errorf("Error in retrieval params: %s", err))
return
}
ppb := types.BigDiv(order.Total, types.NewInt(order.Size))
store, err = a.RetrievalStoreMgr.NewStore()
if err != nil {
finish(xerrors.Errorf("Error setting up new store: %w", err))
return
}
params, err := rm.NewParamsV1(ppb, order.PaymentInterval, order.PaymentIntervalIncrease, shared.AllSelector(), order.Piece, order.UnsealPrice)
if err != nil {
finish(xerrors.Errorf("Error in retrieval params: %s", err))
return
}
defer func() {
_ = a.RetrievalStoreMgr.ReleaseStore(store)
}()
store, err := a.RetrievalStoreMgr.NewStore()
if err != nil {
finish(xerrors.Errorf("Error setting up new store: %w", err))
return
}
defer func() {
_ = a.RetrievalStoreMgr.ReleaseStore(store)
}()
// Subscribe to events before retrieving to avoid losing events.
subscribeEvents := make(chan retrievalSubscribeEvent, 1)
subscribeCtx, cancel := context.WithCancel(ctx)
defer cancel()
unsubscribe := a.Retrieval.SubscribeToEvents(func(event rm.ClientEvent, state rm.ClientDealState) {
// We'll check the deal IDs inside readSubscribeEvents.
if state.PayloadCID.Equals(order.Root) {
select {
case <-subscribeCtx.Done():
case subscribeEvents <- retrievalSubscribeEvent{event, state}:
// Subscribe to events before retrieving to avoid losing events.
subscribeEvents := make(chan retrievalSubscribeEvent, 1)
subscribeCtx, cancel := context.WithCancel(ctx)
defer cancel()
unsubscribe := a.Retrieval.SubscribeToEvents(func(event rm.ClientEvent, state rm.ClientDealState) {
// We'll check the deal IDs inside readSubscribeEvents.
if state.PayloadCID.Equals(order.Root) {
select {
case <-subscribeCtx.Done():
case subscribeEvents <- retrievalSubscribeEvent{event, state}:
}
}
})
dealID, err := a.Retrieval.Retrieve(
ctx,
order.Root,
params,
order.Total,
*order.MinerPeer,
order.Client,
order.Miner,
store.StoreID())
if err != nil {
unsubscribe()
finish(xerrors.Errorf("Retrieve failed: %w", err))
return
}
})
dealID, err := a.Retrieval.Retrieve(
ctx,
order.Root,
params,
order.Total,
order.MinerPeer,
order.Client,
order.Miner,
store.StoreID())
err = readSubscribeEvents(ctx, dealID, subscribeEvents, events)
if err != nil {
unsubscribe()
finish(xerrors.Errorf("Retrieve failed: %w", err))
return
}
if err != nil {
finish(xerrors.Errorf("Retrieve: %w", err))
return
}
} else {
// local retrieval
st, err := ((*multistore.MultiStore)(a.Mds)).Get(*order.LocalStore)
if err != nil {
finish(xerrors.Errorf("Retrieve: %w", err))
return
}
err = readSubscribeEvents(ctx, dealID, subscribeEvents, events)
unsubscribe()
if err != nil {
finish(xerrors.Errorf("Retrieve: %w", err))
return
store = &multiStoreRetrievalStore{
storeID: *order.LocalStore,
store: st,
}
}
// If ref is nil, it only fetches the data into the configured blockstore.
@@ -776,6 +821,19 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
return
}
type multiStoreRetrievalStore struct {
storeID multistore.StoreID
store *multistore.Store
}
func (mrs *multiStoreRetrievalStore) StoreID() *multistore.StoreID {
return &mrs.storeID
}
func (mrs *multiStoreRetrievalStore) DAGService() ipld.DAGService {
return mrs.store.DAG
}
func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.StorageAsk, error) {
mi, err := a.StateMinerInfo(ctx, miner, types.EmptyTSK)
if err != nil {
node/impl/common/common.go
+5
View File
@@ -24,6 +24,7 @@ import (
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/filecoin-project/lotus/api"
apitypes "github.com/filecoin-project/lotus/api/types"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/modules/lp2p"
@@ -207,6 +208,10 @@ func (a *CommonAPI) NetBandwidthStatsByProtocol(ctx context.Context) (map[protoc
return a.Reporter.GetBandwidthByProtocol(), nil
}
func (a *CommonAPI) Discover(ctx context.Context) (apitypes.OpenRPCDocument, error) {
return build.OpenRPCDiscoverJSON_Full(), nil
}
func (a *CommonAPI) ID(context.Context) (peer.ID, error) {
return a.Host.ID(), nil
}
node/impl/full/gas.go
+61 -22
View File
@@ -8,6 +8,7 @@ import (
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
lru "github.com/hashicorp/golang-lru"
"go.uber.org/fx"
"golang.org/x/xerrors"
@@ -39,6 +40,8 @@ type GasModule struct {
Chain *store.ChainStore
Mpool *messagepool.MessagePool
GetMaxFee dtypes.DefaultMaxFeeFunc
PriceCache *GasPriceCache
}
var _ GasModuleAPI = (*GasModule)(nil)
@@ -51,6 +54,53 @@ type GasAPI struct {
Stmgr *stmgr.StateManager
Chain *store.ChainStore
Mpool *messagepool.MessagePool
PriceCache *GasPriceCache
}
func NewGasPriceCache() *GasPriceCache {
// 50 because we usually won't access more than 40
c, err := lru.New2Q(50)
if err != nil {
// err only if parameter is bad
panic(err)
}
return &GasPriceCache{
c: c,
}
}
type GasPriceCache struct {
c *lru.TwoQueueCache
}
type GasMeta struct {
Price big.Int
Limit int64
}
func (g *GasPriceCache) GetTSGasStats(cstore *store.ChainStore, ts *types.TipSet) ([]GasMeta, error) {
i, has := g.c.Get(ts.Key())
if has {
return i.([]GasMeta), nil
}
var prices []GasMeta
msgs, err := cstore.MessagesForTipset(ts)
if err != nil {
return nil, xerrors.Errorf("loading messages: %w", err)
}
for _, msg := range msgs {
prices = append(prices, GasMeta{
Price: msg.VMMessage().GasPremium,
Limit: msg.VMMessage().GasLimit,
})
}
g.c.Add(ts.Key(), prices)
return prices, nil
}
const MinGasPremium = 100e3
@@ -88,24 +138,19 @@ func gasEstimateFeeCap(cstore *store.ChainStore, msg *types.Message, maxqueueblk
return out, nil
}
type gasMeta struct {
price big.Int
limit int64
}
// finds 55th percntile instead of median to put negative pressure on gas price
func medianGasPremium(prices []gasMeta, blocks int) abi.TokenAmount {
func medianGasPremium(prices []GasMeta, blocks int) abi.TokenAmount {
sort.Slice(prices, func(i, j int) bool {
// sort desc by price
return prices[i].price.GreaterThan(prices[j].price)
return prices[i].Price.GreaterThan(prices[j].Price)
})
at := build.BlockGasTarget * int64(blocks) / 2 // 50th
at += build.BlockGasTarget * int64(blocks) / (2 * 20) // move 5% further
prev1, prev2 := big.Zero(), big.Zero()
for _, price := range prices {
prev1, prev2 = price.price, prev1
at -= price.limit
prev1, prev2 = price.Price, prev1
at -= price.Limit
if at < 0 {
break
}
@@ -126,7 +171,7 @@ func (a *GasAPI) GasEstimateGasPremium(
gaslimit int64,
_ types.TipSetKey,
) (types.BigInt, error) {
return gasEstimateGasPremium(a.Chain, nblocksincl)
return gasEstimateGasPremium(a.Chain, a.PriceCache, nblocksincl)
}
func (m *GasModule) GasEstimateGasPremium(
ctx context.Context,
@@ -135,14 +180,14 @@ func (m *GasModule) GasEstimateGasPremium(
gaslimit int64,
_ types.TipSetKey,
) (types.BigInt, error) {
return gasEstimateGasPremium(m.Chain, nblocksincl)
return gasEstimateGasPremium(m.Chain, m.PriceCache, nblocksincl)
}
func gasEstimateGasPremium(cstore *store.ChainStore, nblocksincl uint64) (types.BigInt, error) {
func gasEstimateGasPremium(cstore *store.ChainStore, cache *GasPriceCache, nblocksincl uint64) (types.BigInt, error) {
if nblocksincl == 0 {
nblocksincl = 1
}
var prices []gasMeta
var prices []GasMeta
var blocks int
ts := cstore.GetHeaviestTipSet()
@@ -157,17 +202,11 @@ func gasEstimateGasPremium(cstore *store.ChainStore, nblocksincl uint64) (types.
}
blocks += len(pts.Blocks())
msgs, err := cstore.MessagesForTipset(pts)
meta, err := cache.GetTSGasStats(cstore, pts)
if err != nil {
return types.BigInt{}, xerrors.Errorf("loading messages: %w", err)
}
for _, msg := range msgs {
prices = append(prices, gasMeta{
price: msg.VMMessage().GasPremium,
limit: msg.VMMessage().GasLimit,
})
return types.BigInt{}, err
}
prices = append(prices, meta...)
ts = pts
}
node/impl/full/gas_test.go
+5 -5
View File
@@ -12,27 +12,27 @@ import (
)
func TestMedian(t *testing.T) {
require.Equal(t, types.NewInt(5), medianGasPremium([]gasMeta{
require.Equal(t, types.NewInt(5), medianGasPremium([]GasMeta{
{big.NewInt(5), build.BlockGasTarget},
}, 1))
require.Equal(t, types.NewInt(10), medianGasPremium([]gasMeta{
require.Equal(t, types.NewInt(10), medianGasPremium([]GasMeta{
{big.NewInt(5), build.BlockGasTarget},
{big.NewInt(10), build.BlockGasTarget},
}, 1))
require.Equal(t, types.NewInt(15), medianGasPremium([]gasMeta{
require.Equal(t, types.NewInt(15), medianGasPremium([]GasMeta{
{big.NewInt(10), build.BlockGasTarget / 2},
{big.NewInt(20), build.BlockGasTarget / 2},
}, 1))
require.Equal(t, types.NewInt(25), medianGasPremium([]gasMeta{
require.Equal(t, types.NewInt(25), medianGasPremium([]GasMeta{
{big.NewInt(10), build.BlockGasTarget / 2},
{big.NewInt(20), build.BlockGasTarget / 2},
{big.NewInt(30), build.BlockGasTarget / 2},
}, 1))
require.Equal(t, types.NewInt(15), medianGasPremium([]gasMeta{
require.Equal(t, types.NewInt(15), medianGasPremium([]GasMeta{
{big.NewInt(10), build.BlockGasTarget / 2},
{big.NewInt(20), build.BlockGasTarget / 2},
{big.NewInt(30), build.BlockGasTarget / 2},
node/impl/full/state.go
+1 -1
View File
@@ -76,7 +76,7 @@ type StateAPI struct {
// TODO: the wallet here is only needed because we have the MinerCreateBlock
// API attached to the state API. It probably should live somewhere better
Wallet api.WalletAPI
Wallet api.Wallet
DefWallet wallet.Default
StateModuleAPI
node/impl/full/wallet.go
+3 -3
View File
@@ -22,7 +22,7 @@ type WalletAPI struct {
StateManagerAPI stmgr.StateManagerAPI
Default wallet.Default
api.WalletAPI
api.Wallet
}
func (a *WalletAPI) WalletBalance(ctx context.Context, addr address.Address) (types.BigInt, error) {
@@ -40,7 +40,7 @@ func (a *WalletAPI) WalletSign(ctx context.Context, k address.Address, msg []byt
if err != nil {
return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr)
}
return a.WalletAPI.WalletSign(ctx, keyAddr, msg, api.MsgMeta{
return a.Wallet.WalletSign(ctx, keyAddr, msg, api.MsgMeta{
Type: api.MTUnknown,
})
}
@@ -56,7 +56,7 @@ func (a *WalletAPI) WalletSignMessage(ctx context.Context, k address.Address, ms
return nil, xerrors.Errorf("serializing message: %w", err)
}
sig, err := a.WalletAPI.WalletSign(ctx, keyAddr, mb.Cid().Bytes(), api.MsgMeta{
sig, err := a.Wallet.WalletSign(ctx, keyAddr, mb.Cid().Bytes(), api.MsgMeta{
Type: api.MTChainMsg,
Extra: mb.RawData(),
})
node/impl/remoteworker.go
+1 -1
View File
@@ -16,7 +16,7 @@ import (
)
type remoteWorker struct {
api.WorkerAPI
api.Worker
closer jsonrpc.ClientCloser
}
node/impl/storminer.go
+15 -1
View File
@@ -8,6 +8,10 @@ import (
"strconv"
"time"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/build"
"github.com/google/uuid"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/host"
@@ -31,6 +35,7 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/apistruct"
apitypes "github.com/filecoin-project/lotus/api/types"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/markets/storageadapter"
"github.com/filecoin-project/lotus/miner"
@@ -61,7 +66,8 @@ type StorageMinerAPI struct {
AddrSel *storage.AddressSelector
DealPublisher *storageadapter.DealPublisher
DS dtypes.MetadataDS
Epp gen.WinningPoStProver
DS dtypes.MetadataDS
ConsiderOnlineStorageDealsConfigFunc dtypes.ConsiderOnlineStorageDealsConfigFunc
SetConsiderOnlineStorageDealsConfigFunc dtypes.SetConsiderOnlineStorageDealsConfigFunc
@@ -690,4 +696,12 @@ func (sm *StorageMinerAPI) ActorAddressConfig(ctx context.Context) (api.AddressC
return sm.AddrSel.AddressConfig, nil
}
func (sm *StorageMinerAPI) Discover(ctx context.Context) (apitypes.OpenRPCDocument, error) {
return build.OpenRPCDiscoverJSON_Miner(), nil
}
func (sm *StorageMinerAPI) ComputeProof(ctx context.Context, ssi []builtin.SectorInfo, rand abi.PoStRandomness) ([]builtin.PoStProof, error) {
return sm.Epp.ComputeProof(ctx, ssi, rand)
}
var _ api.StorageMiner = &StorageMinerAPI{}