cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge remote-tracking branch 'origin/testnet/3' into feat/dht-upgrade

Browse Source
This commit is contained in:
Łukasz Magiera
2020-04-24 00:50:23 +02:00
parent 18f4e9d828 96f9d58066
commit e6ce6211de
33 changed files with 725 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node/builder.go
-1
View File
@@ -283,7 +283,6 @@ func Online() Option {
Override(new(storage2.Prover), From(new(sectorstorage.SectorManager))),
Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks),
Override(new(sealing.TicketFn), modules.SealTicketGen),
Override(new(*storage.Miner), modules.StorageMiner),
Override(new(dtypes.NetworkName), modules.StorageNetworkName),
Override(new(beacon.RandomBeacon), modules.MinerRandomBeacon),
node/impl/full/mpool.go
+4
View File
@@ -118,3 +118,7 @@ func (a *MpoolAPI) MpoolGetNonce(ctx context.Context, addr address.Address) (uin
func (a *MpoolAPI) MpoolSub(ctx context.Context) (<-chan api.MpoolUpdate, error) {
return a.Mpool.Updates(ctx)
}
func (a *MpoolAPI) MpoolEstimateGasPrice(ctx context.Context, nblocksincl uint64, sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error) {
return a.Mpool.EstimateGasPrice(ctx, nblocksincl, sender, gaslimit, tsk)
}
node/impl/full/state.go
+125
View File
@@ -16,6 +16,7 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-amt-ipld/v2"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/state"
"github.com/filecoin-project/lotus/chain/stmgr"
@@ -32,6 +33,7 @@ import (
"github.com/filecoin-project/specs-actors/actors/builtin/market"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
samsig "github.com/filecoin-project/specs-actors/actors/builtin/multisig"
"github.com/filecoin-project/specs-actors/actors/builtin/power"
)
type StateAPI struct {
@@ -604,3 +606,126 @@ func (a *StateAPI) MsigGetAvailableBalance(ctx context.Context, addr address.Add
minBalance = types.BigMul(minBalance, types.NewInt(uint64(offset)))
return types.BigSub(act.Balance, minBalance), nil
}
func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr address.Address, snum abi.SectorNumber, tsk types.TipSetKey) (types.BigInt, error) {
ts, err := a.Chain.GetTipSetFromKey(tsk)
if err != nil {
return types.EmptyInt, xerrors.Errorf("loading tipset %s: %w", tsk, err)
}
act, err := a.StateManager.GetActor(maddr, ts)
if err != nil {
return types.EmptyInt, err
}
as := store.ActorStore(ctx, a.Chain.Blockstore())
var st miner.State
if err := as.Get(ctx, act.Head, &st); err != nil {
return types.EmptyInt, err
}
precommit, found, err := st.GetPrecommittedSector(as, snum)
if err != nil {
return types.EmptyInt, err
}
if !found {
return types.EmptyInt, xerrors.Errorf("no precommit found for sector %d", snum)
}
var dealWeights market.VerifyDealsOnSectorProveCommitReturn
{
var err error
params, err := actors.SerializeParams(&market.VerifyDealsOnSectorProveCommitParams{
DealIDs: precommit.Info.DealIDs,
SectorSize: st.GetSectorSize(),
SectorExpiry: precommit.Info.Expiration,
})
if err != nil {
return types.EmptyInt, err
}
ret, err := a.StateManager.Call(ctx, &types.Message{
From: maddr,
To: builtin.StorageMarketActorAddr,
Method: builtin.MethodsMarket.VerifyDealsOnSectorProveCommit,
GasLimit: 100000000000,
GasPrice: types.NewInt(0),
Params: params,
}, ts)
if err != nil {
return types.EmptyInt, err
}
if err := dealWeights.UnmarshalCBOR(bytes.NewReader(ret.MsgRct.Return)); err != nil {
return types.BigInt{}, err
}
}
initialPledge := big.Zero()
{
ssize, err := precommit.Info.RegisteredProof.SectorSize()
if err != nil {
return types.EmptyInt, err
}
params, err := actors.SerializeParams(&power.OnSectorProveCommitParams{
Weight: power.SectorStorageWeightDesc{
SectorSize: ssize,
Duration: precommit.Info.Expiration - ts.Height(), // NB: not exactly accurate, but should always lead us to *over* estimate, not under
DealWeight: dealWeights.DealWeight,
VerifiedDealWeight: dealWeights.VerifiedDealWeight,
},
})
if err != nil {
return types.EmptyInt, err
}
ret, err := a.StateManager.Call(ctx, &types.Message{
From: maddr,
To: builtin.StoragePowerActorAddr,
Method: builtin.MethodsPower.OnSectorProveCommit,
GasLimit: 10000000000,
GasPrice: types.NewInt(0),
Params: params,
}, ts)
if err != nil {
return types.EmptyInt, err
}
if err := initialPledge.UnmarshalCBOR(bytes.NewReader(ret.MsgRct.Return)); err != nil {
return types.BigInt{}, err
}
}
return initialPledge, nil
}
func (a *StateAPI) StateMinerAvailableBalance(ctx context.Context, maddr address.Address, tsk types.TipSetKey) (types.BigInt, error) {
ts, err := a.Chain.GetTipSetFromKey(tsk)
if err != nil {
return types.EmptyInt, xerrors.Errorf("loading tipset %s: %w", tsk, err)
}
act, err := a.StateManager.GetActor(maddr, ts)
if err != nil {
return types.EmptyInt, err
}
as := store.ActorStore(ctx, a.Chain.Blockstore())
var st miner.State
if err := as.Get(ctx, act.Head, &st); err != nil {
return types.EmptyInt, err
}
// TODO: !!!! Use method that doesnt trigger additional state mutations, this is going to cause lots of objects to be created and written to disk
log.Warnf("calling inefficient unlock vested funds method, fixme")
vested, err := st.UnlockVestedFunds(as, ts.Height())
if err != nil {
return types.EmptyInt, err
}
return types.BigAdd(st.GetAvailableBalance(act.Balance), vested), nil
}
node/impl/storminer.go
+8
View File
@@ -59,6 +59,14 @@ func (sm *StorageMinerAPI) ActorAddress(context.Context) (address.Address, error
return sm.Miner.Address(), nil
}
func (sm *StorageMinerAPI) MiningBase(ctx context.Context) (*types.TipSet, error) {
mb, err := sm.BlockMiner.GetBestMiningCandidate(ctx)
if err != nil {
return nil, err
}
return mb.TipSet, nil
}
func (sm *StorageMinerAPI) ActorSectorSize(ctx context.Context, addr address.Address) (abi.SectorSize, error) {
mi, err := sm.Full.StateMinerInfo(ctx, addr, types.EmptyTSK)
if err != nil {
node/modules/storageminer.go
+6 -40
View File
@@ -1,7 +1,6 @@
package modules
import (
"bytes"
"context"
"net/http"
"reflect"
@@ -35,12 +34,6 @@ import (
"github.com/filecoin-project/go-fil-markets/storedcounter"
paramfetch "github.com/filecoin-project/go-paramfetch"
"github.com/filecoin-project/go-statestore"
sectorstorage "github.com/filecoin-project/sector-storage"
"github.com/filecoin-project/sector-storage/ffiwrapper"
"github.com/filecoin-project/sector-storage/stores"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/crypto"
lapi "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/beacon"
@@ -53,6 +46,10 @@ import (
"github.com/filecoin-project/lotus/node/modules/helpers"
"github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/lotus/storage"
sectorstorage "github.com/filecoin-project/sector-storage"
"github.com/filecoin-project/sector-storage/ffiwrapper"
"github.com/filecoin-project/sector-storage/stores"
"github.com/filecoin-project/specs-actors/actors/abi"
sealing "github.com/filecoin-project/storage-fsm"
)
@@ -124,7 +121,7 @@ func SectorIDCounter(ds dtypes.MetadataDS) sealing.SectorIDCounter {
return &sidsc{sc}
}
func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h host.Host, ds dtypes.MetadataDS, sealer sectorstorage.SectorManager, sc sealing.SectorIDCounter, verif ffiwrapper.Verifier, tktFn sealing.TicketFn) (*storage.Miner, error) {
func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h host.Host, ds dtypes.MetadataDS, sealer sectorstorage.SectorManager, sc sealing.SectorIDCounter, verif ffiwrapper.Verifier) (*storage.Miner, error) {
maddr, err := minerAddrFromDS(ds)
if err != nil {
return nil, err
@@ -147,7 +144,7 @@ func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h
return nil, err
}
sm, err := storage.NewMiner(api, maddr, worker, h, ds, sealer, sc, verif, tktFn)
sm, err := storage.NewMiner(api, maddr, worker, h, ds, sealer, sc, verif)
if err != nil {
return nil, err
}
@@ -283,37 +280,6 @@ func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api lapi.FullNode
return m, nil
}
func SealTicketGen(fapi lapi.FullNode, ds dtypes.MetadataDS) (sealing.TicketFn, error) {
minerAddr, err := minerAddrFromDS(ds)
if err != nil {
return nil, err
}
entropy := new(bytes.Buffer)
if err := minerAddr.MarshalCBOR(entropy); err != nil {
return nil, err
}
return func(ctx context.Context, tok sealing.TipSetToken) (abi.SealRandomness, abi.ChainEpoch, error) {
tsk, err := types.TipSetKeyFromBytes(tok)
if err != nil {
return nil, 0, xerrors.Errorf("could not unmarshal TipSetToken to TipSetKey: %w", err)
}
ts, err := fapi.ChainGetTipSet(ctx, tsk)
if err != nil {
return nil, 0, xerrors.Errorf("getting TipSet for key failed: %w", err)
}
r, err := fapi.ChainGetRandomness(ctx, ts.Key(), crypto.DomainSeparationTag_SealRandomness, ts.Height()-build.SealRandomnessLookback, entropy.Bytes())
if err != nil {
return nil, 0, xerrors.Errorf("getting randomness for SealTicket failed: %w", err)
}
return abi.SealRandomness(r), ts.Height() - build.SealRandomnessLookback, nil
}, nil
}
func NewProviderRequestValidator(deals dtypes.ProviderDealStore) *requestvalidation.ProviderRequestValidator {
return requestvalidation.NewProviderRequestValidator(deals)
}
node/node_test.go
+50 -36
View File
@@ -79,7 +79,7 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
require.NoError(t, err)
nic := storedcounter.New(ds, datastore.NewKey("/storage/nextid"))
for i := 0; i < nPreseal; i++ {
for i := 0; i < nGenesisPreseals; i++ {
nic.Next()
}
nic.Next()
@@ -109,7 +109,7 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
// start node
var minerapi api.StorageMiner
mineBlock := make(chan struct{})
mineBlock := make(chan func(bool))
// TODO: use stop
_, err = node.New(ctx,
node.StorageMiner(&minerapi),
@@ -134,9 +134,9 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
err = minerapi.NetConnect(ctx, remoteAddrs)
require.NoError(t, err)*/
mineOne := func(ctx context.Context) error {
mineOne := func(ctx context.Context, cb func(bool)) error {
select {
case mineBlock <- struct{}{}:
case mineBlock <- cb:
return nil
case <-ctx.Done():
return ctx.Err()
@@ -146,7 +146,7 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
return test.TestStorageNode{StorageMiner: minerapi, MineOne: mineOne}
}
func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.TestStorageNode) {
func builder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) {
ctx := context.Background()
mn := mocknet.New(ctx)
@@ -182,7 +182,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
if err != nil {
t.Fatal(err)
}
genm, k, err := seed.PreSeal(maddr, abi.RegisteredProof_StackedDRG2KiBPoSt, 0, nPreseal, tdir, []byte("make genesis mem random"), nil)
genm, k, err := seed.PreSeal(maddr, abi.RegisteredProof_StackedDRG2KiBPoSt, 0, nGenesisPreseals, tdir, []byte("make genesis mem random"), nil)
if err != nil {
t.Fatal(err)
}
@@ -195,7 +195,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
genaccs = append(genaccs, genesis.Actor{
Type: genesis.TAccount,
Balance: big.NewInt(40000000000),
Balance: big.NewInt(5000000000000000000),
Meta: (&genesis.AccountMeta{Owner: wk.Address}).ActorMeta(),
})
@@ -238,16 +238,16 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
}
for i, full := range storage {
for i, def := range storage {
// TODO: support non-bootstrap miners
if i != 0 {
t.Fatal("only one storage node supported")
}
if full != 0 {
if def.Full != 0 {
t.Fatal("storage nodes only supported on the first full node")
}
f := fulls[full]
f := fulls[def.Full]
if _, err := f.FullNode.WalletImport(ctx, &keys[i].KeyInfo); err != nil {
t.Fatal(err)
}
@@ -276,26 +276,17 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
return fulls, storers
}
const nPreseal = 2
const nGenesisPreseals = 2
func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.TestStorageNode) {
func mockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) {
ctx := context.Background()
mn := mocknet.New(ctx)
fulls := make([]test.TestNode, nFull)
storers := make([]test.TestStorageNode, len(storage))
pk, _, err := crypto.GenerateEd25519Key(rand.Reader)
require.NoError(t, err)
minerPid, err := peer.IDFromPrivateKey(pk)
require.NoError(t, err)
var genbuf bytes.Buffer
if len(storage) > 1 {
panic("need more peer IDs")
}
// PRESEAL SECTION, TRY TO REPLACE WITH BETTER IN THE FUTURE
// TODO: would be great if there was a better way to fake the preseals
@@ -304,6 +295,7 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t
var maddrs []address.Address
var presealDirs []string
var keys []*wallet.Key
var pidKeys []crypto.PrivKey
for i := 0; i < len(storage); i++ {
maddr, err := address.NewIDAddress(genesis2.MinerStart + uint64(i))
if err != nil {
@@ -313,10 +305,23 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t
if err != nil {
t.Fatal(err)
}
genm, k, err := mockstorage.PreSeal(2048, maddr, nPreseal)
preseals := storage[i].Preseal
if preseals == test.PresealGenesis {
preseals = nGenesisPreseals
}
genm, k, err := mockstorage.PreSeal(2048, maddr, preseals)
if err != nil {
t.Fatal(err)
}
pk, _, err := crypto.GenerateEd25519Key(rand.Reader)
require.NoError(t, err)
minerPid, err := peer.IDFromPrivateKey(pk)
require.NoError(t, err)
genm.PeerId = minerPid
wk, err := wallet.NewKey(*k)
@@ -326,11 +331,12 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t
genaccs = append(genaccs, genesis.Actor{
Type: genesis.TAccount,
Balance: big.NewInt(40000000000),
Balance: big.NewInt(5000000000000000000),
Meta: (&genesis.AccountMeta{Owner: wk.Address}).ActorMeta(),
})
keys = append(keys, wk)
pidKeys = append(pidKeys, pk)
presealDirs = append(presealDirs, tdir)
maddrs = append(maddrs, maddr)
genms = append(genms, *genm)
@@ -369,16 +375,13 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t
}
}
for i, full := range storage {
for i, def := range storage {
// TODO: support non-bootstrap miners
if i != 0 {
t.Fatal("only one storage node supported")
}
if full != 0 {
if def.Full != 0 {
t.Fatal("storage nodes only supported on the first full node")
}
f := fulls[full]
f := fulls[def.Full]
if _, err := f.FullNode.WalletImport(ctx, &keys[i].KeyInfo); err != nil {
return nil, nil
}
@@ -386,10 +389,7 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t
return nil, nil
}
genMiner := maddrs[i]
wa := genms[i].Worker
storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn, node.Options(
storers[i] = testStorageNode(ctx, t, genms[i].Worker, maddrs[i], pidKeys[i], f, mn, node.Options(
node.Override(new(sectorstorage.SectorManager), func() (sectorstorage.SectorManager, error) {
return mock.NewMockSectorMgr(5, build.SectorSizes[0]), nil
}),
@@ -409,7 +409,7 @@ func TestAPI(t *testing.T) {
test.TestApis(t, builder)
}
func rpcBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.TestStorageNode) {
func rpcBuilder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) {
fullApis, storaApis := builder(t, nFull, storage)
fulls := make([]test.TestNode, nFull)
storers := make([]test.TestStorageNode, len(storage))
@@ -453,11 +453,15 @@ func TestAPIDealFlow(t *testing.T) {
logging.SetLogLevel("sub", "ERROR")
logging.SetLogLevel("storageminer", "ERROR")
test.TestDealFlow(t, mockSbBuilder, 10*time.Millisecond, false)
t.Run("TestDealFlow", func(t *testing.T) {
test.TestDealFlow(t, mockSbBuilder, 10*time.Millisecond, false)
})
t.Run("WithExportedCAR", func(t *testing.T) {
test.TestDealFlow(t, mockSbBuilder, 10*time.Millisecond, true)
})
t.Run("TestDoubleDealFlow", func(t *testing.T) {
test.TestDoubleDealFlow(t, mockSbBuilder, 10*time.Millisecond)
})
}
func TestAPIDealFlowReal(t *testing.T) {
@@ -473,3 +477,13 @@ func TestAPIDealFlowReal(t *testing.T) {
test.TestDealFlow(t, builder, time.Second, false)
}
func TestDealMining(t *testing.T) {
logging.SetLogLevel("miner", "ERROR")
logging.SetLogLevel("chainstore", "ERROR")
logging.SetLogLevel("chain", "ERROR")
logging.SetLogLevel("sub", "ERROR")
logging.SetLogLevel("storageminer", "ERROR")
test.TestDealMining(t, mockSbBuilder, 50*time.Millisecond, false)
}