cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge branch 'master' into feat/event-states

Browse Source
This commit is contained in:
Łukasz Magiera
2020-01-14 16:31:10 +01:00
parent 2fecf3a8f1 0011c321e2
commit 8a9707788c
58 changed files with 1165 additions and 528 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node/builder.go
+5 -3
View File
@@ -23,6 +23,7 @@ import (
"github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery"
"github.com/filecoin-project/go-fil-markets/storagemarket"
deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain"
"github.com/filecoin-project/lotus/chain/blocksync"
@@ -33,6 +34,7 @@ import (
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/markets/storageadapter"
"github.com/filecoin-project/lotus/miner"
@@ -58,7 +60,6 @@ type special struct{ id int }
//nolint:golint
var (
DefaultTransportsKey = special{0} // Libp2p option
PNetKey = special{1} // Option + multiret
DiscoveryHandlerKey = special{2} // Private type
AddrsFactoryKey = special{3} // Libp2p option
SmuxTransportKey = special{4} // Libp2p option
@@ -142,7 +143,6 @@ func libp2p() Option {
Override(new(peerstore.Peerstore), pstoremem.NewPeerstore),
Override(DefaultTransportsKey, lp2p.DefaultTransports),
Override(PNetKey, lp2p.PNet),
Override(new(lp2p.RawHost), lp2p.Host),
Override(new(host.Host), lp2p.RoutedHost),
@@ -195,6 +195,8 @@ func Online() Option {
Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages),
Override(new(sectorbuilder.Verifier), sectorbuilder.ProofVerifier),
Override(new(*types.VMSyscalls), vm.Syscalls),
Override(new(*store.ChainStore), modules.ChainStore),
Override(new(*stmgr.StateManager), stmgr.NewStateManager),
Override(new(*wallet.Wallet), wallet.NewWallet),
@@ -241,7 +243,7 @@ func Online() Option {
// Storage miner
ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner },
Override(new(storage.SectorBuilder), modules.SectorBuilder),
Override(new(sectorbuilder.Interface), modules.SectorBuilder),
Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks),
Override(new(storage.TicketFn), modules.SealTicketGen),
Override(new(*storage.Miner), modules.StorageMiner),
node/impl/storminer.go
+1 -1
View File
@@ -26,7 +26,7 @@ type StorageMinerAPI struct {
CommonAPI
SectorBuilderConfig *sectorbuilder.Config
SectorBuilder storage.SectorBuilder
SectorBuilder sectorbuilder.Interface
SectorBlocks *sectorblocks.SectorBlocks
Miner *storage.Miner
node/modules/chain.go
+2 -2
View File
@@ -73,8 +73,8 @@ func ChainBlockservice(bs dtypes.ChainBlockstore, rem dtypes.ChainExchange) dtyp
return blockservice.New(bs, rem)
}
func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS) *store.ChainStore {
chain := store.NewChainStore(bs, ds)
func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS, syscalls *types.VMSyscalls) *store.ChainStore {
chain := store.NewChainStore(bs, ds, syscalls)
if err := chain.Load(); err != nil {
log.Warnf("loading chain state from disk: %s", err)
node/modules/client.go
+12 -16
View File
@@ -5,40 +5,36 @@ import (
"path/filepath"
"reflect"
"github.com/filecoin-project/lotus/markets/retrievaladapter"
"github.com/filecoin-project/go-data-transfer/impl/graphsync"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery"
retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl"
"github.com/filecoin-project/go-fil-markets/storagemarket"
deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
"github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/lotus/node/modules/helpers"
"github.com/filecoin-project/lotus/paych"
"github.com/ipfs/go-bitswap"
"github.com/ipfs/go-bitswap/network"
graphsync "github.com/ipfs/go-graphsync/impl"
"github.com/ipfs/go-graphsync/ipldbridge"
gsnet "github.com/ipfs/go-graphsync/network"
"github.com/ipfs/go-graphsync/storeutil"
"github.com/libp2p/go-libp2p-core/host"
"github.com/libp2p/go-libp2p-core/routing"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
"github.com/ipfs/go-filestore"
graphsync "github.com/ipfs/go-graphsync/impl"
"github.com/ipfs/go-graphsync/ipldbridge"
gsnet "github.com/ipfs/go-graphsync/network"
"github.com/ipfs/go-graphsync/storeutil"
blockstore "github.com/ipfs/go-ipfs-blockstore"
"github.com/ipfs/go-merkledag"
"github.com/libp2p/go-libp2p-core/host"
"github.com/libp2p/go-libp2p-core/routing"
"go.uber.org/fx"
"github.com/filecoin-project/go-data-transfer/impl/graphsync"
deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
"github.com/filecoin-project/lotus/markets/retrievaladapter"
payapi "github.com/filecoin-project/lotus/node/impl/paych"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/modules/helpers"
"github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/lotus/paych"
)
func ClientFstore(r repo.LockedRepo) (dtypes.ClientFilestore, error) {
node/modules/lp2p/host.go
+1
View File
@@ -79,6 +79,7 @@ func DHTRouting(client bool) interface{} {
dhtopts.Client(client),
dhtopts.Datastore(dstore),
dhtopts.Validator(validator),
dhtopts.Protocols("/lotus/kad/1.0.0"),
)
if err != nil {
node/modules/lp2p/pnet.go
-63
View File
@@ -1,63 +0,0 @@
package lp2p
import (
"fmt"
"strings"
"github.com/libp2p/go-libp2p"
pnet "github.com/libp2p/go-libp2p-pnet"
)
var LotusKey = "/key/swarm/psk/1.0.0/\n/base16/\n20c72388e6299c7bbc1b501fdcc8abe4f89f798e9b93b2d2bc02e3c29b6a088e"
type PNetFingerprint []byte
func PNet() (opts Libp2pOpts, fp PNetFingerprint, err error) {
protec, err := pnet.NewProtector(strings.NewReader(LotusKey))
if err != nil {
return opts, nil, fmt.Errorf("failed to configure private network: %s", err)
}
fp = protec.Fingerprint()
opts.Opts = append(opts.Opts, libp2p.PrivateNetwork(protec))
return opts, fp, nil
}
/*
func PNetChecker(repo repo.Repo, ph host.Host, lc fx.Lifecycle) error {
// TODO: better check?
swarmkey, err := repo.SwarmKey()
if err != nil || swarmkey == nil {
return err
}
done := make(chan struct{})
lc.Append(fx.Hook{
OnStart: func(_ context.Context) error {
go func() {
t := time.NewTicker(30 * time.Second)
defer t.Stop()
<-t.C // swallow one tick
for {
select {
case <-t.C:
if len(ph.Network().Peers()) == 0 {
log.Warn("We are in private network and have no peers.")
log.Warn("This might be configuration mistake.")
}
case <-done:
return
}
}
}()
return nil
},
OnStop: func(_ context.Context) error {
close(done)
return nil
},
})
return nil
}
*/
node/modules/storageminer.go
+1 -1
View File
@@ -98,7 +98,7 @@ func SectorBuilderConfig(storagePath string, threads uint, noprecommit, nocommit
}
}
func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, sb storage.SectorBuilder, tktFn storage.TicketFn) (*storage.Miner, error) {
func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, sb sectorbuilder.Interface, tktFn storage.TicketFn) (*storage.Miner, error) {
maddr, err := minerAddrFromDS(ds)
if err != nil {
return nil, err
node/modules/testing/genesis.go
+6 -6
View File
@@ -30,8 +30,8 @@ import (
var glog = logging.Logger("genesis")
func MakeGenesisMem(out io.Writer, gmc *gen.GenMinerCfg) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
return func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
func MakeGenesisMem(out io.Writer, gmc *gen.GenMinerCfg) func(bs dtypes.ChainBlockstore, w *wallet.Wallet, syscalls *types.VMSyscalls) modules.Genesis {
return func(bs dtypes.ChainBlockstore, w *wallet.Wallet, syscalls *types.VMSyscalls) modules.Genesis {
return func() (*types.BlockHeader, error) {
glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network")
defk, err := w.GenerateKey(types.KTBLS)
@@ -51,7 +51,7 @@ func MakeGenesisMem(out io.Writer, gmc *gen.GenMinerCfg) func(bs dtypes.ChainBlo
alloc[waddr] = types.FromFil(10000)
}
b, err := gen.MakeGenesisBlock(bs, alloc, gmc, 100000)
b, err := gen.MakeGenesisBlock(bs, syscalls, alloc, gmc, 100000)
if err != nil {
return nil, err
}
@@ -68,8 +68,8 @@ func MakeGenesisMem(out io.Writer, gmc *gen.GenMinerCfg) func(bs dtypes.ChainBlo
}
}
func MakeGenesis(outFile, presealInfo, timestamp string) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
return func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
func MakeGenesis(outFile, presealInfo, timestamp string) func(bs dtypes.ChainBlockstore, w *wallet.Wallet, syscalls *types.VMSyscalls) modules.Genesis {
return func(bs dtypes.ChainBlockstore, w *wallet.Wallet, syscalls *types.VMSyscalls) modules.Genesis {
return func() (*types.BlockHeader, error) {
glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network")
presealInfo, err := homedir.Expand(presealInfo)
@@ -130,7 +130,7 @@ func MakeGenesis(outFile, presealInfo, timestamp string) func(bs dtypes.ChainBlo
ts = uint64(t.Unix())
}
b, err := gen.MakeGenesisBlock(bs, addrs, gmc, ts)
b, err := gen.MakeGenesisBlock(bs, syscalls, addrs, gmc, ts)
if err != nil {
return nil, err
}
node/node_test.go
+111 -3
View File
@@ -8,6 +8,7 @@ import (
"net/http/httptest"
"path/filepath"
"testing"
"time"
"github.com/filecoin-project/go-address"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
@@ -36,6 +37,7 @@ import (
"github.com/filecoin-project/lotus/node/modules"
modtest "github.com/filecoin-project/lotus/node/modules/testing"
"github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/lotus/storage/sbmock"
)
func init() {
@@ -45,7 +47,7 @@ func init() {
build.MinimumMinerPower = 1024
}
func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, pk crypto.PrivKey, tnd test.TestNode, mn mocknet.Mocknet) test.TestStorageNode {
func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, pk crypto.PrivKey, tnd test.TestNode, mn mocknet.Mocknet, opts node.Option) test.TestStorageNode {
r := repo.NewMemory(nil)
lr, err := r.Lock(repo.StorageMiner)
@@ -105,6 +107,8 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
node.Override(new(api.FullNode), tnd),
node.Override(new(*miner.Miner), miner.NewTestMiner(mineBlock, act)),
opts,
)
if err != nil {
t.Fatalf("failed to construct node: %v", err)
@@ -214,7 +218,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
genMiner := gmc.MinerAddrs[i]
wa := gmc.PreSeals[genMiner.String()].Worker
storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn)
storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn, node.Options())
sma := storers[i].StorageMiner.(*impl.StorageMinerAPI)
@@ -247,6 +251,106 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
return fulls, storers
}
func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.TestStorageNode) {
ctx := context.Background()
mn := mocknet.New(ctx)
fulls := make([]test.TestNode, nFull)
storers := make([]test.TestStorageNode, len(storage))
pk, _, err := crypto.GenerateEd25519Key(rand.Reader)
require.NoError(t, err)
minerPid, err := peer.IDFromPrivateKey(pk)
require.NoError(t, err)
var genbuf bytes.Buffer
if len(storage) > 1 {
panic("need more peer IDs")
}
// PRESEAL SECTION, TRY TO REPLACE WITH BETTER IN THE FUTURE
// TODO: would be great if there was a better way to fake the preseals
gmc := &gen.GenMinerCfg{
PeerIDs: []peer.ID{minerPid}, // TODO: if we have more miners, need more peer IDs
PreSeals: map[string]genesis.GenesisMiner{},
}
var presealDirs []string
for i := 0; i < len(storage); i++ {
maddr, err := address.NewIDAddress(300 + uint64(i))
if err != nil {
t.Fatal(err)
}
tdir, err := ioutil.TempDir("", "preseal-memgen")
if err != nil {
t.Fatal(err)
}
genm, err := sbmock.PreSeal(1024, maddr, 1)
if err != nil {
t.Fatal(err)
}
presealDirs = append(presealDirs, tdir)
gmc.MinerAddrs = append(gmc.MinerAddrs, maddr)
gmc.PreSeals[maddr.String()] = *genm
}
// END PRESEAL SECTION
for i := 0; i < nFull; i++ {
var genesis node.Option
if i == 0 {
genesis = node.Override(new(modules.Genesis), modtest.MakeGenesisMem(&genbuf, gmc))
} else {
genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes()))
}
var err error
// TODO: Don't ignore stop
_, err = node.New(ctx,
node.FullAPI(&fulls[i].FullNode),
node.Online(),
node.Repo(repo.NewMemory(nil)),
node.MockHost(mn),
node.Test(),
node.Override(new(sectorbuilder.Verifier), sbmock.MockVerifier),
genesis,
)
if err != nil {
t.Fatal(err)
}
}
for i, full := range storage {
// TODO: support non-bootstrap miners
if i != 0 {
t.Fatal("only one storage node supported")
}
if full != 0 {
t.Fatal("storage nodes only supported on the first full node")
}
f := fulls[full]
genMiner := gmc.MinerAddrs[i]
wa := gmc.PreSeals[genMiner.String()].Worker
storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn, node.Options(
node.Override(new(sectorbuilder.Interface), sbmock.NewMockSectorBuilder(5, build.SectorSizes[0])),
))
}
if err := mn.LinkAll(); err != nil {
t.Fatal(err)
}
return fulls, storers
}
func TestAPI(t *testing.T) {
test.TestApis(t, builder)
}
@@ -289,8 +393,12 @@ func TestAPIRPC(t *testing.T) {
}
func TestAPIDealFlow(t *testing.T) {
test.TestDealFlow(t, mockSbBuilder, 10 * time.Millisecond)
}
func TestAPIDealFlowReal(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode")
}
test.TestDealFlow(t, builder)
test.TestDealFlow(t, builder, time.Second)
}