Merge pull request #30 from filecoin-project/lotus-testground-initial

testplan instantiating lotus nodes in-memory

lotus-testground/go.mod

@@ -0,0 +1,24 @@
+module github.com/testground/testground/plans/network
+
+go 1.14
+
+require (
+	github.com/davecgh/go-spew v1.1.1
+	github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef
+	github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
+	github.com/filecoin-project/lotus v0.3.1-0.20200518172415-1ed618334471
+	github.com/filecoin-project/sector-storage v0.0.0-20200522011946-a59ca7536a95
+	github.com/filecoin-project/specs-actors v0.5.4-0.20200521014528-0df536f7e461
+	github.com/ipfs/go-datastore v0.4.4
+	github.com/ipfs/go-ipfs-blockstore v1.0.0
+	github.com/labstack/gommon v0.3.0
+	github.com/libp2p/go-libp2p v0.9.4
+	github.com/libp2p/go-libp2p-core v0.5.7
+	github.com/libp2p/go-libp2p-crypto v0.1.0
+	github.com/testground/sdk-go v0.2.0
+	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
+)
+
+replace github.com/filecoin-project/filecoin-ffi => ../lotus/extern/filecoin-ffi
+
+replace github.com/filecoin-project/lotus => ../lotus

lotus-testground/main.go

@@ -0,0 +1,443 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"time"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/ipfs/go-datastore"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	peer "github.com/libp2p/go-libp2p-core/peer"
+	crypto "github.com/libp2p/go-libp2p-crypto"
+	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
+	"github.com/testground/sdk-go/network"
+	"github.com/testground/sdk-go/runtime"
+	"github.com/testground/sdk-go/sync"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-storedcounter"
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/vm"
+	"github.com/filecoin-project/lotus/chain/wallet"
+	"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
+	"github.com/filecoin-project/lotus/genesis"
+	"github.com/filecoin-project/lotus/miner"
+	"github.com/filecoin-project/lotus/node"
+	"github.com/filecoin-project/lotus/node/modules"
+	modtest "github.com/filecoin-project/lotus/node/modules/testing"
+	"github.com/filecoin-project/lotus/node/repo"
+	"github.com/filecoin-project/sector-storage/ffiwrapper"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/specs-actors/actors/builtin"
+	saminer "github.com/filecoin-project/specs-actors/actors/builtin/miner"
+	"github.com/filecoin-project/specs-actors/actors/builtin/power"
+	"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
+)
+
+func init() {
+	power.ConsensusMinerMinPower = big.NewInt(2048)
+	saminer.SupportedProofTypes = map[abi.RegisteredProof]struct{}{
+		abi.RegisteredProof_StackedDRG2KiBSeal: {},
+	}
+	verifreg.MinVerifiedDealSize = big.NewInt(256)
+}
+
+var testcases = map[string]runtime.TestCaseFn{
+	"lotus-network": lotusNetwork,
+}
+
+type GenesisMessage struct {
+	GenBuf []byte
+}
+
+func main() {
+	runtime.InvokeMap(testcases)
+}
+
+func setupLotusNode(genesis []byte) (api.FullNode, error) {
+	ctx := context.Background()
+	mn := mocknet.New(ctx)
+
+	genOpt := node.Override(new(modules.Genesis), modules.LoadGenesis(genesis))
+
+	var fullNode api.FullNode
+	_, err := node.New(ctx,
+		node.FullAPI(&fullNode),
+		node.Online(),
+		node.Repo(repo.NewMemory(nil)),
+		node.MockHost(mn),
+		node.Test(),
+
+		genOpt,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return fullNode, nil
+}
+
+func testStorageNode(ctx context.Context, waddr address.Address, act address.Address, tnd api.FullNode, opts node.Option, numPreSeals int) (api.StorageMiner, error) {
+	r := repo.NewMemory(nil)
+
+	pk, _, err := crypto.GenerateEd25519Key(rand.Reader)
+	if err != nil {
+		return nil, err
+	}
+
+	lr, err := r.Lock(repo.StorageMiner)
+	if err != nil {
+		return nil, err
+	}
+
+	ks, err := lr.KeyStore()
+	if err != nil {
+		return nil, err
+	}
+
+	kbytes, err := pk.Bytes()
+	if err != nil {
+		return nil, err
+	}
+
+	err = ks.Put("libp2p-host", types.KeyInfo{
+		Type:       "libp2p-host",
+		PrivateKey: kbytes,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	ds, err := lr.Datastore("/metadata")
+	if err != nil {
+		return nil, err
+	}
+
+	err = ds.Put(datastore.NewKey("miner-address"), act.Bytes())
+	if err != nil {
+		return nil, err
+	}
+
+	nic := storedcounter.New(ds, datastore.NewKey("/storage/nextid"))
+	for i := 0; i < numPreSeals; i++ {
+		nic.Next()
+	}
+	nic.Next()
+
+	if err := lr.Close(); err != nil {
+		return nil, err
+	}
+
+	peerid, err := peer.IDFromPrivateKey(pk)
+	if err != nil {
+		return nil, err
+	}
+
+	enc, err := actors.SerializeParams(&saminer.ChangePeerIDParams{NewID: peerid})
+	if err != nil {
+		return nil, err
+	}
+
+	msg := &types.Message{
+		To:       act,
+		From:     waddr,
+		Method:   builtin.MethodsMiner.ChangePeerID,
+		Params:   enc,
+		Value:    types.NewInt(0),
+		GasPrice: types.NewInt(0),
+		GasLimit: 1000000,
+	}
+
+	if _, err := tnd.MpoolPushMessage(ctx, msg); err != nil {
+		return nil, err
+	}
+
+	// start node
+	var minerapi api.StorageMiner
+
+	mineBlock := make(chan func(bool))
+	// TODO: use stop
+	_, err = node.New(ctx,
+		node.StorageMiner(&minerapi),
+		node.Online(),
+		node.Repo(r),
+		node.Test(),
+
+		node.Override(new(api.FullNode), tnd),
+		node.Override(new(*miner.Miner), miner.NewTestMiner(mineBlock, act)),
+
+		opts,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return minerapi, nil
+}
+
+func setupStorageNode(fnode api.FullNode, key *wallet.Key, maddr, worker address.Address, preSealDir string, numPreSeals int) (api.StorageMiner, error) {
+	ctx := context.TODO()
+	if _, err := fnode.WalletImport(ctx, &key.KeyInfo); err != nil {
+		return nil, err
+	}
+	if err := fnode.WalletSetDefault(ctx, key.Address); err != nil {
+		return nil, err
+	}
+
+	storageNode, err := testStorageNode(ctx, worker, maddr, fnode, node.Options(), numPreSeals)
+	if err != nil {
+		return nil, err
+	}
+
+	if preSealDir != "" {
+		if err := storageNode.StorageAddLocal(ctx, preSealDir); err != nil {
+			return nil, err
+		}
+	}
+	return storageNode, nil
+}
+
+type PreSealInfo struct {
+	Dir      string
+	GenAct   genesis.Actor
+	GenMiner genesis.Miner
+	WKey     *wallet.Key
+}
+
+const nGenesisPreseals = 4
+
+func runPreSeal(runenv *runtime.RunEnv, maddr address.Address, minerPid peer.ID) (*PreSealInfo, error) {
+	tdir, err := ioutil.TempDir("", "preseal-memgen")
+	if err != nil {
+		return nil, err
+	}
+
+	genm, k, err := seed.PreSeal(maddr, abi.RegisteredProof_StackedDRG2KiBPoSt, 0, nGenesisPreseals, tdir, []byte("make genesis mem random"), nil)
+	if err != nil {
+		return nil, err
+	}
+	genm.PeerId = minerPid
+
+	wk, err := wallet.NewKey(*k)
+	if err != nil {
+		return nil, nil
+	}
+
+	genAct := genesis.Actor{
+		Type:    genesis.TAccount,
+		Balance: big.NewInt(5000000000000000000),
+		Meta:    (&genesis.AccountMeta{Owner: wk.Address}).ActorMeta(),
+	}
+
+	return &PreSealInfo{
+		Dir:      tdir,
+		GenAct:   genAct,
+		GenMiner: *genm,
+		WKey:     wk,
+	}, nil
+}
+
+func lotusNetwork(runenv *runtime.RunEnv) error {
+
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second)
+	defer cancel()
+
+	runenv.RecordMessage("before sync.MustBoundClient")
+	client := sync.MustBoundClient(ctx, runenv)
+	defer client.Close()
+
+	if !runenv.TestSidecar {
+		//return nil
+	}
+
+	netclient := network.NewClient(client, runenv)
+	runenv.RecordMessage("before netclient.MustWaitNetworkInitialized")
+	netclient.MustWaitNetworkInitialized(ctx)
+
+	minerCount := runenv.IntParam("miner-count")
+	if minerCount > runenv.TestInstanceCount {
+		return fmt.Errorf("cannot have more miners than nodes")
+	}
+
+	config := &network.Config{
+		// Control the "default" network. At the moment, this is the only network.
+		Network: "default",
+
+		// Enable this network. Setting this to false will disconnect this test
+		// instance from this network. You probably don't want to do that.
+		Enable:        true,
+		CallbackState: "network-configured",
+	}
+
+	_ = config
+	runenv.RecordMessage("before netclient.MustConfigureNetwork")
+	//netclient.MustConfigureNetwork(ctx, config)
+
+	seq := client.MustSignalAndWait(ctx, "ip-allocation", runenv.TestInstanceCount)
+
+	var maddr address.Address
+	var minerPid peer.ID
+	var mpsi *PreSealInfo
+
+	preSealTopic := sync.NewTopic("preseals", &PreSealInfo{})
+	if seq <= int64(minerCount) { // sequence numbers start at 1
+		runenv.RecordMessage("Running preseal (seq = %d)", seq)
+
+		pk, _, err := crypto.GenerateEd25519Key(rand.Reader)
+		if err != nil {
+			return err
+		}
+
+		mpid, err := peer.IDFromPrivateKey(pk)
+		if err != nil {
+			return err
+		}
+		minerPid = mpid
+
+		ma, err := address.NewIDAddress(uint64(1000 + seq - 1))
+		if err != nil {
+			return err
+		}
+		maddr = ma
+
+		psi, err := runPreSeal(runenv, maddr, minerPid)
+		if err != nil {
+			return err
+		}
+
+		mpsi = psi
+		client.Publish(ctx, preSealTopic, psi)
+	}
+
+	genesisTopic := sync.NewTopic("genesis", &GenesisMessage{})
+
+	var genesisBytes []byte
+
+	if seq == 1 {
+		var genaccs []genesis.Actor
+		var genms []genesis.Miner
+
+		var preSeals []*PreSealInfo
+
+		psch := make(chan *PreSealInfo)
+		client.MustSubscribe(ctx, preSealTopic, psch)
+		for i := 0; i < minerCount; i++ {
+			psi := <-psch
+			preSeals = append(preSeals, psi)
+			genms = append(genms, psi.GenMiner)
+			genaccs = append(genaccs, psi.GenAct)
+		}
+
+		runenv.RecordMessage("have %d genesis miners", len(preSeals))
+
+		templ := &genesis.Template{
+			Accounts:  genaccs,
+			Miners:    genms,
+			Timestamp: uint64(time.Now().Unix() - 10000), // some time sufficiently far in the past
+		}
+
+		runenv.RecordMessage("genminer: %s %s", genms[0].Owner, genms[0].Worker)
+
+		runenv.RecordMessage("making a genesis file: %d %d", len(templ.Accounts), len(templ.Miners))
+
+		bs := blockstore.NewBlockstore(datastore.NewMapDatastore())
+		bs = blockstore.NewIdStore(bs)
+		var genbuf bytes.Buffer
+		_, err := modtest.MakeGenesisMem(&genbuf, *templ)(bs, vm.Syscalls(&genFakeVerifier{}))()
+		if err != nil {
+			runenv.RecordMessage("genesis file failure: %v", err)
+			return xerrors.Errorf("failed to make genesis file: %w", err)
+		}
+
+		runenv.RecordMessage("now broadcasting genesis file (len = %d)", genbuf.Len())
+
+		genesisBytes = genbuf.Bytes()
+		client.MustPublish(ctx, genesisTopic, &GenesisMessage{
+			GenBuf: genbuf.Bytes(),
+		})
+	} else {
+		gench := make(chan *GenesisMessage)
+		client.MustSubscribe(ctx, genesisTopic, gench)
+
+		genm := <-gench
+		genesisBytes = genm.GenBuf
+	}
+
+	runenv.RecordMessage("about to set up lotus node (len = %d)", len(genesisBytes))
+	lnode, err := setupLotusNode(genesisBytes)
+	if err != nil {
+		return err
+	}
+
+	id, err := lnode.ID(ctx)
+	if err != nil {
+		return err
+	}
+
+	runenv.RecordMessage("Lotus node ID is: %s", id)
+
+	gents, err := lnode.ChainGetGenesis(ctx)
+	if err != nil {
+		return err
+	}
+
+	runenv.RecordMessage("Genesis cid: %s", gents.Key())
+
+	withMiner := seq < int64(minerCount)
+
+	if withMiner {
+		sminer, err := setupStorageNode(lnode, mpsi.WKey, maddr, mpsi.WKey.Address, mpsi.Dir, nGenesisPreseals)
+		if err != nil {
+			return xerrors.Errorf("failed to set up storage miner: %w", err)
+		}
+
+		spew.Dump(sminer)
+	}
+
+	return nil
+}
+
+func sameAddrs(a, b []net.Addr) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	aset := make(map[string]bool, len(a))
+	for _, addr := range a {
+		aset[addr.String()] = true
+	}
+	for _, addr := range b {
+		if !aset[addr.String()] {
+			return false
+		}
+	}
+	return true
+}
+
+type genFakeVerifier struct{}
+
+var _ ffiwrapper.Verifier = (*genFakeVerifier)(nil)
+
+func (m genFakeVerifier) VerifySeal(svi abi.SealVerifyInfo) (bool, error) {
+	return true, nil
+}
+
+func (m genFakeVerifier) VerifyWinningPoSt(ctx context.Context, info abi.WinningPoStVerifyInfo) (bool, error) {
+	panic("not supported")
+}
+
+func (m genFakeVerifier) VerifyWindowPoSt(ctx context.Context, info abi.WindowPoStVerifyInfo) (bool, error) {
+	panic("not supported")
+}
+
+func (m genFakeVerifier) GenerateWinningPoStSectorChallenge(ctx context.Context, proof abi.RegisteredProof, id abi.ActorID, randomness abi.PoStRandomness, u uint64) ([]uint64, error) {
+	panic("not supported")
+}

lotus-testground/manifest.toml

@@ -0,0 +1,43 @@
+name = "lotus-testground"
+
+[defaults]
+builder = "exec:go"
+runner = "local:exec"
+
+[builders."docker:go"]
+enabled = true
+skip_runtime_image = true
+
+[builders."docker:go".dockerfile_extensions]
+pre_mod_download = """
+RUN apt-get update && apt-get install -y ca-certificates llvm clang mesa-opencl-icd ocl-icd-opencl-dev jq gcc git bzr pkg-config
+ARG LOTUS_VERSION=70e964d9f947ae2d5e4cb61b5afebd4a5bfc8654
+RUN git clone https://github.com/filecoin-project/lotus.git ${PLAN_DIR}/../lotus && cd ${PLAN_DIR}/../lotus && git checkout ${LOTUS_VERSION} && git submodule update --init && make 2k
+RUN cd ${PLAN_DIR}/../lotus/extern/filecoin-ffi \
+    && make \
+    && mkdir /tmp/filecoin-ffi \
+    && cp -R ${PLAN_DIR}/../lotus/extern/filecoin-ffi /tmp
+"""
+
+[builders."exec:go"]
+enabled = true
+
+[runners."local:docker"]
+enabled = true
+
+[runners."local:exec"]
+enabled = true
+
+[runners."cluster:swarm"]
+enabled = true
+
+[runners."cluster:k8s"]
+enabled = true
+
+# seq 0
+[[testcases]]
+name = "lotus-network"
+instances = { min = 2, max = 2, default = 2 }
+
+[testcases.params]
+   miner-count = { type = "int", default = 1 }