From 4a337785b54075e7b33f81990cb629288eff33da Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Wed, 24 Jun 2020 10:41:57 -0400 Subject: [PATCH] rebase drand setup onto master --- lotus-soup/baseline.go | 14 +++ lotus-soup/compositions/composition.toml | 14 ++- lotus-soup/drand.go | 148 +++++++++++++++++++++++ lotus-soup/go.mod | 3 +- lotus-soup/go.sum | 4 +- lotus-soup/manifest.toml | 1 + lotus-soup/node.go | 40 ++++++ 7 files changed, 220 insertions(+), 4 deletions(-) create mode 100644 lotus-soup/drand.go diff --git a/lotus-soup/baseline.go b/lotus-soup/baseline.go index 502d22cb8..0a4c93488 100644 --- a/lotus-soup/baseline.go +++ b/lotus-soup/baseline.go @@ -46,6 +46,7 @@ var baselineRoles = map[string]func(*TestEnvironment) error{ "bootstrapper": runBootstrapper, "miner": runMiner, "client": runBaselineClient, + "drand": runDrandNode, } func runBaselineClient(t *TestEnvironment) error { @@ -236,3 +237,16 @@ func extractCarData(ctx context.Context, rdata []byte, rpath string) []byte { } return rdata } + +func runDrandNode(t *TestEnvironment) error { + t.RecordMessage("running drand node") + _, err := prepareDrandNode(t) + if err != nil { + return err + } + + // TODO add ability to halt / recover on demand + ctx := context.Background() + t.SyncClient.MustSignalAndWait(ctx, stateDone, t.TestInstanceCount) + return nil +} diff --git a/lotus-soup/compositions/composition.toml b/lotus-soup/compositions/composition.toml index 2068ea2d6..3f5dc7555 100644 --- a/lotus-soup/compositions/composition.toml +++ b/lotus-soup/compositions/composition.toml @@ -5,7 +5,7 @@ [global] plan = "lotus-soup" case = "lotus-baseline" - total_instances = 3 + total_instances = 6 builder = "docker:go" runner = "local:docker" @@ -56,3 +56,15 @@ miners = "1" balance = "2000000000" sectors = "10" + +[[groups]] + id = "drand" + [groups.resources] + memory = "120Mi" + cpu = "10m" + [groups.instances] + count = 3 + percentage = 0.0 + [groups.run] + [groups.run.test_params] + role = "drand" diff --git a/lotus-soup/drand.go b/lotus-soup/drand.go new file mode 100644 index 000000000..e5f14b6c2 --- /dev/null +++ b/lotus-soup/drand.go @@ -0,0 +1,148 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "time" + + "github.com/drand/drand/chain" + hclient "github.com/drand/drand/client/http" + "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/testground/sdk-go/sync" + + "github.com/drand/drand/demo/node" +) + +var ( + PrepareDrandTimeout = time.Minute + drandConfigTopic = sync.NewTopic("drand-config", &dtypes.DrandConfig{}) +) + +// waitForDrandConfig should be called by filecoin instances before constructing the lotus Node +// you can use the returned dtypes.DrandConfig to override the default production config. +func waitForDrandConfig(ctx context.Context, client sync.Client) (*dtypes.DrandConfig, error) { + ch := make(chan *dtypes.DrandConfig, 1) + sub := client.MustSubscribe(ctx, drandConfigTopic, ch) + select { + case cfg := <-ch: + return cfg, nil + case err := <-sub.Done(): + return nil, err + } +} + +// prepareDrandNode starts a drand instance and runs a DKG with the other members of the composition group. +// Once the chain is running, the leader publishes the chain info needed by lotus nodes on +// drandConfigTopic +func prepareDrandNode(t *TestEnvironment) (node.Node, error) { + ctx, cancel := context.WithTimeout(context.Background(), PrepareDrandTimeout) + defer cancel() + + startTime := time.Now() + + seq := t.GroupSeq + isLeader := seq == 1 + nNodes := t.TestGroupInstanceCount + + myAddr := t.NetClient.MustGetDataNetworkIP() + // TODO: add test params for drand + period := "10s" + beaconOffset := 12 + threshold := 2 + + // TODO(maybe): use TLS? + n := node.NewLocalNode(int(seq), period, "~/", false, myAddr.String()) + + // share the node addresses with other nodes + // TODO: if we implement TLS, this is where we'd share public TLS keys + type NodeAddr struct { + PrivateAddr string + PublicAddr string + IsLeader bool + } + addrTopic := sync.NewTopic("drand-addrs", &NodeAddr{}) + var publicAddrs []string + var leaderAddr string + ch := make(chan *NodeAddr) + t.SyncClient.MustPublishSubscribe(ctx, addrTopic, &NodeAddr{ + PrivateAddr: n.PrivateAddr(), + PublicAddr: n.PublicAddr(), + IsLeader: isLeader, + }, ch) + for i := 0; i < nNodes; i++ { + msg, ok := <-ch + if !ok { + return nil, fmt.Errorf("failed to read drand node addr from sync service") + } + publicAddrs = append(publicAddrs, fmt.Sprintf("http://%s", msg.PublicAddr)) + if msg.IsLeader { + leaderAddr = msg.PrivateAddr + } + } + if leaderAddr == "" { + return nil, fmt.Errorf("got %d drand addrs, but no leader", len(publicAddrs)) + } + + t.SyncClient.MustSignalAndWait(ctx, "drand-start", nNodes) + t.RecordMessage("Starting drand sharing ceremony") + if err := n.Start("~/"); err != nil { + return nil, err + } + + alive := false + waitSecs := 10 + for i := 0; i < waitSecs; i++ { + if !n.Ping() { + time.Sleep(time.Second) + continue + } + t.R().RecordPoint("drand_first_ping", time.Now().Sub(startTime).Seconds()) + alive = true + break + } + if !alive { + return nil, fmt.Errorf("drand node %d failed to start after %d seconds", t.GroupSeq, waitSecs) + } + + // run DKG + t.SyncClient.MustSignalAndWait(ctx, "drand-dkg-start", nNodes) + grp := n.RunDKG(nNodes, threshold, period, isLeader, leaderAddr, beaconOffset) + if grp == nil { + return nil, fmt.Errorf("drand dkg failed") + } + t.R().RecordPoint("drand_dkg_complete", time.Now().Sub(startTime).Seconds()) + + // wait for chain to begin + to := time.Until(time.Unix(grp.GenesisTime, 0).Add(3 * time.Second).Add(grp.Period)) + time.Sleep(to) + + + + // verify that we can get a round of randomness from the chain using an http client + info := chain.NewChainInfo(grp) + client, err := hclient.NewWithInfo(publicAddrs[0], info, nil) + if err != nil { + return nil, fmt.Errorf("unable to create drand http client: %w", err) + } + + _, err = client.Get(ctx, 1) + if err != nil { + return nil, fmt.Errorf("unable to get initial drand round: %w", err) + } + + // if we're the leader, publish the config to the sync service + if isLeader { + buf := bytes.Buffer{} + if err := info.ToJSON(&buf); err != nil { + return nil, fmt.Errorf("error marshaling chain info: %w", err) + } + msg := dtypes.DrandConfig{ + Servers: publicAddrs, + ChainInfoJSON: buf.String(), + } + t.SyncClient.MustPublish(ctx, drandConfigTopic, &msg) + } + + return n, nil +} diff --git a/lotus-soup/go.mod b/lotus-soup/go.mod index 61590f237..91704679b 100644 --- a/lotus-soup/go.mod +++ b/lotus-soup/go.mod @@ -4,10 +4,11 @@ go 1.14 require ( github.com/davecgh/go-spew v1.1.1 + github.com/drand/drand v0.9.2-0.20200616080806-a94e9c1636a4 github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef github.com/filecoin-project/go-fil-markets v0.3.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/lotus v0.4.1-0.20200623104442-68d38eff33e4 + github.com/filecoin-project/lotus v0.4.1-0.20200623211458-e8642442267b github.com/filecoin-project/specs-actors v0.6.2-0.20200617175406-de392ca14121 github.com/ipfs/go-cid v0.0.6 github.com/ipfs/go-datastore v0.4.4 diff --git a/lotus-soup/go.sum b/lotus-soup/go.sum index e490587d9..433b3cb64 100644 --- a/lotus-soup/go.sum +++ b/lotus-soup/go.sum @@ -246,8 +246,8 @@ github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIi github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= -github.com/filecoin-project/lotus v0.4.1-0.20200623104442-68d38eff33e4 h1:h0iwqUzi4+E+xs1vLtmM2uxTUoTSNk4P/hFG7Dk9jnw= -github.com/filecoin-project/lotus v0.4.1-0.20200623104442-68d38eff33e4/go.mod h1:uxvEKQiyuXisy/7MB6pFwb4WxxcA3UY3hyJnhL+k7M4= +github.com/filecoin-project/lotus v0.4.1-0.20200623211458-e8642442267b h1:Oq1ABSZVYNFtvWqnj3bHINcw34T0if/0/zlsRq8rGgc= +github.com/filecoin-project/lotus v0.4.1-0.20200623211458-e8642442267b/go.mod h1:uxvEKQiyuXisy/7MB6pFwb4WxxcA3UY3hyJnhL+k7M4= github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= github.com/filecoin-project/sector-storage v0.0.0-20200618073200-d9de9b7cb4b4 h1:lQC8Fbyn31/H4QxYAYwVV3PYZ9vS61EmjktZc5CaiYs= github.com/filecoin-project/sector-storage v0.0.0-20200618073200-d9de9b7cb4b4/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= diff --git a/lotus-soup/manifest.toml b/lotus-soup/manifest.toml index 118a59b23..825269bfd 100644 --- a/lotus-soup/manifest.toml +++ b/lotus-soup/manifest.toml @@ -27,4 +27,5 @@ instances = { min = 1, max = 100, default = 5 } miners = { type = "int", default = 1 } balance = { type = "int", default = 1 } sectors = { type = "int", default = 1 } + real_drand = { type = "bool", default = "false", desc = "set to true to use the real drand network config baked into lotus" } role = { type = "string" } diff --git a/lotus-soup/node.go b/lotus-soup/node.go index c0dcf554e..3a93bea51 100644 --- a/lotus-soup/node.go +++ b/lotus-soup/node.go @@ -116,6 +116,11 @@ func prepareBootstrapper(t *TestEnvironment) (*Node, error) { miners := t.IntParam("miners") nodes := clients + miners + drandOpt, err := getDrandConfig(ctx, t) + if err != nil { + return nil, err + } + // the first duty of the boostrapper is to construct the genesis block // first collect all client and miner balances to assign initial funds balances, err := waitForBalances(t, ctx, nodes) @@ -177,6 +182,7 @@ func prepareBootstrapper(t *TestEnvironment) (*Node, error) { withListenAddress(bootstrapperIP), withBootstrapper(nil), withPubsubConfig(true), + drandOpt, ) if err != nil { return nil, err @@ -229,6 +235,11 @@ func prepareMiner(t *TestEnvironment) (*Node, error) { ctx, cancel := context.WithTimeout(context.Background(), PrepareNodeTimeout) defer cancel() + drandOpt, err := getDrandConfig(ctx, t) + if err != nil { + return nil, err + } + // first create a wallet walletKey, err := wallet.GenerateKey(crypto.SigTypeBLS) if err != nil { @@ -342,6 +353,7 @@ func prepareMiner(t *TestEnvironment) (*Node, error) { withListenAddress(minerIP), withBootstrapper(genesisMsg.Bootstrapper), withPubsubConfig(false), + drandOpt, ) if err != nil { return nil, err @@ -446,6 +458,11 @@ func prepareClient(t *TestEnvironment) (*Node, error) { ctx, cancel := context.WithTimeout(context.Background(), PrepareNodeTimeout) defer cancel() + drandOpt, err := getDrandConfig(ctx, t) + if err != nil { + return nil, err + } + // first create a wallet walletKey, err := wallet.GenerateKey(crypto.SigTypeBLS) if err != nil { @@ -475,6 +492,7 @@ func prepareClient(t *TestEnvironment) (*Node, error) { withListenAddress(clientIP), withBootstrapper(genesisMsg.Bootstrapper), withPubsubConfig(false), + drandOpt, ) if err != nil { return nil, err @@ -636,3 +654,25 @@ func collectClientAddrs(t *TestEnvironment, ctx context.Context, clients int) ([ return addrs, nil } + +func getDrandConfig(ctx context.Context, t *TestEnvironment) (node.Option, error) { + if t.BooleanParam("real_drand") { + noop := func(settings *node.Settings) error { + return nil + } + return noop, nil + } + cfg, err := waitForDrandConfig(ctx, t.SyncClient) + if err != nil { + t.RecordMessage("error getting drand config: %w", err) + return nil, err + } + t.RecordMessage("setting drand config: %v", cfg) + + return node.Options( + node.Override(new(dtypes.DrandConfig), *cfg), + + // FIXME: re-enable drand bootstrap peers once drand gossip relays are running in testground + node.Override(new(dtypes.DrandBootstrap), dtypes.DrandBootstrap{}), + ), nil +}