From 4a337785b54075e7b33f81990cb629288eff33da Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Wed, 24 Jun 2020 10:41:57 -0400 Subject: [PATCH 1/8] rebase drand setup onto master --- lotus-soup/baseline.go | 14 +++ lotus-soup/compositions/composition.toml | 14 ++- lotus-soup/drand.go | 148 +++++++++++++++++++++++ lotus-soup/go.mod | 3 +- lotus-soup/go.sum | 4 +- lotus-soup/manifest.toml | 1 + lotus-soup/node.go | 40 ++++++ 7 files changed, 220 insertions(+), 4 deletions(-) create mode 100644 lotus-soup/drand.go diff --git a/lotus-soup/baseline.go b/lotus-soup/baseline.go index 502d22cb8..0a4c93488 100644 --- a/lotus-soup/baseline.go +++ b/lotus-soup/baseline.go @@ -46,6 +46,7 @@ var baselineRoles = map[string]func(*TestEnvironment) error{ "bootstrapper": runBootstrapper, "miner": runMiner, "client": runBaselineClient, + "drand": runDrandNode, } func runBaselineClient(t *TestEnvironment) error { @@ -236,3 +237,16 @@ func extractCarData(ctx context.Context, rdata []byte, rpath string) []byte { } return rdata } + +func runDrandNode(t *TestEnvironment) error { + t.RecordMessage("running drand node") + _, err := prepareDrandNode(t) + if err != nil { + return err + } + + // TODO add ability to halt / recover on demand + ctx := context.Background() + t.SyncClient.MustSignalAndWait(ctx, stateDone, t.TestInstanceCount) + return nil +} diff --git a/lotus-soup/compositions/composition.toml b/lotus-soup/compositions/composition.toml index 2068ea2d6..3f5dc7555 100644 --- a/lotus-soup/compositions/composition.toml +++ b/lotus-soup/compositions/composition.toml @@ -5,7 +5,7 @@ [global] plan = "lotus-soup" case = "lotus-baseline" - total_instances = 3 + total_instances = 6 builder = "docker:go" runner = "local:docker" @@ -56,3 +56,15 @@ miners = "1" balance = "2000000000" sectors = "10" + +[[groups]] + id = "drand" + [groups.resources] + memory = "120Mi" + cpu = "10m" + [groups.instances] + count = 3 + percentage = 0.0 + [groups.run] + [groups.run.test_params] + role = "drand" diff --git a/lotus-soup/drand.go b/lotus-soup/drand.go new file mode 100644 index 000000000..e5f14b6c2 --- /dev/null +++ b/lotus-soup/drand.go @@ -0,0 +1,148 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "time" + + "github.com/drand/drand/chain" + hclient "github.com/drand/drand/client/http" + "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/testground/sdk-go/sync" + + "github.com/drand/drand/demo/node" +) + +var ( + PrepareDrandTimeout = time.Minute + drandConfigTopic = sync.NewTopic("drand-config", &dtypes.DrandConfig{}) +) + +// waitForDrandConfig should be called by filecoin instances before constructing the lotus Node +// you can use the returned dtypes.DrandConfig to override the default production config. +func waitForDrandConfig(ctx context.Context, client sync.Client) (*dtypes.DrandConfig, error) { + ch := make(chan *dtypes.DrandConfig, 1) + sub := client.MustSubscribe(ctx, drandConfigTopic, ch) + select { + case cfg := <-ch: + return cfg, nil + case err := <-sub.Done(): + return nil, err + } +} + +// prepareDrandNode starts a drand instance and runs a DKG with the other members of the composition group. +// Once the chain is running, the leader publishes the chain info needed by lotus nodes on +// drandConfigTopic +func prepareDrandNode(t *TestEnvironment) (node.Node, error) { + ctx, cancel := context.WithTimeout(context.Background(), PrepareDrandTimeout) + defer cancel() + + startTime := time.Now() + + seq := t.GroupSeq + isLeader := seq == 1 + nNodes := t.TestGroupInstanceCount + + myAddr := t.NetClient.MustGetDataNetworkIP() + // TODO: add test params for drand + period := "10s" + beaconOffset := 12 + threshold := 2 + + // TODO(maybe): use TLS? + n := node.NewLocalNode(int(seq), period, "~/", false, myAddr.String()) + + // share the node addresses with other nodes + // TODO: if we implement TLS, this is where we'd share public TLS keys + type NodeAddr struct { + PrivateAddr string + PublicAddr string + IsLeader bool + } + addrTopic := sync.NewTopic("drand-addrs", &NodeAddr{}) + var publicAddrs []string + var leaderAddr string + ch := make(chan *NodeAddr) + t.SyncClient.MustPublishSubscribe(ctx, addrTopic, &NodeAddr{ + PrivateAddr: n.PrivateAddr(), + PublicAddr: n.PublicAddr(), + IsLeader: isLeader, + }, ch) + for i := 0; i < nNodes; i++ { + msg, ok := <-ch + if !ok { + return nil, fmt.Errorf("failed to read drand node addr from sync service") + } + publicAddrs = append(publicAddrs, fmt.Sprintf("http://%s", msg.PublicAddr)) + if msg.IsLeader { + leaderAddr = msg.PrivateAddr + } + } + if leaderAddr == "" { + return nil, fmt.Errorf("got %d drand addrs, but no leader", len(publicAddrs)) + } + + t.SyncClient.MustSignalAndWait(ctx, "drand-start", nNodes) + t.RecordMessage("Starting drand sharing ceremony") + if err := n.Start("~/"); err != nil { + return nil, err + } + + alive := false + waitSecs := 10 + for i := 0; i < waitSecs; i++ { + if !n.Ping() { + time.Sleep(time.Second) + continue + } + t.R().RecordPoint("drand_first_ping", time.Now().Sub(startTime).Seconds()) + alive = true + break + } + if !alive { + return nil, fmt.Errorf("drand node %d failed to start after %d seconds", t.GroupSeq, waitSecs) + } + + // run DKG + t.SyncClient.MustSignalAndWait(ctx, "drand-dkg-start", nNodes) + grp := n.RunDKG(nNodes, threshold, period, isLeader, leaderAddr, beaconOffset) + if grp == nil { + return nil, fmt.Errorf("drand dkg failed") + } + t.R().RecordPoint("drand_dkg_complete", time.Now().Sub(startTime).Seconds()) + + // wait for chain to begin + to := time.Until(time.Unix(grp.GenesisTime, 0).Add(3 * time.Second).Add(grp.Period)) + time.Sleep(to) + + + + // verify that we can get a round of randomness from the chain using an http client + info := chain.NewChainInfo(grp) + client, err := hclient.NewWithInfo(publicAddrs[0], info, nil) + if err != nil { + return nil, fmt.Errorf("unable to create drand http client: %w", err) + } + + _, err = client.Get(ctx, 1) + if err != nil { + return nil, fmt.Errorf("unable to get initial drand round: %w", err) + } + + // if we're the leader, publish the config to the sync service + if isLeader { + buf := bytes.Buffer{} + if err := info.ToJSON(&buf); err != nil { + return nil, fmt.Errorf("error marshaling chain info: %w", err) + } + msg := dtypes.DrandConfig{ + Servers: publicAddrs, + ChainInfoJSON: buf.String(), + } + t.SyncClient.MustPublish(ctx, drandConfigTopic, &msg) + } + + return n, nil +} diff --git a/lotus-soup/go.mod b/lotus-soup/go.mod index 61590f237..91704679b 100644 --- a/lotus-soup/go.mod +++ b/lotus-soup/go.mod @@ -4,10 +4,11 @@ go 1.14 require ( github.com/davecgh/go-spew v1.1.1 + github.com/drand/drand v0.9.2-0.20200616080806-a94e9c1636a4 github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef github.com/filecoin-project/go-fil-markets v0.3.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/lotus v0.4.1-0.20200623104442-68d38eff33e4 + github.com/filecoin-project/lotus v0.4.1-0.20200623211458-e8642442267b github.com/filecoin-project/specs-actors v0.6.2-0.20200617175406-de392ca14121 github.com/ipfs/go-cid v0.0.6 github.com/ipfs/go-datastore v0.4.4 diff --git a/lotus-soup/go.sum b/lotus-soup/go.sum index e490587d9..433b3cb64 100644 --- a/lotus-soup/go.sum +++ b/lotus-soup/go.sum @@ -246,8 +246,8 @@ github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIi github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= -github.com/filecoin-project/lotus v0.4.1-0.20200623104442-68d38eff33e4 h1:h0iwqUzi4+E+xs1vLtmM2uxTUoTSNk4P/hFG7Dk9jnw= -github.com/filecoin-project/lotus v0.4.1-0.20200623104442-68d38eff33e4/go.mod h1:uxvEKQiyuXisy/7MB6pFwb4WxxcA3UY3hyJnhL+k7M4= +github.com/filecoin-project/lotus v0.4.1-0.20200623211458-e8642442267b h1:Oq1ABSZVYNFtvWqnj3bHINcw34T0if/0/zlsRq8rGgc= +github.com/filecoin-project/lotus v0.4.1-0.20200623211458-e8642442267b/go.mod h1:uxvEKQiyuXisy/7MB6pFwb4WxxcA3UY3hyJnhL+k7M4= github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= github.com/filecoin-project/sector-storage v0.0.0-20200618073200-d9de9b7cb4b4 h1:lQC8Fbyn31/H4QxYAYwVV3PYZ9vS61EmjktZc5CaiYs= github.com/filecoin-project/sector-storage v0.0.0-20200618073200-d9de9b7cb4b4/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= diff --git a/lotus-soup/manifest.toml b/lotus-soup/manifest.toml index 118a59b23..825269bfd 100644 --- a/lotus-soup/manifest.toml +++ b/lotus-soup/manifest.toml @@ -27,4 +27,5 @@ instances = { min = 1, max = 100, default = 5 } miners = { type = "int", default = 1 } balance = { type = "int", default = 1 } sectors = { type = "int", default = 1 } + real_drand = { type = "bool", default = "false", desc = "set to true to use the real drand network config baked into lotus" } role = { type = "string" } diff --git a/lotus-soup/node.go b/lotus-soup/node.go index c0dcf554e..3a93bea51 100644 --- a/lotus-soup/node.go +++ b/lotus-soup/node.go @@ -116,6 +116,11 @@ func prepareBootstrapper(t *TestEnvironment) (*Node, error) { miners := t.IntParam("miners") nodes := clients + miners + drandOpt, err := getDrandConfig(ctx, t) + if err != nil { + return nil, err + } + // the first duty of the boostrapper is to construct the genesis block // first collect all client and miner balances to assign initial funds balances, err := waitForBalances(t, ctx, nodes) @@ -177,6 +182,7 @@ func prepareBootstrapper(t *TestEnvironment) (*Node, error) { withListenAddress(bootstrapperIP), withBootstrapper(nil), withPubsubConfig(true), + drandOpt, ) if err != nil { return nil, err @@ -229,6 +235,11 @@ func prepareMiner(t *TestEnvironment) (*Node, error) { ctx, cancel := context.WithTimeout(context.Background(), PrepareNodeTimeout) defer cancel() + drandOpt, err := getDrandConfig(ctx, t) + if err != nil { + return nil, err + } + // first create a wallet walletKey, err := wallet.GenerateKey(crypto.SigTypeBLS) if err != nil { @@ -342,6 +353,7 @@ func prepareMiner(t *TestEnvironment) (*Node, error) { withListenAddress(minerIP), withBootstrapper(genesisMsg.Bootstrapper), withPubsubConfig(false), + drandOpt, ) if err != nil { return nil, err @@ -446,6 +458,11 @@ func prepareClient(t *TestEnvironment) (*Node, error) { ctx, cancel := context.WithTimeout(context.Background(), PrepareNodeTimeout) defer cancel() + drandOpt, err := getDrandConfig(ctx, t) + if err != nil { + return nil, err + } + // first create a wallet walletKey, err := wallet.GenerateKey(crypto.SigTypeBLS) if err != nil { @@ -475,6 +492,7 @@ func prepareClient(t *TestEnvironment) (*Node, error) { withListenAddress(clientIP), withBootstrapper(genesisMsg.Bootstrapper), withPubsubConfig(false), + drandOpt, ) if err != nil { return nil, err @@ -636,3 +654,25 @@ func collectClientAddrs(t *TestEnvironment, ctx context.Context, clients int) ([ return addrs, nil } + +func getDrandConfig(ctx context.Context, t *TestEnvironment) (node.Option, error) { + if t.BooleanParam("real_drand") { + noop := func(settings *node.Settings) error { + return nil + } + return noop, nil + } + cfg, err := waitForDrandConfig(ctx, t.SyncClient) + if err != nil { + t.RecordMessage("error getting drand config: %w", err) + return nil, err + } + t.RecordMessage("setting drand config: %v", cfg) + + return node.Options( + node.Override(new(dtypes.DrandConfig), *cfg), + + // FIXME: re-enable drand bootstrap peers once drand gossip relays are running in testground + node.Override(new(dtypes.DrandBootstrap), dtypes.DrandBootstrap{}), + ), nil +} From e225a644f4cfcf03f6d191167361ecea67d4df7c Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Wed, 24 Jun 2020 13:39:06 -0400 Subject: [PATCH 2/8] run drand gossip relays alongside each drand node --- lotus-soup/drand.go | 118 ++++++++++++++++++++++++++++++++------- lotus-soup/manifest.toml | 8 ++- lotus-soup/node.go | 17 ++++-- 3 files changed, 117 insertions(+), 26 deletions(-) diff --git a/lotus-soup/drand.go b/lotus-soup/drand.go index e5f14b6c2..13f24f0c1 100644 --- a/lotus-soup/drand.go +++ b/lotus-soup/drand.go @@ -3,12 +3,20 @@ package main import ( "bytes" "context" + "encoding/hex" + "encoding/json" "fmt" + "net" + "os" "time" "github.com/drand/drand/chain" hclient "github.com/drand/drand/client/http" + "github.com/drand/drand/log" + "github.com/drand/drand/lp2p" "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/libp2p/go-libp2p-core/peer" + ma "github.com/multiformats/go-multiaddr" "github.com/testground/sdk-go/sync" "github.com/drand/drand/demo/node" @@ -16,13 +24,23 @@ import ( var ( PrepareDrandTimeout = time.Minute - drandConfigTopic = sync.NewTopic("drand-config", &dtypes.DrandConfig{}) + drandConfigTopic = sync.NewTopic("drand-config", &DrandRuntimeInfo{}) ) +type DrandRuntimeInfo struct { + Config dtypes.DrandConfig + GossipBootstrap dtypes.DrandBootstrap +} + +type DrandInstance struct { + Node node.Node + GossipRelay *lp2p.GossipRelayNode +} + // waitForDrandConfig should be called by filecoin instances before constructing the lotus Node // you can use the returned dtypes.DrandConfig to override the default production config. -func waitForDrandConfig(ctx context.Context, client sync.Client) (*dtypes.DrandConfig, error) { - ch := make(chan *dtypes.DrandConfig, 1) +func waitForDrandConfig(ctx context.Context, client sync.Client) (*DrandRuntimeInfo, error) { + ch := make(chan *DrandRuntimeInfo, 1) sub := client.MustSubscribe(ctx, drandConfigTopic, ch) select { case cfg := <-ch: @@ -35,7 +53,7 @@ func waitForDrandConfig(ctx context.Context, client sync.Client) (*dtypes.DrandC // prepareDrandNode starts a drand instance and runs a DKG with the other members of the composition group. // Once the chain is running, the leader publishes the chain info needed by lotus nodes on // drandConfigTopic -func prepareDrandNode(t *TestEnvironment) (node.Node, error) { +func prepareDrandNode(t *TestEnvironment) (*DrandInstance, error) { ctx, cancel := context.WithTimeout(context.Background(), PrepareDrandTimeout) defer cancel() @@ -46,13 +64,14 @@ func prepareDrandNode(t *TestEnvironment) (node.Node, error) { nNodes := t.TestGroupInstanceCount myAddr := t.NetClient.MustGetDataNetworkIP() - // TODO: add test params for drand - period := "10s" - beaconOffset := 12 - threshold := 2 + period := t.DurationParam("drand_period") + threshold := t.IntParam("drand_threshold") + runGossipRelay := t.BooleanParam("drand_gossip_relay") + + beaconOffset := 3 // TODO(maybe): use TLS? - n := node.NewLocalNode(int(seq), period, "~/", false, myAddr.String()) + n := node.NewLocalNode(int(seq), period.String(), "~/", false, myAddr.String()) // share the node addresses with other nodes // TODO: if we implement TLS, this is where we'd share public TLS keys @@ -67,8 +86,8 @@ func prepareDrandNode(t *TestEnvironment) (node.Node, error) { ch := make(chan *NodeAddr) t.SyncClient.MustPublishSubscribe(ctx, addrTopic, &NodeAddr{ PrivateAddr: n.PrivateAddr(), - PublicAddr: n.PublicAddr(), - IsLeader: isLeader, + PublicAddr: n.PublicAddr(), + IsLeader: isLeader, }, ch) for i := 0; i < nNodes; i++ { msg, ok := <-ch @@ -107,21 +126,25 @@ func prepareDrandNode(t *TestEnvironment) (node.Node, error) { // run DKG t.SyncClient.MustSignalAndWait(ctx, "drand-dkg-start", nNodes) - grp := n.RunDKG(nNodes, threshold, period, isLeader, leaderAddr, beaconOffset) + if !isLeader { + time.Sleep(time.Second) + } + grp := n.RunDKG(nNodes, threshold, period.String(), isLeader, leaderAddr, beaconOffset) if grp == nil { return nil, fmt.Errorf("drand dkg failed") } t.R().RecordPoint("drand_dkg_complete", time.Now().Sub(startTime).Seconds()) + t.RecordMessage("drand dkg complete, waiting for chain start") // wait for chain to begin to := time.Until(time.Unix(grp.GenesisTime, 0).Add(3 * time.Second).Add(grp.Period)) time.Sleep(to) - - + t.RecordMessage("drand beacon chain started, fetching initial round via http") // verify that we can get a round of randomness from the chain using an http client info := chain.NewChainInfo(grp) - client, err := hclient.NewWithInfo(publicAddrs[0], info, nil) + myPublicAddr := fmt.Sprintf("http://%s", n.PublicAddr()) + client, err := hclient.NewWithInfo(myPublicAddr, info, nil) if err != nil { return nil, fmt.Errorf("unable to create drand http client: %w", err) } @@ -131,18 +154,73 @@ func prepareDrandNode(t *TestEnvironment) (node.Node, error) { return nil, fmt.Errorf("unable to get initial drand round: %w", err) } + // start gossip relay (unless disabled via testplan parameter) + var gossipRelay *lp2p.GossipRelayNode + var relayAddrs []peer.AddrInfo + + if runGossipRelay { + _ = os.Mkdir("~/drand-gossip", os.ModePerm) + listenAddr := fmt.Sprintf("/ip4/%s/tcp/7777", myAddr.String()) + relayCfg := lp2p.GossipRelayConfig{ + ChainHash: hex.EncodeToString(info.Hash()), + Addr: listenAddr, + DataDir: "~/drand-gossip", + IdentityPath: "~/drand-gossip/identity.key", + Insecure: true, + Client: client, + } + t.RecordMessage("starting drand gossip relay") + var err error + gossipRelay, err = lp2p.NewGossipRelayNode(log.DefaultLogger, &relayCfg) + if err != nil { + return nil, fmt.Errorf("failed to construct drand gossip relay: %w", err) + } + + t.RecordMessage("sharing gossip relay addrs") + // share the gossip relay addrs so we can publish them in DrandRuntimeInfo + relayInfo, err := relayAddrInfo(gossipRelay.Multiaddrs(), myAddr) + if err != nil { + return nil, err + } + infoCh := make(chan *peer.AddrInfo, nNodes) + infoTopic := sync.NewTopic("drand-gossip-addrs", &peer.AddrInfo{}) + t.SyncClient.MustPublishSubscribe(ctx, infoTopic, relayInfo, infoCh) + for i := 0; i < nNodes; i++ { + ai := <-infoCh + relayAddrs = append(relayAddrs, *ai) + } + } + // if we're the leader, publish the config to the sync service if isLeader { buf := bytes.Buffer{} if err := info.ToJSON(&buf); err != nil { return nil, fmt.Errorf("error marshaling chain info: %w", err) } - msg := dtypes.DrandConfig{ - Servers: publicAddrs, - ChainInfoJSON: buf.String(), + cfg := DrandRuntimeInfo{ + Config: dtypes.DrandConfig{ + Servers: publicAddrs, + ChainInfoJSON: buf.String(), + }, + GossipBootstrap: relayAddrs, } - t.SyncClient.MustPublish(ctx, drandConfigTopic, &msg) + dump, _ := json.Marshal(cfg) + t.RecordMessage("publishing drand config on sync topic: %s", string(dump)) + t.SyncClient.MustPublish(ctx, drandConfigTopic, &cfg) } - return n, nil + return &DrandInstance{ + Node: n, + GossipRelay: gossipRelay, + }, nil } + +func relayAddrInfo(addrs []ma.Multiaddr, dataIP net.IP) (*peer.AddrInfo, error) { + for _, a := range addrs { + if ip, _ := a.ValueForProtocol(ma.P_IP4); ip != dataIP.String() { + continue + } + return peer.AddrInfoFromP2pAddr(a) + } + return nil, fmt.Errorf("no addr found with data ip %s in addrs: %v", dataIP, addrs) +} \ No newline at end of file diff --git a/lotus-soup/manifest.toml b/lotus-soup/manifest.toml index 825269bfd..4ed06d164 100644 --- a/lotus-soup/manifest.toml +++ b/lotus-soup/manifest.toml @@ -27,5 +27,11 @@ instances = { min = 1, max = 100, default = 5 } miners = { type = "int", default = 1 } balance = { type = "int", default = 1 } sectors = { type = "int", default = 1 } - real_drand = { type = "bool", default = "false", desc = "set to true to use the real drand network config baked into lotus" } role = { type = "string" } + + real_drand = { type = "bool", default = "false", desc = "set to true to use the real drand network config baked into lotus. otherwise, make sure there's a group with role='drand' and at least 2 nodes" } + + # params relevant to drand nodes: + drand_period = { type = "duration", default="10s" } + drand_threshold = { type = "int", default = 2 } + drand_gossip_relay = { type = "bool", default = true } \ No newline at end of file diff --git a/lotus-soup/node.go b/lotus-soup/node.go index 3a93bea51..627b2e058 100644 --- a/lotus-soup/node.go +++ b/lotus-soup/node.go @@ -5,6 +5,7 @@ import ( "context" "crypto/rand" "os" + "strings" //"encoding/json" "fmt" @@ -82,6 +83,15 @@ type TestEnvironment struct { *run.InitContext } +func (t *TestEnvironment) DurationParam(name string) time.Duration { + s := strings.ReplaceAll(t.StringParam(name), "\"", "") + d, err := time.ParseDuration(s) + if err != nil { + panic(fmt.Errorf("invalid duration value for param '%s': %w", name, err)) + } + return d +} + type Node struct { fullApi api.FullNode minerApi api.StorageMiner @@ -668,11 +678,8 @@ func getDrandConfig(ctx context.Context, t *TestEnvironment) (node.Option, error return nil, err } t.RecordMessage("setting drand config: %v", cfg) - return node.Options( - node.Override(new(dtypes.DrandConfig), *cfg), - - // FIXME: re-enable drand bootstrap peers once drand gossip relays are running in testground - node.Override(new(dtypes.DrandBootstrap), dtypes.DrandBootstrap{}), + node.Override(new(dtypes.DrandConfig), cfg.Config), + node.Override(new(dtypes.DrandBootstrap), cfg.GossipBootstrap), ), nil } From 75ff8d06894a18b2bd42c3b83851c8202fe8592b Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 25 Jun 2020 10:55:44 -0400 Subject: [PATCH 3/8] add random_beacon_type param, default to mock randomness --- lotus-soup/manifest.toml | 9 ++++++--- lotus-soup/node.go | 42 +++++++++++++++++++++++++++------------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/lotus-soup/manifest.toml b/lotus-soup/manifest.toml index 4ed06d164..33904ce34 100644 --- a/lotus-soup/manifest.toml +++ b/lotus-soup/manifest.toml @@ -29,9 +29,12 @@ instances = { min = 1, max = 100, default = 5 } sectors = { type = "int", default = 1 } role = { type = "string" } - real_drand = { type = "bool", default = "false", desc = "set to true to use the real drand network config baked into lotus. otherwise, make sure there's a group with role='drand' and at least 2 nodes" } + random_beacon_type = { type = "enum", default = "mock", options = ["mock", "local-drand", "external-drand"] } - # params relevant to drand nodes: + # Params relevant to drand nodes. drand nodes should have role="drand", and must all be + # in the same composition group. There must be at least threshold drand nodes. + # To get lotus nodes to actually use the drand nodes, you must set random_beacon_type="local-drand" + # for the lotus node groups. drand_period = { type = "duration", default="10s" } drand_threshold = { type = "int", default = 2 } - drand_gossip_relay = { type = "bool", default = true } \ No newline at end of file + drand_gossip_relay = { type = "bool", default = true } diff --git a/lotus-soup/node.go b/lotus-soup/node.go index 627b2e058..20c63af07 100644 --- a/lotus-soup/node.go +++ b/lotus-soup/node.go @@ -17,6 +17,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/beacon" genesis_chain "github.com/filecoin-project/lotus/chain/gen/genesis" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" @@ -83,9 +84,13 @@ type TestEnvironment struct { *run.InitContext } +// workaround for default params being wrapped in quote chars +func (t *TestEnvironment) StringParam(name string) string { + return strings.Trim(t.RunEnv.StringParam(name), "\"") +} + func (t *TestEnvironment) DurationParam(name string) time.Duration { - s := strings.ReplaceAll(t.StringParam(name), "\"", "") - d, err := time.ParseDuration(s) + d, err := time.ParseDuration(t.StringParam(name)) if err != nil { panic(fmt.Errorf("invalid duration value for param '%s': %w", name, err)) } @@ -666,20 +671,31 @@ func collectClientAddrs(t *TestEnvironment, ctx context.Context, clients int) ([ } func getDrandConfig(ctx context.Context, t *TestEnvironment) (node.Option, error) { - if t.BooleanParam("real_drand") { + beaconType := t.StringParam("random_beacon_type") + switch beaconType { + case "external-drand": noop := func(settings *node.Settings) error { return nil } return noop, nil + + case "local-drand": + cfg, err := waitForDrandConfig(ctx, t.SyncClient) + if err != nil { + t.RecordMessage("error getting drand config: %w", err) + return nil, err + + } + t.RecordMessage("setting drand config: %v", cfg) + return node.Options( + node.Override(new(dtypes.DrandConfig), cfg.Config), + node.Override(new(dtypes.DrandBootstrap), cfg.GossipBootstrap), + ), nil + + case "mock": + return node.Override(new(beacon.RandomBeacon), modtest.RandomBeacon), nil + + default: + return nil, fmt.Errorf("unknown random_beacon_type: %s", beaconType) } - cfg, err := waitForDrandConfig(ctx, t.SyncClient) - if err != nil { - t.RecordMessage("error getting drand config: %w", err) - return nil, err - } - t.RecordMessage("setting drand config: %v", cfg) - return node.Options( - node.Override(new(dtypes.DrandConfig), cfg.Config), - node.Override(new(dtypes.DrandBootstrap), cfg.GossipBootstrap), - ), nil } From 3f15146c197db7ff87fedd639555221c744de2dd Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 25 Jun 2020 11:07:53 -0400 Subject: [PATCH 4/8] add composition with mock randomness --- .../compositions/composition-local-drand.toml | 75 +++++++++++++++++++ lotus-soup/compositions/composition.toml | 17 ++--- 2 files changed, 80 insertions(+), 12 deletions(-) create mode 100644 lotus-soup/compositions/composition-local-drand.toml diff --git a/lotus-soup/compositions/composition-local-drand.toml b/lotus-soup/compositions/composition-local-drand.toml new file mode 100644 index 000000000..f6ba2e1d6 --- /dev/null +++ b/lotus-soup/compositions/composition-local-drand.toml @@ -0,0 +1,75 @@ +[metadata] + name = "lotus-soup" + author = "" + +[global] + plan = "lotus-soup" + case = "lotus-baseline" + total_instances = 6 + builder = "docker:go" + runner = "local:docker" + +[[groups]] + id = "bootstrapper" + [groups.resources] + memory = "120Mi" + cpu = "10m" + [groups.instances] + count = 1 + percentage = 0.0 + [groups.run] + [groups.run.test_params] + role = "bootstrapper" + clients = "1" + miners = "1" + balance = "2000" + sectors = "10" + random_beacon_type = "local-drand" + +[[groups]] + id = "miners" + [groups.resources] + memory = "120Mi" + cpu = "10m" + [groups.instances] + count = 1 + percentage = 0.0 + [groups.run] + [groups.run.test_params] + role = "miner" + clients = "1" + miners = "1" + balance = "2000" + sectors = "10" + random_beacon_type = "local-drand" + + +[[groups]] + id = "clients" + [groups.resources] + memory = "120Mi" + cpu = "10m" + [groups.instances] + count = 1 + percentage = 0.0 + [groups.run] + [groups.run.test_params] + role = "client" + clients = "1" + miners = "1" + balance = "2000" + sectors = "10" + random_beacon_type = "local-drand" + + +[[groups]] + id = "drand" + [groups.resources] + memory = "120Mi" + cpu = "10m" + [groups.instances] + count = 3 + percentage = 0.0 + [groups.run] + [groups.run.test_params] + role = "drand" diff --git a/lotus-soup/compositions/composition.toml b/lotus-soup/compositions/composition.toml index 3f5dc7555..b4da0f2e4 100644 --- a/lotus-soup/compositions/composition.toml +++ b/lotus-soup/compositions/composition.toml @@ -5,7 +5,7 @@ [global] plan = "lotus-soup" case = "lotus-baseline" - total_instances = 6 + total_instances = 3 builder = "docker:go" runner = "local:docker" @@ -24,6 +24,7 @@ miners = "1" balance = "2000000000" sectors = "10" + random_beacon_type = "mock" [[groups]] id = "miners" @@ -40,6 +41,8 @@ miners = "1" balance = "2000000000" sectors = "10" + random_beacon_type = "mock" + [[groups]] id = "clients" @@ -56,15 +59,5 @@ miners = "1" balance = "2000000000" sectors = "10" + random_beacon_type = "mock" -[[groups]] - id = "drand" - [groups.resources] - memory = "120Mi" - cpu = "10m" - [groups.instances] - count = 3 - percentage = 0.0 - [groups.run] - [groups.run.test_params] - role = "drand" From fda66f5bd9b66d2e4d3297652e67f5cd4cb86204 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 25 Jun 2020 11:56:12 -0400 Subject: [PATCH 5/8] move drand to common_roles & fix mock randomness override --- lotus-soup/baseline.go | 12 ------------ lotus-soup/common_roles.go | 13 +++++++++++++ lotus-soup/node.go | 8 +++++++- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/lotus-soup/baseline.go b/lotus-soup/baseline.go index 0a4c93488..6f5cf9678 100644 --- a/lotus-soup/baseline.go +++ b/lotus-soup/baseline.go @@ -238,15 +238,3 @@ func extractCarData(ctx context.Context, rdata []byte, rpath string) []byte { return rdata } -func runDrandNode(t *TestEnvironment) error { - t.RecordMessage("running drand node") - _, err := prepareDrandNode(t) - if err != nil { - return err - } - - // TODO add ability to halt / recover on demand - ctx := context.Background() - t.SyncClient.MustSignalAndWait(ctx, stateDone, t.TestInstanceCount) - return nil -} diff --git a/lotus-soup/common_roles.go b/lotus-soup/common_roles.go index 140318807..d488921b6 100644 --- a/lotus-soup/common_roles.go +++ b/lotus-soup/common_roles.go @@ -66,3 +66,16 @@ func runMiner(t *TestEnvironment) error { t.SyncClient.MustSignalAndWait(ctx, stateDone, t.TestInstanceCount) return nil } + +func runDrandNode(t *TestEnvironment) error { + t.RecordMessage("running drand node") + _, err := prepareDrandNode(t) + if err != nil { + return err + } + + // TODO add ability to halt / recover on demand + ctx := context.Background() + t.SyncClient.MustSignalAndWait(ctx, stateDone, t.TestInstanceCount) + return nil +} diff --git a/lotus-soup/node.go b/lotus-soup/node.go index 20c63af07..a57fb5f82 100644 --- a/lotus-soup/node.go +++ b/lotus-soup/node.go @@ -693,7 +693,13 @@ func getDrandConfig(ctx context.Context, t *TestEnvironment) (node.Option, error ), nil case "mock": - return node.Override(new(beacon.RandomBeacon), modtest.RandomBeacon), nil + return node.Options( + node.Override(new(beacon.RandomBeacon), modtest.RandomBeacon), + node.Override(new(dtypes.DrandConfig), dtypes.DrandConfig{ + ChainInfoJSON: "{\"Hash\":\"wtf\"}", + }), + node.Override(new(dtypes.DrandBootstrap), dtypes.DrandBootstrap{}), + ), nil default: return nil, fmt.Errorf("unknown random_beacon_type: %s", beaconType) From 3cbf5efddddc35e49f43806107e2a095413f64e2 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 25 Jun 2020 12:38:40 -0400 Subject: [PATCH 6/8] use temp dir for drand state --- lotus-soup/common_roles.go | 3 ++- lotus-soup/drand.go | 25 +++++++++++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/lotus-soup/common_roles.go b/lotus-soup/common_roles.go index d488921b6..20e0aa67d 100644 --- a/lotus-soup/common_roles.go +++ b/lotus-soup/common_roles.go @@ -69,10 +69,11 @@ func runMiner(t *TestEnvironment) error { func runDrandNode(t *TestEnvironment) error { t.RecordMessage("running drand node") - _, err := prepareDrandNode(t) + dr, err := prepareDrandNode(t) if err != nil { return err } + defer dr.Cleanup() // TODO add ability to halt / recover on demand ctx := context.Background() diff --git a/lotus-soup/drand.go b/lotus-soup/drand.go index 13f24f0c1..0c455fb9d 100644 --- a/lotus-soup/drand.go +++ b/lotus-soup/drand.go @@ -6,8 +6,10 @@ import ( "encoding/hex" "encoding/json" "fmt" + "io/ioutil" "net" "os" + "path" "time" "github.com/drand/drand/chain" @@ -35,6 +37,12 @@ type DrandRuntimeInfo struct { type DrandInstance struct { Node node.Node GossipRelay *lp2p.GossipRelayNode + + stateDir string +} + +func (d *DrandInstance) Cleanup() error { + return os.RemoveAll(d.stateDir) } // waitForDrandConfig should be called by filecoin instances before constructing the lotus Node @@ -70,8 +78,13 @@ func prepareDrandNode(t *TestEnvironment) (*DrandInstance, error) { beaconOffset := 3 + stateDir, err := ioutil.TempDir("", fmt.Sprintf("drand-%d", t.GroupSeq)) + if err != nil { + return nil, err + } + // TODO(maybe): use TLS? - n := node.NewLocalNode(int(seq), period.String(), "~/", false, myAddr.String()) + n := node.NewLocalNode(int(seq), period.String(), stateDir, false, myAddr.String()) // share the node addresses with other nodes // TODO: if we implement TLS, this is where we'd share public TLS keys @@ -105,7 +118,7 @@ func prepareDrandNode(t *TestEnvironment) (*DrandInstance, error) { t.SyncClient.MustSignalAndWait(ctx, "drand-start", nNodes) t.RecordMessage("Starting drand sharing ceremony") - if err := n.Start("~/"); err != nil { + if err := n.Start(stateDir); err != nil { return nil, err } @@ -159,18 +172,17 @@ func prepareDrandNode(t *TestEnvironment) (*DrandInstance, error) { var relayAddrs []peer.AddrInfo if runGossipRelay { - _ = os.Mkdir("~/drand-gossip", os.ModePerm) + gossipDir := path.Join(stateDir, "gossip-relay") listenAddr := fmt.Sprintf("/ip4/%s/tcp/7777", myAddr.String()) relayCfg := lp2p.GossipRelayConfig{ ChainHash: hex.EncodeToString(info.Hash()), Addr: listenAddr, - DataDir: "~/drand-gossip", - IdentityPath: "~/drand-gossip/identity.key", + DataDir: gossipDir, + IdentityPath: path.Join(gossipDir, "identity.key"), Insecure: true, Client: client, } t.RecordMessage("starting drand gossip relay") - var err error gossipRelay, err = lp2p.NewGossipRelayNode(log.DefaultLogger, &relayCfg) if err != nil { return nil, fmt.Errorf("failed to construct drand gossip relay: %w", err) @@ -212,6 +224,7 @@ func prepareDrandNode(t *TestEnvironment) (*DrandInstance, error) { return &DrandInstance{ Node: n, GossipRelay: gossipRelay, + stateDir: stateDir, }, nil } From 8b0f6c355cc0e2bd42d7408484f72b282b7c5764 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 25 Jun 2020 13:00:55 -0400 Subject: [PATCH 7/8] read from sub.Done() --- lotus-soup/drand.go | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/lotus-soup/drand.go b/lotus-soup/drand.go index 0c455fb9d..507b05060 100644 --- a/lotus-soup/drand.go +++ b/lotus-soup/drand.go @@ -97,19 +97,20 @@ func prepareDrandNode(t *TestEnvironment) (*DrandInstance, error) { var publicAddrs []string var leaderAddr string ch := make(chan *NodeAddr) - t.SyncClient.MustPublishSubscribe(ctx, addrTopic, &NodeAddr{ + _, sub := t.SyncClient.MustPublishSubscribe(ctx, addrTopic, &NodeAddr{ PrivateAddr: n.PrivateAddr(), PublicAddr: n.PublicAddr(), IsLeader: isLeader, }, ch) for i := 0; i < nNodes; i++ { - msg, ok := <-ch - if !ok { - return nil, fmt.Errorf("failed to read drand node addr from sync service") - } - publicAddrs = append(publicAddrs, fmt.Sprintf("http://%s", msg.PublicAddr)) - if msg.IsLeader { - leaderAddr = msg.PrivateAddr + select { + case msg := <-ch: + publicAddrs = append(publicAddrs, fmt.Sprintf("http://%s", msg.PublicAddr)) + if msg.IsLeader { + leaderAddr = msg.PrivateAddr + } + case err := <-sub.Done(): + return nil, fmt.Errorf("unable to read drand addrs from sync service: %w", err) } } if leaderAddr == "" { @@ -196,10 +197,15 @@ func prepareDrandNode(t *TestEnvironment) (*DrandInstance, error) { } infoCh := make(chan *peer.AddrInfo, nNodes) infoTopic := sync.NewTopic("drand-gossip-addrs", &peer.AddrInfo{}) - t.SyncClient.MustPublishSubscribe(ctx, infoTopic, relayInfo, infoCh) + + _, sub := t.SyncClient.MustPublishSubscribe(ctx, infoTopic, relayInfo, infoCh) for i := 0; i < nNodes; i++ { - ai := <-infoCh - relayAddrs = append(relayAddrs, *ai) + select { + case ai := <-infoCh: + relayAddrs = append(relayAddrs, *ai) + case err := <-sub.Done(): + return nil, fmt.Errorf("unable to get drand relay addr from sync service: %w", err) + } } } From 19f0aa75ebaeeae06abd7488c285d34c6ac4e578 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 25 Jun 2020 13:01:46 -0400 Subject: [PATCH 8/8] rm resources block from drand composition --- lotus-soup/compositions/composition-local-drand.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/lotus-soup/compositions/composition-local-drand.toml b/lotus-soup/compositions/composition-local-drand.toml index f6ba2e1d6..9994008e7 100644 --- a/lotus-soup/compositions/composition-local-drand.toml +++ b/lotus-soup/compositions/composition-local-drand.toml @@ -64,9 +64,6 @@ [[groups]] id = "drand" - [groups.resources] - memory = "120Mi" - cpu = "10m" [groups.instances] count = 3 percentage = 0.0