diff --git a/lotus-soup/compositions/composition-k8s-10-3.toml b/lotus-soup/compositions/composition-k8s-10-3.toml index bc4a44220..061a1c984 100644 --- a/lotus-soup/compositions/composition-k8s-10-3.toml +++ b/lotus-soup/compositions/composition-k8s-10-3.toml @@ -4,7 +4,7 @@ [global] plan = "lotus-soup" - case = "lotus-baseline" + case = "deals-e2e" total_instances = 14 builder = "docker:go" runner = "cluster:k8s" diff --git a/lotus-soup/compositions/composition-k8s-3-1.toml b/lotus-soup/compositions/composition-k8s-3-1.toml index a4fb82ce2..3470c3093 100644 --- a/lotus-soup/compositions/composition-k8s-3-1.toml +++ b/lotus-soup/compositions/composition-k8s-3-1.toml @@ -4,7 +4,7 @@ [global] plan = "lotus-soup" - case = "lotus-baseline" + case = "deals-e2e" total_instances = 5 builder = "docker:go" runner = "cluster:k8s" diff --git a/lotus-soup/compositions/composition-k8s-3-2.toml b/lotus-soup/compositions/composition-k8s-3-2.toml index b0c7f3c74..8740d4f0c 100644 --- a/lotus-soup/compositions/composition-k8s-3-2.toml +++ b/lotus-soup/compositions/composition-k8s-3-2.toml @@ -4,7 +4,7 @@ [global] plan = "lotus-soup" - case = "lotus-baseline" + case = "deals-e2e" total_instances = 6 builder = "docker:go" runner = "cluster:k8s" diff --git a/lotus-soup/compositions/composition-k8s.toml b/lotus-soup/compositions/composition-k8s.toml index b4e2d563e..548293588 100644 --- a/lotus-soup/compositions/composition-k8s.toml +++ b/lotus-soup/compositions/composition-k8s.toml @@ -4,7 +4,7 @@ [global] plan = "lotus-soup" - case = "lotus-baseline" + case = "deals-e2e" total_instances = 3 builder = "docker:go" runner = "cluster:k8s" diff --git a/lotus-soup/compositions/composition-local-drand.toml b/lotus-soup/compositions/composition-local-drand.toml index 060409936..6f517010e 100644 --- a/lotus-soup/compositions/composition-local-drand.toml +++ b/lotus-soup/compositions/composition-local-drand.toml @@ -4,7 +4,7 @@ [global] plan = "lotus-soup" - case = "lotus-baseline" + case = "deals-e2e" total_instances = 6 builder = "docker:go" runner = "local:docker" diff --git a/lotus-soup/compositions/composition-natural.toml b/lotus-soup/compositions/composition-natural.toml index e82c6a397..36e8908bd 100644 --- a/lotus-soup/compositions/composition-natural.toml +++ b/lotus-soup/compositions/composition-natural.toml @@ -4,7 +4,7 @@ [global] plan = "lotus-soup" - case = "lotus-baseline" + case = "deals-e2e" total_instances = 6 builder = "docker:go" runner = "local:docker" diff --git a/lotus-soup/compositions/composition-tracer.toml b/lotus-soup/compositions/composition-tracer.toml index d632d287e..af9207fb0 100644 --- a/lotus-soup/compositions/composition-tracer.toml +++ b/lotus-soup/compositions/composition-tracer.toml @@ -4,7 +4,7 @@ [global] plan = "lotus-soup" - case = "lotus-baseline" + case = "deals-e2e" total_instances = 7 builder = "docker:go" runner = "local:docker" diff --git a/lotus-soup/compositions/composition.toml b/lotus-soup/compositions/composition.toml index 9a561a355..5a2b8b87f 100644 --- a/lotus-soup/compositions/composition.toml +++ b/lotus-soup/compositions/composition.toml @@ -4,8 +4,8 @@ [global] plan = "lotus-soup" - case = "lotus-baseline" - total_instances = 3 + case = "deals-e2e" + total_instances = 6 builder = "docker:go" runner = "local:docker" @@ -35,7 +35,7 @@ [[groups]] id = "miners" [groups.instances] - count = 1 + count = 2 percentage = 0.0 [groups.run] [groups.run.test_params] @@ -44,7 +44,7 @@ [[groups]] id = "clients" [groups.instances] - count = 1 + count = 3 percentage = 0.0 [groups.run] [groups.run.test_params] diff --git a/lotus-soup/go.sum b/lotus-soup/go.sum index 9e1791cf9..c69f66650 100644 --- a/lotus-soup/go.sum +++ b/lotus-soup/go.sum @@ -264,6 +264,7 @@ github.com/filecoin-project/sector-storage v0.0.0-20200623224636-de544b531601/go github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= +github.com/filecoin-project/specs-actors v0.6.1/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-actors v0.6.2-0.20200617175406-de392ca14121 h1:oRA+b4iN4H86xXDXbU3TOyvmBZp7//c5VqTc0oJ6nLg= github.com/filecoin-project/specs-actors v0.6.2-0.20200617175406-de392ca14121/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-storage v0.1.0/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= diff --git a/lotus-soup/testkit/role_drand.go b/lotus-soup/testkit/role_drand.go index e7cc1848b..ec285b9f9 100644 --- a/lotus-soup/testkit/role_drand.go +++ b/lotus-soup/testkit/role_drand.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/hex" + "encoding/json" "fmt" "io/ioutil" "net" @@ -12,164 +13,37 @@ import ( "time" "github.com/drand/drand/chain" - "github.com/drand/drand/client" hclient "github.com/drand/drand/client/http" - "github.com/drand/drand/core" - "github.com/drand/drand/key" + "github.com/drand/drand/demo/node" "github.com/drand/drand/log" "github.com/drand/drand/lp2p" - dnet "github.com/drand/drand/net" - "github.com/drand/drand/protobuf/drand" - dtest "github.com/drand/drand/test" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/libp2p/go-libp2p-core/peer" ma "github.com/multiformats/go-multiaddr" "github.com/testground/sdk-go/sync" - - "github.com/filecoin-project/oni/lotus-soup/statemachine" ) -var ( - PrepareDrandTimeout = time.Minute - secretDKG = "dkgsecret" -) +var PrepareDrandTimeout = time.Minute type DrandInstance struct { - daemon *core.Drand - httpClient client.Client - ctrlClient *dnet.ControlClient - gossipRelay *lp2p.GossipRelayNode + t *TestEnvironment + + Node node.Node + GossipRelay *lp2p.GossipRelayNode - t *TestEnvironment stateDir string - priv *key.Pair - pubAddr string - privAddr string - ctrlAddr string } -func (dr *DrandInstance) Start() error { - opts := []core.ConfigOption{ - core.WithLogLevel(getLogLevel(dr.t)), - core.WithConfigFolder(dr.stateDir), - core.WithPublicListenAddress(dr.pubAddr), - core.WithPrivateListenAddress(dr.privAddr), - core.WithControlPort(dr.ctrlAddr), - core.WithInsecure(), - } - conf := core.NewConfig(opts...) - fs := key.NewFileStore(conf.ConfigFolder()) - fs.SaveKeyPair(dr.priv) - key.Save(path.Join(dr.stateDir, "public.toml"), dr.priv.Public, false) - if dr.daemon == nil { - drand, err := core.NewDrand(fs, conf) - if err != nil { - return err - } - dr.daemon = drand - } else { - drand, err := core.LoadDrand(fs, conf) - if err != nil { - return err - } - drand.StartBeacon(true) - dr.daemon = drand - } - return nil +func (d *DrandInstance) Cleanup() error { + return os.RemoveAll(d.stateDir) } -func (dr *DrandInstance) Ping() bool { - cl := dr.ctrl() - if err := cl.Ping(); err != nil { - return false - } - return true -} +func (d *DrandInstance) RunDefault() error { + d.t.RecordMessage("running drand node") + defer d.Cleanup() -func (dr *DrandInstance) Close() error { - dr.gossipRelay.Shutdown() - dr.daemon.Stop(context.Background()) - return os.RemoveAll(dr.stateDir) -} - -func (dr *DrandInstance) ctrl() *dnet.ControlClient { - if dr.ctrlClient != nil { - return dr.ctrlClient - } - cl, err := dnet.NewControlClient(dr.ctrlAddr) - if err != nil { - dr.t.RecordMessage("drand can't instantiate control client: %w", err) - return nil - } - dr.ctrlClient = cl - return cl -} - -func (dr *DrandInstance) RunDKG(nodes, thr int, period time.Duration, timeout string, leader bool, leaderAddr string, beaconOffset int) *key.Group { - cl := dr.ctrl() - t, _ := time.ParseDuration(timeout) - var grp *drand.GroupPacket - var err error - if leader { - grp, err = cl.InitDKGLeader(nodes, thr, period, t, nil, secretDKG, beaconOffset) - } else { - leader := dnet.CreatePeer(leaderAddr, false) - grp, err = cl.InitDKG(leader, nil, secretDKG) - } - if err != nil { - dr.t.RecordMessage("drand dkg run failed: %w", err) - return nil - } - kg, _ := key.GroupFromProto(grp) - return kg -} - -func (dr *DrandInstance) Halt() { - dr.t.RecordMessage("drand node #%d halting", dr.t.GroupSeq) - dr.daemon.StopBeacon() -} - -func (dr *DrandInstance) Resume() { - dr.t.RecordMessage("drand node #%d resuming", dr.t.GroupSeq) - dr.daemon.StartBeacon(true) - // block until we can fetch the round corresponding to the current time - startTime := time.Now() - round := dr.httpClient.RoundAt(startTime) - timeout := 30 * time.Second - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - done := make(chan struct{}, 1) - go func() { - for { - res, err := dr.httpClient.Get(ctx, round) - if err == nil { - dr.t.RecordMessage("drand chain caught up to round %d", res.Round()) - done <- struct{}{} - return - } - } - }() - - select { - case <-ctx.Done(): - dr.t.RecordMessage("drand chain failed to catch up after %s", timeout.String()) - case <-done: - dr.t.RecordMessage("drand chain resumed after %s catchup time", time.Since(startTime)) - } -} - - -func (dr *DrandInstance) RunDefault() error { - dr.t.RecordMessage("running drand node") - defer dr.Close() - - if dr.t.IsParamSet("suspend_events") { - suspender := statemachine.NewSuspender(dr, dr.t.RecordMessage) - suspender.RunEvents(dr.t.StringParam("suspend_events")) - } - - dr.t.WaitUntilAllDone() + // TODO add ability to halt / recover on demand + d.t.WaitUntilAllDone() return nil } @@ -196,20 +70,13 @@ func PrepareDrandInstance(t *TestEnvironment) (*DrandInstance, error) { ctx, cancel := context.WithTimeout(context.Background(), PrepareDrandTimeout) defer cancel() - stateDir, err := ioutil.TempDir("/tmp", fmt.Sprintf("drand-%d", t.GroupSeq)) - + stateDir, err := ioutil.TempDir("", fmt.Sprintf("drand-%d", t.GroupSeq)) if err != nil { return nil, err } - dr := DrandInstance{ - t: t, - stateDir: stateDir, - pubAddr: dtest.FreeBind(myAddr.String()), - privAddr: dtest.FreeBind(myAddr.String()), - ctrlAddr: dtest.FreeBind("localhost"), - } - dr.priv = key.NewKeyPair(dr.privAddr) + // TODO(maybe): use TLS? + n := node.NewLocalNode(int(seq), period.String(), stateDir, false, myAddr.String()) // share the node addresses with other nodes // TODO: if we implement TLS, this is where we'd share public TLS keys @@ -224,8 +91,8 @@ func PrepareDrandInstance(t *TestEnvironment) (*DrandInstance, error) { var leaderAddr string ch := make(chan *NodeAddr) _, sub := t.SyncClient.MustPublishSubscribe(ctx, addrTopic, &NodeAddr{ - PrivateAddr: dr.privAddr, - PublicAddr: dr.pubAddr, + PrivateAddr: n.PrivateAddr(), + PublicAddr: n.PublicAddr(), IsLeader: isLeader, }, ch) @@ -247,14 +114,14 @@ func PrepareDrandInstance(t *TestEnvironment) (*DrandInstance, error) { t.SyncClient.MustSignalAndWait(ctx, "drand-start", nNodes) t.RecordMessage("Starting drand sharing ceremony") - if err := dr.Start(); err != nil { + if err := n.Start(stateDir); err != nil { return nil, err } alive := false waitSecs := 10 for i := 0; i < waitSecs; i++ { - if !dr.Ping() { + if !n.Ping() { time.Sleep(time.Second) continue } @@ -271,7 +138,7 @@ func PrepareDrandInstance(t *TestEnvironment) (*DrandInstance, error) { if !isLeader { time.Sleep(time.Second) } - grp := dr.RunDKG(nNodes, threshold, period, "10s", isLeader, leaderAddr, beaconOffset) + grp := n.RunDKG(nNodes, threshold, period.String(), isLeader, leaderAddr, beaconOffset) if grp == nil { return nil, fmt.Errorf("drand dkg failed") } @@ -285,18 +152,19 @@ func PrepareDrandInstance(t *TestEnvironment) (*DrandInstance, error) { t.RecordMessage("drand beacon chain started, fetching initial round via http") // verify that we can get a round of randomness from the chain using an http client info := chain.NewChainInfo(grp) - myPublicAddr := fmt.Sprintf("http://%s", dr.pubAddr) - dr.httpClient, err = hclient.NewWithInfo(myPublicAddr, info, nil) + myPublicAddr := fmt.Sprintf("http://%s", n.PublicAddr()) + client, err := hclient.NewWithInfo(myPublicAddr, info, nil) if err != nil { return nil, fmt.Errorf("unable to create drand http client: %w", err) } - _, err = dr.httpClient.Get(ctx, 1) + _, err = client.Get(ctx, 1) if err != nil { return nil, fmt.Errorf("unable to get initial drand round: %w", err) } // start gossip relay (unless disabled via testplan parameter) + var gossipRelay *lp2p.GossipRelayNode var relayAddrs []peer.AddrInfo if runGossipRelay { @@ -308,17 +176,17 @@ func PrepareDrandInstance(t *TestEnvironment) (*DrandInstance, error) { DataDir: gossipDir, IdentityPath: path.Join(gossipDir, "identity.key"), Insecure: true, - Client: dr.httpClient, + Client: client, } t.RecordMessage("starting drand gossip relay") - dr.gossipRelay, err = lp2p.NewGossipRelayNode(log.NewLogger(getLogLevel(t)), &relayCfg) + gossipRelay, err = lp2p.NewGossipRelayNode(log.DefaultLogger, &relayCfg) if err != nil { return nil, fmt.Errorf("failed to construct drand gossip relay: %w", err) } t.RecordMessage("sharing gossip relay addrs") // share the gossip relay addrs so we can publish them in DrandRuntimeInfo - relayInfo, err := relayAddrInfo(dr.gossipRelay.Multiaddrs(), myAddr) + relayInfo, err := relayAddrInfo(gossipRelay.Multiaddrs(), myAddr) if err != nil { return nil, err } @@ -349,13 +217,17 @@ func PrepareDrandInstance(t *TestEnvironment) (*DrandInstance, error) { }, GossipBootstrap: relayAddrs, } - t.DebugSpew("publishing drand config on sync topic: %#v", cfg) + dump, _ := json.Marshal(cfg) + t.RecordMessage("publishing drand config on sync topic: %s", string(dump)) t.SyncClient.MustPublish(ctx, DrandConfigTopic, &cfg) } - // signal that we're ready to start the test - t.SyncClient.MustSignalAndWait(ctx, StateReady, t.TestInstanceCount) - return &dr, nil + return &DrandInstance{ + t: t, + Node: n, + GossipRelay: gossipRelay, + stateDir: stateDir, + }, nil } // waitForDrandConfig should be called by filecoin instances before constructing the lotus Node @@ -380,14 +252,3 @@ func relayAddrInfo(addrs []ma.Multiaddr, dataIP net.IP) (*peer.AddrInfo, error) } return nil, fmt.Errorf("no addr found with data ip %s in addrs: %v", dataIP, addrs) } - -func getLogLevel(t *TestEnvironment) int { - switch t.StringParam("drand_log_level") { - case "info": - return log.LogInfo - case "debug": - return log.LogDebug - default: - return log.LogNone - } -} diff --git a/lotus-soup/testkit/testenv.go b/lotus-soup/testkit/testenv.go index 18e1dee99..c9c2f8a53 100644 --- a/lotus-soup/testkit/testenv.go +++ b/lotus-soup/testkit/testenv.go @@ -7,6 +7,7 @@ import ( "time" "github.com/davecgh/go-spew/spew" + "github.com/testground/sdk-go/run" "github.com/testground/sdk-go/runtime" )