lotus/lotus-soup/role_drand.go

package main

import (
	"bytes"
	"context"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path"
	"time"

	"github.com/drand/drand/chain"
	"github.com/drand/drand/client"
	hclient "github.com/drand/drand/client/http"
	"github.com/drand/drand/core"
	"github.com/drand/drand/key"
	"github.com/drand/drand/log"
	"github.com/drand/drand/lp2p"
	dnet "github.com/drand/drand/net"
	"github.com/drand/drand/protobuf/drand"
	dtest "github.com/drand/drand/test"
	"github.com/filecoin-project/lotus/node/modules/dtypes"
	"github.com/libp2p/go-libp2p-core/peer"
	ma "github.com/multiformats/go-multiaddr"
	"github.com/testground/sdk-go/sync"

	"github.com/filecoin-project/oni/lotus-soup/statemachine"
)

var (
	PrepareDrandTimeout = time.Minute
	secretDKG           = "dkgsecret"
)

type DrandInstance struct {
	daemon      *core.Drand
	httpClient  client.Client
	ctrlClient  *dnet.ControlClient
	gossipRelay *lp2p.GossipRelayNode

	t        *TestEnvironment
	stateDir string
	priv     *key.Pair
	pubAddr  string
	privAddr string
	ctrlAddr string
}

func (dr *DrandInstance) Start() error {
	opts := []core.ConfigOption{
		core.WithLogLevel(getLogLevel(dr.t)),
		core.WithConfigFolder(dr.stateDir),
		core.WithPublicListenAddress(dr.pubAddr),
		core.WithPrivateListenAddress(dr.privAddr),
		core.WithControlPort(dr.ctrlAddr),
		core.WithInsecure(),
	}
	conf := core.NewConfig(opts...)
	fs := key.NewFileStore(conf.ConfigFolder())
	fs.SaveKeyPair(dr.priv)
	key.Save(path.Join(dr.stateDir, "public.toml"), dr.priv.Public, false)
	if dr.daemon == nil {
		drand, err := core.NewDrand(fs, conf)
		if err != nil {
			return err
		}
		dr.daemon = drand
	} else {
		drand, err := core.LoadDrand(fs, conf)
		if err != nil {
			return err
		}
		drand.StartBeacon(true)
		dr.daemon = drand
	}
	return nil
}

func (dr *DrandInstance) Ping() bool {
	cl := dr.ctrl()
	if err := cl.Ping(); err != nil {
		return false
	}
	return true
}

func (dr *DrandInstance) Close() error {
	dr.gossipRelay.Shutdown()
	dr.daemon.Stop(context.Background())
	return os.RemoveAll(dr.stateDir)
}

func (dr *DrandInstance) ctrl() *dnet.ControlClient {
	if dr.ctrlClient != nil {
		return dr.ctrlClient
	}
	cl, err := dnet.NewControlClient(dr.ctrlAddr)
	if err != nil {
		dr.t.RecordMessage("drand can't instantiate control client: %w", err)
		return nil
	}
	dr.ctrlClient = cl
	return cl
}

func (dr *DrandInstance) RunDKG(nodes, thr int, timeout string, leader bool, leaderAddr string, beaconOffset int) *key.Group {
	cl := dr.ctrl()
	p := dr.t.DurationParam("drand_period")
	t, _ := time.ParseDuration(timeout)
	var grp *drand.GroupPacket
	var err error
	if leader {
		grp, err = cl.InitDKGLeader(nodes, thr, p, t, nil, secretDKG, beaconOffset)
	} else {
		leader := dnet.CreatePeer(leaderAddr, false)
		grp, err = cl.InitDKG(leader, nil, secretDKG)
	}
	if err != nil {
		dr.t.RecordMessage("drand dkg run failed: %w", err)
		return nil
	}
	kg, _ := key.GroupFromProto(grp)
	return kg
}

func (dr *DrandInstance) Halt() {
	dr.t.RecordMessage("drand node #%d halting", dr.t.GroupSeq)
	dr.daemon.StopBeacon()
}

func (dr *DrandInstance) Resume() {
	dr.t.RecordMessage("drand node #d resuming", dr.t.GroupSeq)
	dr.daemon.StartBeacon(true)
	// block until we can fetch the round corresponding to the current time
	startTime := time.Now()
	round := dr.httpClient.RoundAt(startTime)
	timeout := 30 * time.Second
	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()

	done := make(chan struct{}, 1)
	go func() {
		for {
			res, err := dr.httpClient.Get(ctx, round)
			if err == nil {
				dr.t.RecordMessage("drand chain caught up to round %d", res.Round())
				done <- struct{}{}
				return
			}
		}
	}()

	select {
	case <-ctx.Done():
		dr.t.RecordMessage("drand chain failed to catch up after %s", timeout.String())
	case <-done:
		dr.t.RecordMessage("drand chain resumed after %s catchup time", time.Since(startTime))
	}
}

func runDrandNode(t *TestEnvironment) error {
	t.RecordMessage("running drand node")
	dr, err := prepareDrandNode(t)
	if err != nil {
		return err
	}
	defer dr.Close()

	ctx := context.Background()
	t.SyncClient.MustSignalAndWait(ctx, stateReady, t.TestInstanceCount)

	if t.IsParamSet("suspend_events") {
		suspender := statemachine.NewSuspender(dr, t.RecordMessage)
		suspender.RunEvents(t.StringParam("suspend_events"))
	}

	t.SyncClient.MustSignalAndWait(ctx, stateDone, t.TestInstanceCount)
	return nil
}

// prepareDrandNode starts a drand instance and runs a DKG with the other members of the composition group.
// Once the chain is running, the leader publishes the chain info needed by lotus nodes on
// drandConfigTopic
func prepareDrandNode(t *TestEnvironment) (*DrandInstance, error) {
	ctx, cancel := context.WithTimeout(context.Background(), PrepareDrandTimeout)
	defer cancel()

	startTime := time.Now()

	seq := t.GroupSeq
	isLeader := seq == 1
	nNodes := t.TestGroupInstanceCount

	myAddr := t.NetClient.MustGetDataNetworkIP()
	threshold := t.IntParam("drand_threshold")
	runGossipRelay := t.BooleanParam("drand_gossip_relay")

	beaconOffset := 3

	stateDir, err := ioutil.TempDir("/tmp", fmt.Sprintf("drand-%d", t.GroupSeq))
	if err != nil {
		return nil, err
	}

	dr := DrandInstance{
		t:        t,
		stateDir: stateDir,
		pubAddr:  dtest.FreeBind(myAddr.String()),
		privAddr: dtest.FreeBind(myAddr.String()),
		ctrlAddr: dtest.FreeBind("localhost"),
	}
	dr.priv = key.NewKeyPair(dr.privAddr)

	// share the node addresses with other nodes
	// TODO: if we implement TLS, this is where we'd share public TLS keys
	type NodeAddr struct {
		PrivateAddr string
		PublicAddr  string
		IsLeader    bool
	}
	addrTopic := sync.NewTopic("drand-addrs", &NodeAddr{})
	var publicAddrs []string
	var leaderAddr string
	ch := make(chan *NodeAddr)
	_, sub := t.SyncClient.MustPublishSubscribe(ctx, addrTopic, &NodeAddr{
		PrivateAddr: dr.privAddr,
		PublicAddr:  dr.pubAddr,
		IsLeader:    isLeader,
	}, ch)
	for i := 0; i < nNodes; i++ {
		select {
		case msg := <-ch:
			publicAddrs = append(publicAddrs, fmt.Sprintf("http://%s", msg.PublicAddr))
			if msg.IsLeader {
				leaderAddr = msg.PrivateAddr
			}
		case err := <-sub.Done():
			return nil, fmt.Errorf("unable to read drand addrs from sync service: %w", err)
		}
	}
	if leaderAddr == "" {
		return nil, fmt.Errorf("got %d drand addrs, but no leader", len(publicAddrs))
	}

	t.SyncClient.MustSignalAndWait(ctx, "drand-start", nNodes)
	t.RecordMessage("Starting drand sharing ceremony")
	if err := dr.Start(); err != nil {
		return nil, err
	}

	alive := false
	waitSecs := 10
	for i := 0; i < waitSecs; i++ {
		if !dr.Ping() {
			time.Sleep(time.Second)
			continue
		}
		t.R().RecordPoint("drand_first_ping", time.Now().Sub(startTime).Seconds())
		alive = true
		break
	}
	if !alive {
		return nil, fmt.Errorf("drand node %d failed to start after %d seconds", t.GroupSeq, waitSecs)
	}

	// run DKG
	t.SyncClient.MustSignalAndWait(ctx, "drand-dkg-start", nNodes)
	if !isLeader {
		time.Sleep(time.Second)
	}
	grp := dr.RunDKG(nNodes, threshold, "10s", isLeader, leaderAddr, beaconOffset)
	if grp == nil {
		return nil, fmt.Errorf("drand dkg failed")
	}
	t.R().RecordPoint("drand_dkg_complete", time.Now().Sub(startTime).Seconds())

	t.RecordMessage("drand dkg complete, waiting for chain start")
	// wait for chain to begin
	to := time.Until(time.Unix(grp.GenesisTime, 0).Add(3 * time.Second).Add(grp.Period))
	time.Sleep(to)

	t.RecordMessage("drand beacon chain started, fetching initial round via http")
	// verify that we can get a round of randomness from the chain using an http client
	info := chain.NewChainInfo(grp)
	myPublicAddr := fmt.Sprintf("http://%s", dr.pubAddr)
	dr.httpClient, err = hclient.NewWithInfo(myPublicAddr, info, nil)
	if err != nil {
		return nil, fmt.Errorf("unable to create drand http client: %w", err)
	}

	_, err = dr.httpClient.Get(ctx, 1)
	if err != nil {
		return nil, fmt.Errorf("unable to get initial drand round: %w", err)
	}

	// start gossip relay (unless disabled via testplan parameter)
	var relayAddrs []peer.AddrInfo

	if runGossipRelay {
		gossipDir := path.Join(stateDir, "gossip-relay")
		listenAddr := fmt.Sprintf("/ip4/%s/tcp/7777", myAddr.String())
		relayCfg := lp2p.GossipRelayConfig{
			ChainHash:    hex.EncodeToString(info.Hash()),
			Addr:         listenAddr,
			DataDir:      gossipDir,
			IdentityPath: path.Join(gossipDir, "identity.key"),
			Insecure:     true,
			Client:       dr.httpClient,
		}
		t.RecordMessage("starting drand gossip relay")
		dr.gossipRelay, err = lp2p.NewGossipRelayNode(log.NewLogger(getLogLevel(t)), &relayCfg)
		if err != nil {
			return nil, fmt.Errorf("failed to construct drand gossip relay: %w", err)
		}

		t.RecordMessage("sharing gossip relay addrs")
		// share the gossip relay addrs so we can publish them in DrandRuntimeInfo
		relayInfo, err := relayAddrInfo(dr.gossipRelay.Multiaddrs(), myAddr)
		if err != nil {
			return nil, err
		}
		infoCh := make(chan *peer.AddrInfo, nNodes)
		infoTopic := sync.NewTopic("drand-gossip-addrs", &peer.AddrInfo{})

		_, sub := t.SyncClient.MustPublishSubscribe(ctx, infoTopic, relayInfo, infoCh)
		for i := 0; i < nNodes; i++ {
			select {
			case ai := <-infoCh:
				relayAddrs = append(relayAddrs, *ai)
			case err := <-sub.Done():
				return nil, fmt.Errorf("unable to get drand relay addr from sync service: %w", err)
			}
		}
	}

	// if we're the leader, publish the config to the sync service
	if isLeader {
		buf := bytes.Buffer{}
		if err := info.ToJSON(&buf); err != nil {
			return nil, fmt.Errorf("error marshaling chain info: %w", err)
		}
		cfg := DrandRuntimeInfo{
			Config: dtypes.DrandConfig{
				Servers:       publicAddrs,
				ChainInfoJSON: buf.String(),
			},
			GossipBootstrap: relayAddrs,
		}
		t.DebugSpew("publishing drand config on sync topic: %v", cfg)
		t.SyncClient.MustPublish(ctx, drandConfigTopic, &cfg)
	}

	return &dr, nil
}

// waitForDrandConfig should be called by filecoin instances before constructing the lotus Node
// you can use the returned dtypes.DrandConfig to override the default production config.
func waitForDrandConfig(ctx context.Context, client sync.Client) (*DrandRuntimeInfo, error) {
	ch := make(chan *DrandRuntimeInfo, 1)
	sub := client.MustSubscribe(ctx, drandConfigTopic, ch)
	select {
	case cfg := <-ch:
		return cfg, nil
	case err := <-sub.Done():
		return nil, err
	}
}

func relayAddrInfo(addrs []ma.Multiaddr, dataIP net.IP) (*peer.AddrInfo, error) {
	for _, a := range addrs {
		if ip, _ := a.ValueForProtocol(ma.P_IP4); ip != dataIP.String() {
			continue
		}
		return peer.AddrInfoFromP2pAddr(a)
	}
	return nil, fmt.Errorf("no addr found with data ip %s in addrs: %v", dataIP, addrs)
}

func getLogLevel(t *TestEnvironment) int {
	switch t.StringParam("drand_log_level") {
	case "info":
		return log.LogInfo
	case "debug":
		return log.LogDebug
	default:
		return log.LogNone
	}
}