Merge pull request #4221 from filecoin-project/conformance/randomness

cmd/tvx/extract.go

@@ -200,6 +200,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
 			Message:    m,
 			CircSupply: circSupplyDetail.FilCirculating,
 			BaseFee:    basefee,
+			// recorded randomness will be discarded.
+			Rand: conformance.NewRecordingRand(new(conformance.LogReporter), fapi),
 		})
 		if err != nil {
 			return fmt.Errorf("failed to execute precursor message: %w", err)
@@ -212,6 +214,9 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
 		applyret  *vm.ApplyRet
 		carWriter func(w io.Writer) error
 		retention = opts.retain
+
+		// recordingRand will record randomness so we can embed it in the test vector.
+		recordingRand = conformance.NewRecordingRand(new(conformance.LogReporter), fapi)
 	)
 
 	log.Printf("using state retention strategy: %s", retention)
@@ -231,6 +236,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
 			Message:    msg,
 			CircSupply: circSupplyDetail.FilCirculating,
 			BaseFee:    basefee,
+			Rand:       recordingRand,
 		})
 		if err != nil {
 			return fmt.Errorf("failed to execute message: %w", err)
@@ -262,6 +268,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
 			Message:    msg,
 			CircSupply: circSupplyDetail.FilCirculating,
 			BaseFee:    basefee,
+			Rand:       recordingRand,
 		})
 		if err != nil {
 			return fmt.Errorf("failed to execute message: %w", err)
@@ -356,7 +363,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
 				{Source: fmt.Sprintf("execution_tipset:%s", execTs.Key().String())},
 				{Source: "github.com/filecoin-project/lotus", Version: version.String()}},
 		},
-		CAR: out.Bytes(),
+		Randomness: recordingRand.Recorded(),
+		CAR:        out.Bytes(),
 		Pre: &schema.Preconditions{
 			Epoch:      int64(execTs.Height()),
 			CircSupply: circSupply.Int,

conformance/rand_record.go

@@ -0,0 +1,103 @@
+package conformance
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/crypto"
+
+	"github.com/filecoin-project/test-vectors/schema"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/vm"
+)
+
+type RecordingRand struct {
+	reporter Reporter
+	api      api.FullNode
+
+	// once guards the loading of the head tipset.
+	// can be removed when https://github.com/filecoin-project/lotus/issues/4223
+	// is fixed.
+	once     sync.Once
+	head     types.TipSetKey
+	lk       sync.Mutex
+	recorded schema.Randomness
+}
+
+var _ vm.Rand = (*RecordingRand)(nil)
+
+// NewRecordingRand returns a vm.Rand implementation that proxies calls to a
+// full Lotus node via JSON-RPC, and records matching rules and responses so
+// they can later be embedded in test vectors.
+func NewRecordingRand(reporter Reporter, api api.FullNode) *RecordingRand {
+	return &RecordingRand{reporter: reporter, api: api}
+}
+
+func (r *RecordingRand) loadHead() {
+	head, err := r.api.ChainHead(context.Background())
+	if err != nil {
+		panic(fmt.Sprintf("could not fetch chain head while fetching randomness: %s", err))
+	}
+	r.head = head.Key()
+}
+
+func (r *RecordingRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
+	r.once.Do(r.loadHead)
+	ret, err := r.api.ChainGetRandomnessFromTickets(ctx, r.head, pers, round, entropy)
+	if err != nil {
+		return ret, err
+	}
+
+	r.reporter.Logf("fetched and recorded chain randomness for: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret)
+
+	match := schema.RandomnessMatch{
+		On: schema.RandomnessRule{
+			Kind:                schema.RandomnessChain,
+			DomainSeparationTag: int64(pers),
+			Epoch:               int64(round),
+			Entropy:             entropy,
+		},
+		Return: []byte(ret),
+	}
+	r.lk.Lock()
+	r.recorded = append(r.recorded, match)
+	r.lk.Unlock()
+
+	return ret, err
+}
+
+func (r *RecordingRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
+	r.once.Do(r.loadHead)
+	ret, err := r.api.ChainGetRandomnessFromBeacon(ctx, r.head, pers, round, entropy)
+	if err != nil {
+		return ret, err
+	}
+
+	r.reporter.Logf("fetched and recorded beacon randomness for: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret)
+
+	match := schema.RandomnessMatch{
+		On: schema.RandomnessRule{
+			Kind:                schema.RandomnessBeacon,
+			DomainSeparationTag: int64(pers),
+			Epoch:               int64(round),
+			Entropy:             entropy,
+		},
+		Return: []byte(ret),
+	}
+	r.lk.Lock()
+	r.recorded = append(r.recorded, match)
+	r.lk.Unlock()
+
+	return ret, err
+}
+
+func (r *RecordingRand) Recorded() schema.Randomness {
+	r.lk.Lock()
+	defer r.lk.Unlock()
+
+	return r.recorded
+}

conformance/rand_replay.go

@@ -0,0 +1,79 @@
+package conformance
+
+import (
+	"bytes"
+	"context"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/crypto"
+
+	"github.com/filecoin-project/test-vectors/schema"
+
+	"github.com/filecoin-project/lotus/chain/vm"
+)
+
+type ReplayingRand struct {
+	reporter Reporter
+	recorded schema.Randomness
+	fallback vm.Rand
+}
+
+var _ vm.Rand = (*ReplayingRand)(nil)
+
+// NewReplayingRand replays recorded randomness when requested, falling back to
+// fixed randomness if the value cannot be found; hence this is a safe
+// backwards-compatible replacement for fixedRand.
+func NewReplayingRand(reporter Reporter, recorded schema.Randomness) *ReplayingRand {
+	return &ReplayingRand{
+		reporter: reporter,
+		recorded: recorded,
+		fallback: NewFixedRand(),
+	}
+}
+
+func (r *ReplayingRand) match(requested schema.RandomnessRule) ([]byte, bool) {
+	for _, other := range r.recorded {
+		if other.On.Kind == requested.Kind &&
+			other.On.Epoch == requested.Epoch &&
+			other.On.DomainSeparationTag == requested.DomainSeparationTag &&
+			bytes.Equal(other.On.Entropy, requested.Entropy) {
+			return other.Return, true
+		}
+	}
+	return nil, false
+}
+
+func (r *ReplayingRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
+	rule := schema.RandomnessRule{
+		Kind:                schema.RandomnessChain,
+		DomainSeparationTag: int64(pers),
+		Epoch:               int64(round),
+		Entropy:             entropy,
+	}
+
+	if ret, ok := r.match(rule); ok {
+		r.reporter.Logf("returning saved chain randomness: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret)
+		return ret, nil
+	}
+
+	r.reporter.Logf("returning fallback chain randomness: dst=%d, epoch=%d, entropy=%x", pers, round, entropy)
+	return r.fallback.GetChainRandomness(ctx, pers, round, entropy)
+}
+
+func (r *ReplayingRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
+	rule := schema.RandomnessRule{
+		Kind:                schema.RandomnessBeacon,
+		DomainSeparationTag: int64(pers),
+		Epoch:               int64(round),
+		Entropy:             entropy,
+	}
+
+	if ret, ok := r.match(rule); ok {
+		r.reporter.Logf("returning saved beacon randomness: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret)
+		return ret, nil
+	}
+
+	r.reporter.Logf("returning fallback beacon randomness: dst=%d, epoch=%d, entropy=%x", pers, round, entropy)
+	return r.fallback.GetBeaconRandomness(ctx, pers, round, entropy)
+
+}

conformance/runner.go

@@ -66,6 +66,7 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) {
 			Message:    msg,
 			BaseFee:    BaseFeeOrDefault(vector.Pre.BaseFee),
 			CircSupply: CircSupplyOrDefault(vector.Pre.CircSupply),
+			Rand:       NewReplayingRand(r, vector.Randomness),
 		})
 		if err != nil {
 			r.Fatalf("fatal failure when executing message: %s", err)

go.mod

@@ -41,7 +41,7 @@ require (
 	github.com/filecoin-project/specs-actors v0.9.12
 	github.com/filecoin-project/specs-actors/v2 v2.0.1
 	github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796
-	github.com/filecoin-project/test-vectors/schema v0.0.3
+	github.com/filecoin-project/test-vectors/schema v0.0.4
 	github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
 	github.com/go-kit/kit v0.10.0
 	github.com/go-ole/go-ole v1.2.4 // indirect

go.sum

@@ -280,8 +280,8 @@ github.com/filecoin-project/specs-actors/v2 v2.0.1 h1:bf08x6tqCDfClzrv2q/rmt/A/U
 github.com/filecoin-project/specs-actors/v2 v2.0.1/go.mod h1:v2NZVYinNIKA9acEMBm5wWXxqv5+frFEbekBFemYghY=
 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk=
 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g=
-github.com/filecoin-project/test-vectors/schema v0.0.3 h1:1zuBo25B3016inbygYLgYFdpJ2m1BDTbAOCgABRleiU=
+github.com/filecoin-project/test-vectors/schema v0.0.4 h1:QTRd0gb/NP4ZOTM7Dib5U3xE1/ToGDKnYLfxkC3t/m8=
-github.com/filecoin-project/test-vectors/schema v0.0.3/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E=
+github.com/filecoin-project/test-vectors/schema v0.0.4/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as=
 github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ=