From 5bd6a3cdadd47fc5f26027b522fd0e94724350df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 7 Oct 2020 19:52:37 +0100 Subject: [PATCH 1/3] conformance: record randomness in tvx; replay in driver. --- cmd/tvx/extract.go | 10 ++++- conformance/rand_record.go | 86 ++++++++++++++++++++++++++++++++++++++ conformance/rand_replay.go | 78 ++++++++++++++++++++++++++++++++++ conformance/runner.go | 1 + go.mod | 2 +- go.sum | 4 +- 6 files changed, 177 insertions(+), 4 deletions(-) create mode 100644 conformance/rand_record.go create mode 100644 conformance/rand_replay.go diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index afdeb9540..b0ed574df 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -200,6 +200,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { Message: m, CircSupply: circSupplyDetail.FilCirculating, BaseFee: basefee, + // recorded randomness will be discarded. + Rand: conformance.NewRecordingRand(new(conformance.LogReporter), fapi), }) if err != nil { return fmt.Errorf("failed to execute precursor message: %w", err) @@ -212,6 +214,9 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { applyret *vm.ApplyRet carWriter func(w io.Writer) error retention = opts.retain + + // recordingRand will record randomness so we can embed it in the test vector. + recordingRand = conformance.NewRecordingRand(new(conformance.LogReporter), fapi) ) log.Printf("using state retention strategy: %s", retention) @@ -231,6 +236,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { Message: msg, CircSupply: circSupplyDetail.FilCirculating, BaseFee: basefee, + Rand: recordingRand, }) if err != nil { return fmt.Errorf("failed to execute message: %w", err) @@ -262,6 +268,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { Message: msg, CircSupply: circSupplyDetail.FilCirculating, BaseFee: basefee, + Rand: recordingRand, }) if err != nil { return fmt.Errorf("failed to execute message: %w", err) @@ -356,7 +363,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { {Source: fmt.Sprintf("execution_tipset:%s", execTs.Key().String())}, {Source: "github.com/filecoin-project/lotus", Version: version.String()}}, }, - CAR: out.Bytes(), + Randomness: recordingRand.Recorded(), + CAR: out.Bytes(), Pre: &schema.Preconditions{ Epoch: int64(execTs.Height()), CircSupply: circSupply.Int, diff --git a/conformance/rand_record.go b/conformance/rand_record.go new file mode 100644 index 000000000..2d3477c94 --- /dev/null +++ b/conformance/rand_record.go @@ -0,0 +1,86 @@ +package conformance + +import ( + "context" + "sync" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/crypto" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/test-vectors/schema" +) + +type RecordingRand struct { + reporter Reporter + api api.FullNode + + lk sync.Mutex + recorded schema.Randomness +} + +var _ vm.Rand = (*RecordingRand)(nil) + +// NewRecordingRand returns a vm.Rand implementation that proxies calls to a +// full Lotus node via JSON-RPC, and records matching rules and responses so +// they can later be embedded in test vectors. +func NewRecordingRand(reporter Reporter, api api.FullNode) *RecordingRand { + return &RecordingRand{reporter: reporter, api: api} +} + +func (r *RecordingRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { + ret, err := r.api.ChainGetRandomnessFromTickets(ctx, types.EmptyTSK, pers, round, entropy) + if err != nil { + return ret, err + } + + r.reporter.Logf("fetched and recorded chain randomness for: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret) + + match := schema.RandomnessMatch{ + On: schema.RandomnessRule{ + Kind: schema.RandomnessChain, + DomainSeparationTag: int64(pers), + Epoch: int64(round), + Entropy: entropy, + }, + Return: []byte(ret), + } + r.lk.Lock() + r.recorded = append(r.recorded, match) + r.lk.Unlock() + + return ret, err +} + +func (r *RecordingRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { + ret, err := r.api.ChainGetRandomnessFromBeacon(ctx, types.EmptyTSK, pers, round, entropy) + if err != nil { + return ret, err + } + + r.reporter.Logf("fetched and recorded beacon randomness for: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret) + + match := schema.RandomnessMatch{ + On: schema.RandomnessRule{ + Kind: schema.RandomnessBeacon, + DomainSeparationTag: int64(pers), + Epoch: int64(round), + Entropy: entropy, + }, + Return: []byte(ret), + } + r.lk.Lock() + r.recorded = append(r.recorded, match) + r.lk.Unlock() + + return ret, err +} + +func (r *RecordingRand) Recorded() schema.Randomness { + r.lk.Lock() + defer r.lk.Unlock() + + return r.recorded +} diff --git a/conformance/rand_replay.go b/conformance/rand_replay.go new file mode 100644 index 000000000..b820b8ced --- /dev/null +++ b/conformance/rand_replay.go @@ -0,0 +1,78 @@ +package conformance + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/crypto" + + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/test-vectors/schema" +) + +type ReplayingRand struct { + reporter Reporter + recorded schema.Randomness + fallback vm.Rand +} + +var _ vm.Rand = (*ReplayingRand)(nil) + +// NewReplayingRand replays recorded randomness when requested, falling back to +// fixed randomness if the value cannot be found; hence this is a safe +// backwards-compatible replacement for fixedRand. +func NewReplayingRand(reporter Reporter, recorded schema.Randomness) *ReplayingRand { + return &ReplayingRand{ + reporter: reporter, + recorded: recorded, + fallback: NewFixedRand(), + } +} + +func (r *ReplayingRand) match(requested schema.RandomnessRule) ([]byte, bool) { + for _, other := range r.recorded { + if other.On.Kind == requested.Kind && + other.On.Epoch == requested.Epoch && + other.On.DomainSeparationTag == requested.DomainSeparationTag && + bytes.Equal(other.On.Entropy, requested.Entropy) { + return other.Return, true + } + } + return nil, false +} + +func (r *ReplayingRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { + rule := schema.RandomnessRule{ + Kind: schema.RandomnessChain, + DomainSeparationTag: int64(pers), + Epoch: int64(round), + Entropy: entropy, + } + + if ret, ok := r.match(rule); ok { + r.reporter.Logf("returning saved chain randomness: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret) + return ret, nil + } + + r.reporter.Logf("returning fallback chain randomness: dst=%d, epoch=%d, entropy=%x", pers, round, entropy) + return r.fallback.GetChainRandomness(ctx, pers, round, entropy) +} + +func (r *ReplayingRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { + rule := schema.RandomnessRule{ + Kind: schema.RandomnessBeacon, + DomainSeparationTag: int64(pers), + Epoch: int64(round), + Entropy: entropy, + } + + if ret, ok := r.match(rule); ok { + r.reporter.Logf("returning saved beacon randomness: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret) + return ret, nil + } + + r.reporter.Logf("returning fallback beacon randomness: dst=%d, epoch=%d, entropy=%x", pers, round, entropy) + return r.fallback.GetBeaconRandomness(ctx, pers, round, entropy) + +} diff --git a/conformance/runner.go b/conformance/runner.go index a240c9067..3897ad853 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -66,6 +66,7 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { Message: msg, BaseFee: BaseFeeOrDefault(vector.Pre.BaseFee), CircSupply: CircSupplyOrDefault(vector.Pre.CircSupply), + Rand: NewReplayingRand(r, vector.Randomness), }) if err != nil { r.Fatalf("fatal failure when executing message: %s", err) diff --git a/go.mod b/go.mod index 170dc3003..e401af36d 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.11 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 - github.com/filecoin-project/test-vectors/schema v0.0.3 + github.com/filecoin-project/test-vectors/schema v0.0.4-0.20201007174510-548c9399aa6a github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index e820ede3a..7d6c743ae 100644 --- a/go.sum +++ b/go.sum @@ -274,8 +274,8 @@ github.com/filecoin-project/specs-actors v0.9.11 h1:TnpG7HAeiUrfj0mJM7UaPW0P2137 github.com/filecoin-project/specs-actors v0.9.11/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= -github.com/filecoin-project/test-vectors/schema v0.0.3 h1:1zuBo25B3016inbygYLgYFdpJ2m1BDTbAOCgABRleiU= -github.com/filecoin-project/test-vectors/schema v0.0.3/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= +github.com/filecoin-project/test-vectors/schema v0.0.4-0.20201007174510-548c9399aa6a h1:2jRLaT3/sHyAinWR2asCAkvzcnOAIi13eTlWf3YEVvY= +github.com/filecoin-project/test-vectors/schema v0.0.4-0.20201007174510-548c9399aa6a/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= From c02daca9e973354a75660c18bf1cb921e4ff096e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 7 Oct 2020 20:12:35 +0100 Subject: [PATCH 2/3] work around #4223. --- conformance/rand_record.go | 23 ++++++++++++++++++++--- conformance/rand_replay.go | 3 ++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/conformance/rand_record.go b/conformance/rand_record.go index 2d3477c94..6f6d064dc 100644 --- a/conformance/rand_record.go +++ b/conformance/rand_record.go @@ -2,21 +2,28 @@ package conformance import ( "context" + "fmt" "sync" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/test-vectors/schema" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/test-vectors/schema" ) type RecordingRand struct { reporter Reporter api api.FullNode + // once guards the loading of the head tipset. + // can be removed when https://github.com/filecoin-project/lotus/issues/4223 + // is fixed. + once sync.Once + head types.TipSetKey lk sync.Mutex recorded schema.Randomness } @@ -30,8 +37,17 @@ func NewRecordingRand(reporter Reporter, api api.FullNode) *RecordingRand { return &RecordingRand{reporter: reporter, api: api} } +func (r *RecordingRand) loadHead() { + head, err := r.api.ChainHead(context.Background()) + if err != nil { + panic(fmt.Sprintf("could not fetch chain head while fetching randomness: %s", err)) + } + r.head = head.Key() +} + func (r *RecordingRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { - ret, err := r.api.ChainGetRandomnessFromTickets(ctx, types.EmptyTSK, pers, round, entropy) + r.once.Do(r.loadHead) + ret, err := r.api.ChainGetRandomnessFromTickets(ctx, r.head, pers, round, entropy) if err != nil { return ret, err } @@ -55,7 +71,8 @@ func (r *RecordingRand) GetChainRandomness(ctx context.Context, pers crypto.Doma } func (r *RecordingRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { - ret, err := r.api.ChainGetRandomnessFromBeacon(ctx, types.EmptyTSK, pers, round, entropy) + r.once.Do(r.loadHead) + ret, err := r.api.ChainGetRandomnessFromBeacon(ctx, r.head, pers, round, entropy) if err != nil { return ret, err } diff --git a/conformance/rand_replay.go b/conformance/rand_replay.go index b820b8ced..1b73e5a08 100644 --- a/conformance/rand_replay.go +++ b/conformance/rand_replay.go @@ -7,8 +7,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/test-vectors/schema" + + "github.com/filecoin-project/lotus/chain/vm" ) type ReplayingRand struct { From 1f9446f91fc19473f51408680585035b4dc468ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 8 Oct 2020 15:00:35 +0100 Subject: [PATCH 3/3] upgrade to test vectors schema v0.0.4. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e401af36d..5473dbb59 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.11 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 - github.com/filecoin-project/test-vectors/schema v0.0.4-0.20201007174510-548c9399aa6a + github.com/filecoin-project/test-vectors/schema v0.0.4 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index 7d6c743ae..45d20d118 100644 --- a/go.sum +++ b/go.sum @@ -274,8 +274,8 @@ github.com/filecoin-project/specs-actors v0.9.11 h1:TnpG7HAeiUrfj0mJM7UaPW0P2137 github.com/filecoin-project/specs-actors v0.9.11/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= -github.com/filecoin-project/test-vectors/schema v0.0.4-0.20201007174510-548c9399aa6a h1:2jRLaT3/sHyAinWR2asCAkvzcnOAIi13eTlWf3YEVvY= -github.com/filecoin-project/test-vectors/schema v0.0.4-0.20201007174510-548c9399aa6a/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= +github.com/filecoin-project/test-vectors/schema v0.0.4 h1:QTRd0gb/NP4ZOTM7Dib5U3xE1/ToGDKnYLfxkC3t/m8= +github.com/filecoin-project/test-vectors/schema v0.0.4/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ=