conformance: support multiple protocol versions.

This PR introduces support for running multiple variants of a vector,
each of which targets a unique protocol version.

tvx tooling has been adapted to produce and parse the new version
of the schema.

cmd/tvx/codenames.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"github.com/filecoin-project/go-state-types/abi"
+
+	"github.com/filecoin-project/lotus/build"
+)
+
+// ProtocolCodenames is a table that summarises the protocol codenames that
+// will be set on extracted vectors, depending on the original execution height.
+//
+// Implementers rely on these names to filter the vectors they can run through
+// their implementations, based on their support level
+var ProtocolCodenames = []struct {
+	firstEpoch abi.ChainEpoch
+	name       string
+}{
+	{0, "genesis"},
+	// TODO there is some off-by-one trickery in GetNtwkVersion. Not sure if the
+	//  protocol version really kicks in at the designated height, or at the
+	//  following epoch.
+	{build.UpgradeBreezeHeight + 1, "breeze"},
+	{build.UpgradeSmokeHeight + 1, "smoke"},
+	{build.UpgradeIgnitionHeight + 1, "ignition"},
+	{build.UpgradeRefuelHeight + 1, "refuel"},
+	{build.UpgradeActorsV2Height + 1, "actorsv2"},
+	{build.UpgradeTapeHeight + 1, "tape"},
+	{build.UpgradeLiftoffHeight + 1, "liftoff"},
+}
+
+// GetProtocolCodename gets the protocol codename associated with a height.
+func GetProtocolCodename(height abi.ChainEpoch) string {
+	for i, v := range ProtocolCodenames {
+		if height < v.firstEpoch {
+			// found the cutoff, return previous.
+			return ProtocolCodenames[i-1].name
+		}
+	}
+	return ProtocolCodenames[len(ProtocolCodenames)-1].name
+}

cmd/tvx/codenames_test.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"math"
+	"testing"
+
+	"github.com/filecoin-project/go-state-types/abi"
+
+	"github.com/filecoin-project/lotus/build"
+)
+
+func TestProtocolCodenames(t *testing.T) {
+	if height := abi.ChainEpoch(100); GetProtocolCodename(height) != "genesis" {
+		t.Fatal("expected genesis codename")
+	}
+
+	if height := abi.ChainEpoch(build.UpgradeBreezeHeight + 1); GetProtocolCodename(height) != "breeze" {
+		t.Fatal("expected breeze codename")
+	}
+
+	if height := abi.ChainEpoch(build.UpgradeActorsV2Height + 1); GetProtocolCodename(height) != "actorsv2" {
+		t.Fatal("expected actorsv2 codename")
+	}
+
+	if height := abi.ChainEpoch(math.MaxInt64); GetProtocolCodename(height) != ProtocolCodenames[len(ProtocolCodenames)-1].name {
+		t.Fatal("expected last codename")
+	}
+}

cmd/tvx/exec.go

@@ -72,20 +72,24 @@ func runExecLotus(_ *cli.Context) error {
 
 func executeTestVector(tv schema.TestVector) error {
 	log.Println("executing test vector:", tv.Meta.ID)
-	r := new(conformance.LogReporter)
-	switch class := tv.Class; class {
-	case "message":
-		conformance.ExecuteMessageVector(r, &tv)
-	case "tipset":
-		conformance.ExecuteTipsetVector(r, &tv)
-	default:
-		return fmt.Errorf("test vector class %s not supported", class)
-	}
 
-	if r.Failed() {
+	for _, v := range tv.Pre.Variants {
-		log.Println(color.HiRedString("❌ test vector failed"))
+		r := new(conformance.LogReporter)
-	} else {
+
-		log.Println(color.GreenString("✅ test vector succeeded"))
+		switch class := tv.Class; class {
+		case "message":
+			conformance.ExecuteMessageVector(r, &tv, &v)
+		case "tipset":
+			conformance.ExecuteTipsetVector(r, &tv, &v)
+		default:
+			return fmt.Errorf("test vector class %s not supported", class)
+		}
+
+		if r.Failed() {
+			log.Println(color.HiRedString("❌ test vector failed for variant %s", v.ID))
+		} else {
+			log.Println(color.GreenString("✅ test vector succeeded for variant %s", v.ID))
+		}
 	}
 
 	return nil

cmd/tvx/extract.go

@@ -347,6 +347,13 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
 		return err
 	}
 
+	nv, err := fapi.StateNetworkVersion(ctx, execTs.Key())
+	if err != nil {
+		return err
+	}
+
+	codename := GetProtocolCodename(execTs.Height())
+
 	// Write out the test vector.
 	vector := schema.TestVector{
 		Class: schema.ClassMessage,
@@ -363,10 +370,15 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
 				{Source: fmt.Sprintf("execution_tipset:%s", execTs.Key().String())},
 				{Source: "github.com/filecoin-project/lotus", Version: version.String()}},
 		},
+		Selector: schema.Selector{
+			schema.SelectorMinProtocolVersion: codename,
+		},
 		Randomness: recordingRand.Recorded(),
 		CAR:        out.Bytes(),
 		Pre: &schema.Preconditions{
-			Epoch:      int64(execTs.Height()),
+			Variants: []schema.Variant{
+				{ID: codename, Epoch: int64(execTs.Height()), NetworkVersion: uint(nv)},
+			},
 			CircSupply: circSupply.Int,
 			BaseFee:    basefee.Int,
 			StateTree: &schema.StateTree{

conformance/corpus_test.go

@@ -11,6 +11,11 @@ import (
 	"github.com/filecoin-project/test-vectors/schema"
 )
 
+var invokees = map[schema.Class]func(Reporter, *schema.TestVector, *schema.Variant){
+	schema.ClassMessage: ExecuteMessageVector,
+	schema.ClassTipset:  ExecuteTipsetVector,
+}
+
 const (
 	// EnvSkipConformance, if 1, skips the conformance test suite.
 	EnvSkipConformance = "SKIP_CONFORMANCE"
@@ -120,13 +125,16 @@ func TestConformance(t *testing.T) {
 			}
 
 			// dispatch the execution depending on the vector class.
-			switch vector.Class {
+			invokee, ok := invokees[vector.Class]
-			case "message":
+			if !ok {
-				ExecuteMessageVector(t, &vector)
+				t.Fatalf("unsupported test vector class: %s", vector.Class)
-			case "tipset":
+			}
-				ExecuteTipsetVector(t, &vector)
+
-			default:
+			for _, variant := range vector.Pre.Variants {
-				t.Fatalf("test vector class not supported: %s", vector.Class)
+				variant := variant
+				t.Run(variant.ID, func(t *testing.T) {
+					invokee(t, &vector, &variant)
+				})
 			}
 		})
 	}

conformance/driver.go

@@ -82,7 +82,7 @@ type ExecuteTipsetResult struct {
 // This method returns the the receipts root, the poststate root, and the VM
 // message results. The latter _include_ implicit messages, such as cron ticks
 // and reward withdrawal per miner.
-func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot cid.Cid, parentEpoch abi.ChainEpoch, tipset *schema.Tipset) (*ExecuteTipsetResult, error) {
+func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot cid.Cid, parentEpoch abi.ChainEpoch, tipset *schema.Tipset, execEpoch abi.ChainEpoch) (*ExecuteTipsetResult, error) {
 	var (
 		syscalls = vm.Syscalls(ffiwrapper.ProofVerifier)
 		vmRand   = NewFixedRand()
@@ -121,11 +121,10 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot
 		messages []*types.Message
 		results  []*vm.ApplyRet
 
-		epoch   = abi.ChainEpoch(tipset.Epoch)
 		basefee = abi.NewTokenAmount(tipset.BaseFee.Int64())
 	)
 
-	postcid, receiptsroot, err := sm.ApplyBlocks(context.Background(), parentEpoch, preroot, blocks, epoch, vmRand, func(_ cid.Cid, msg *types.Message, ret *vm.ApplyRet) error {
+	postcid, receiptsroot, err := sm.ApplyBlocks(context.Background(), parentEpoch, preroot, blocks, execEpoch, vmRand, func(_ cid.Cid, msg *types.Message, ret *vm.ApplyRet) error {
 		messages = append(messages, msg)
 		results = append(results, ret)
 		return nil

conformance/runner.go

@@ -30,11 +30,11 @@ import (
 )
 
 // ExecuteMessageVector executes a message-class test vector.
-func ExecuteMessageVector(r Reporter, vector *schema.TestVector) {
+func ExecuteMessageVector(r Reporter, vector *schema.TestVector, variant *schema.Variant) {
 	var (
-		ctx   = context.Background()
+		ctx       = context.Background()
-		epoch = vector.Pre.Epoch
+		baseEpoch = variant.Epoch
-		root  = vector.Pre.StateTree.RootCID
+		root      = vector.Pre.StateTree.RootCID
 	)
 
 	// Load the CAR into a new temporary Blockstore.
@@ -53,16 +53,16 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) {
 			r.Fatalf("failed to deserialize message: %s", err)
 		}
 
-		// add an epoch if one's set.
+		// add the epoch offset if one is set.
-		if m.Epoch != nil {
+		if m.EpochOffset != nil {
-			epoch = *m.Epoch
+			baseEpoch += *m.EpochOffset
 		}
 
 		// Execute the message.
 		var ret *vm.ApplyRet
 		ret, root, err = driver.ExecuteMessage(bs, ExecuteMessageParams{
 			Preroot:    root,
-			Epoch:      abi.ChainEpoch(epoch),
+			Epoch:      abi.ChainEpoch(baseEpoch),
 			Message:    msg,
 			BaseFee:    BaseFeeOrDefault(vector.Pre.BaseFee),
 			CircSupply: CircSupplyOrDefault(vector.Pre.CircSupply),
@@ -86,10 +86,10 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) {
 }
 
 // ExecuteTipsetVector executes a tipset-class test vector.
-func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) {
+func ExecuteTipsetVector(r Reporter, vector *schema.TestVector, variant *schema.Variant) {
 	var (
 		ctx       = context.Background()
-		prevEpoch = vector.Pre.Epoch
+		baseEpoch = abi.ChainEpoch(variant.Epoch)
 		root      = vector.Pre.StateTree.RootCID
 		tmpds     = ds.NewMapDatastore()
 	)
@@ -105,9 +105,11 @@ func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) {
 
 	// Apply every tipset.
 	var receiptsIdx int
+	var prevEpoch = baseEpoch
 	for i, ts := range vector.ApplyTipsets {
 		ts := ts // capture
-		ret, err := driver.ExecuteTipset(bs, tmpds, root, abi.ChainEpoch(prevEpoch), &ts)
+		execEpoch := baseEpoch + abi.ChainEpoch(ts.EpochOffset)
+		ret, err := driver.ExecuteTipset(bs, tmpds, root, prevEpoch, &ts, execEpoch)
 		if err != nil {
 			r.Fatalf("failed to apply tipset %d message: %s", i, err)
 		}
@@ -122,7 +124,7 @@ func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) {
 			r.Errorf("post receipts root doesn't match; expected: %s, was: %s", expected, actual)
 		}
 
-		prevEpoch = ts.Epoch
+		prevEpoch = execEpoch
 		root = ret.PostStateRoot
 	}
 

go.mod

@@ -41,7 +41,7 @@ require (
 	github.com/filecoin-project/specs-actors v0.9.12
 	github.com/filecoin-project/specs-actors/v2 v2.1.0
 	github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796
-	github.com/filecoin-project/test-vectors/schema v0.0.4
+	github.com/filecoin-project/test-vectors/schema v0.0.5-0.20201014133607-1352e6bb4e71
 	github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
 	github.com/go-kit/kit v0.10.0
 	github.com/go-ole/go-ole v1.2.4 // indirect

go.sum

@@ -282,8 +282,8 @@ github.com/filecoin-project/specs-actors/v2 v2.1.0 h1:ocEuGz8DG2cUWw32c/tvF8D6xT
 github.com/filecoin-project/specs-actors/v2 v2.1.0/go.mod h1:E7fAX4CZkDVQvDNRCxfq+hc3nx56KcCKyuZf0hlQJ20=
 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk=
 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g=
-github.com/filecoin-project/test-vectors/schema v0.0.4 h1:QTRd0gb/NP4ZOTM7Dib5U3xE1/ToGDKnYLfxkC3t/m8=
+github.com/filecoin-project/test-vectors/schema v0.0.5-0.20201014133607-1352e6bb4e71 h1:qnleaW7X8Gi2e3QtqPMFdqVn/DUaNI5tCnq7wNVMnio=
-github.com/filecoin-project/test-vectors/schema v0.0.4/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E=
+github.com/filecoin-project/test-vectors/schema v0.0.5-0.20201014133607-1352e6bb4e71/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as=
 github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ=