introduce interoperable vector-based conformance testing.

This commit introduces a new package `conformance` containing:

 1. the test driver to exercise Lotus against interoperable
    test vectors, and
 2. the test runner, which integrates go test with the test vector
    corpus hosted at https://github.com/filecoin-project/conformance-vectors.

The corpus is mounted via a git submodule.

Right now, only message-class test vectors are supported. In the
next week, this support will be extended to tipset-class, chain-class,
and block sequence-class vectors.

.gitmodules

@@ -5,3 +5,7 @@
 [submodule "extern/serialization-vectors"]
 	path = extern/serialization-vectors
 	url = https://github.com/filecoin-project/serialization-vectors
+[submodule "extern/conformance-vectors"]
+	path = extern/conformance-vectors
+	url = https://github.com/filecoin-project/conformance-vectors.git
+	branch = initial

conformance/driver.go

@@ -0,0 +1,59 @@
+package conformance
+
+import (
+	"context"
+
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/puppet"
+	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/vm"
+	"github.com/filecoin-project/lotus/lib/blockstore"
+	"github.com/filecoin-project/sector-storage/ffiwrapper"
+)
+
+var (
+	// BaseFee to use in the VM.
+	// TODO make parametrisable through vector.
+	BaseFee = abi.NewTokenAmount(100)
+)
+
+type Driver struct {
+	ctx context.Context
+}
+
+func NewDriver(ctx context.Context) *Driver {
+	return &Driver{ctx: ctx}
+}
+
+// ExecuteMessage executes a conformance test vector message in a temporary VM.
+func (d *Driver) ExecuteMessage(msg *types.Message, preroot cid.Cid, bs blockstore.Blockstore, epoch abi.ChainEpoch) (*vm.ApplyRet, cid.Cid, error) {
+	vmOpts := &vm.VMOpts{
+		StateBase:      preroot,
+		Epoch:          epoch,
+		Rand:           &testRand{}, // TODO always succeeds; need more flexibility.
+		Bstore:         bs,
+		Syscalls:       mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility.
+		CircSupplyCalc: nil,
+		BaseFee:        BaseFee,
+	}
+
+	lvm, err := vm.NewVM(vmOpts)
+	if err != nil {
+		return nil, cid.Undef, err
+	}
+
+	// add support for the puppet actor.
+	invoker := vm.NewInvoker()
+	invoker.Register(puppet.PuppetActorCodeID, puppet.Actor{}, puppet.State{})
+	lvm.SetInvoker(invoker)
+
+	ret, err := lvm.ApplyMessage(d.ctx, msg)
+	if err != nil {
+		return nil, cid.Undef, err
+	}
+
+	root, err := lvm.Flush(d.ctx)
+	return ret, root, err
+}

conformance/runner_test.go

@@ -0,0 +1,180 @@
+package conformance
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"flag"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/vm"
+	"github.com/filecoin-project/lotus/lib/blockstore"
+
+	"github.com/ipld/go-car"
+)
+
+const (
+	// defaultRoot is default root at where the message vectors are hosted.
+	//
+	// You can run this test with the -vectors.root flag to execute
+	// a custom corpus.
+	defaultRoot = "../extern/conformance-vectors"
+)
+
+var (
+	// root is the effective root path, taken from the `-vectors.root` CLI flag,
+	// falling back to defaultRoot if not provided.
+	root string
+	// ignore is a set of paths relative to root to skip.
+	ignore = map[string]struct{}{
+		".git":        {},
+		"schema.json": {},
+	}
+)
+
+func init() {
+	// read the alternative root from the -vectors.root CLI flag.
+	flag.StringVar(&root, "vectors.root", defaultRoot, "root directory containing test vectors")
+}
+
+// TestConformance is the entrypoint test that runs all test vectors found
+// in the root directory.
+//
+// It locates all json files via a recursive walk, skipping over the ignore set,
+// as well as files beginning with _. It parses each file as a test vector, and
+// runs it via the Driver.
+func TestConformance(t *testing.T) {
+	var vectors []string
+	err := filepath.Walk(root+"/", func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		filename := filepath.Base(path)
+		rel, err := filepath.Rel(root, path)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if _, ok := ignore[rel]; ok {
+			// skip over using the right error.
+			if info.IsDir() {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		if info.IsDir() {
+			// dive into directories.
+			return nil
+		}
+		if filepath.Ext(path) != ".json" {
+			// skip if not .json.
+			return nil
+		}
+		if ignored := strings.HasPrefix(filename, "_"); ignored {
+			// ignore files starting with _.
+			t.Logf("ignoring: %s", rel)
+			return nil
+		}
+		vectors = append(vectors, rel)
+		return nil
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(vectors) == 0 {
+		t.Fatalf("no test vectors found")
+	}
+
+	// Run a test for each vector.
+	for _, v := range vectors {
+		v := v
+		t.Run(v, func(t *testing.T) {
+			path := filepath.Join(root, v)
+			raw, err := ioutil.ReadFile(path)
+			if err != nil {
+				t.Fatalf("failed to read test raw file: %s", path)
+			}
+
+			var vector TestVector
+			err = json.Unmarshal(raw, &vector)
+			if err != nil {
+				t.Fatalf("failed to parse test raw: %s", err)
+			}
+
+			// dispatch the execution depending on the vector class.
+			switch vector.Class {
+			case "message":
+				executeMessageVector(t, &vector)
+			default:
+				t.Fatalf("test vector class not supported: %s", vector.Class)
+			}
+		})
+	}
+}
+
+// executeMessageVector executes a message-class test vector.
+func executeMessageVector(t *testing.T, vector *TestVector) {
+	var (
+		ctx   = context.Background()
+		epoch = vector.Pre.Epoch
+		root  = vector.Pre.StateTree.RootCID
+	)
+
+	bs := blockstore.NewTemporary()
+
+	// Read the base64-encoded CAR from the vector, and inflate the gzip.
+	buf := bytes.NewReader(vector.CAR)
+	r, err := gzip.NewReader(buf)
+	if err != nil {
+		t.Fatalf("failed to inflate gzipped CAR: %s", err)
+	}
+	defer r.Close() // nolint
+
+	// Load the CAR embedded in the test vector into the Blockstore.
+	_, err = car.LoadCar(bs, r)
+	if err != nil {
+		t.Fatalf("failed to load state tree car from test vector: %s", err)
+	}
+
+	// Create a new Driver.
+	driver := NewDriver(ctx)
+
+	// Apply every message.
+	for i, m := range vector.ApplyMessages {
+		msg, err := types.DecodeMessage(m.Bytes)
+		if err != nil {
+			t.Fatalf("failed to deserialize message: %s", err)
+		}
+
+		// add an epoch if one's set.
+		if m.Epoch != nil {
+			epoch = *m.Epoch
+		}
+
+		var ret *vm.ApplyRet
+		ret, root, err = driver.ExecuteMessage(msg, root, bs, epoch)
+		if err != nil {
+			t.Fatalf("fatal failure when executing message: %s", err)
+		}
+
+		receipt := vector.Post.Receipts[i]
+		if expected, actual := receipt.ExitCode, ret.ExitCode; expected != actual {
+			t.Errorf("exit code of msg %d did not match; expected: %s, got: %s", i, expected, actual)
+		}
+		if expected, actual := receipt.GasUsed, ret.GasUsed; expected != actual {
+			t.Errorf("gas used of msg %d did not match; expected: %d, got: %d", i, expected, actual)
+		}
+	}
+	if root != vector.Post.StateTree.RootCID {
+		t.Errorf("wrong post root cid; expected %vector , but got %vector", vector.Post.StateTree.RootCID, root)
+	}
+}

conformance/schema.go

@@ -0,0 +1,123 @@
+package conformance
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/runtime/exitcode"
+	"github.com/ipfs/go-cid"
+)
+
+// Class represents the type of test this instance is.
+type Class string
+
+var (
+	// ClassMessage tests the VM transition over a single message
+	ClassMessage Class = "message"
+	// ClassBlock tests the VM transition over a block of messages
+	ClassBlock Class = "block"
+	// ClassTipset tests the VM transition on a tipset update
+	ClassTipset Class = "tipset"
+	// ClassChain tests the VM transition across a chain segment
+	ClassChain Class = "chain"
+)
+
+// Selector provides a filter to indicate what implementations this test is relevant for
+type Selector string
+
+// Metadata provides information on the generation of this test case
+type Metadata struct {
+	ID      string         `json:"id"`
+	Version string         `json:"version,omitempty"`
+	Desc    string         `json:"description,omitempty"`
+	Comment string         `json:"comment,omitempty"`
+	Gen     GenerationData `json:"gen"`
+}
+
+// GenerationData tags the source of this test case
+type GenerationData struct {
+	Source  string `json:"source,omitempty"`
+	Version string `json:"version,omitempty"`
+}
+
+// StateTree represents a state tree within preconditions and postconditions.
+type StateTree struct {
+	RootCID cid.Cid `json:"root_cid"`
+}
+
+// Base64EncodedBytes is a base64-encoded binary value.
+type Base64EncodedBytes []byte
+
+// Preconditions contain a representation of VM state at the beginning of the test
+type Preconditions struct {
+	Epoch     abi.ChainEpoch `json:"epoch"`
+	StateTree *StateTree     `json:"state_tree"`
+}
+
+// Receipt represents a receipt to match against.
+type Receipt struct {
+	ExitCode    exitcode.ExitCode  `json:"exit_code"`
+	ReturnValue Base64EncodedBytes `json:"return"`
+	GasUsed     int64              `json:"gas_used"`
+}
+
+// Postconditions contain a representation of VM state at th end of the test
+type Postconditions struct {
+	StateTree *StateTree `json:"state_tree"`
+	Receipts  []*Receipt `json:"receipts"`
+}
+
+// MarshalJSON implements json.Marshal for Base64EncodedBytes
+func (beb Base64EncodedBytes) MarshalJSON() ([]byte, error) {
+	return json.Marshal(base64.StdEncoding.EncodeToString(beb))
+}
+
+// UnmarshalJSON implements json.Unmarshal for Base64EncodedBytes
+func (beb *Base64EncodedBytes) UnmarshalJSON(v []byte) error {
+	var s string
+	if err := json.Unmarshal(v, &s); err != nil {
+		return err
+	}
+
+	bytes, err := base64.StdEncoding.DecodeString(s)
+	if err != nil {
+		return err
+	}
+	*beb = bytes
+	return nil
+}
+
+// TestVector is a single test case
+type TestVector struct {
+	Class    `json:"class"`
+	Selector `json:"selector,omitempty"`
+	Meta     *Metadata `json:"_meta"`
+
+	// CAR binary data to be loaded into the test environment, usually a CAR
+	// containing multiple state trees, addressed by root CID from the relevant
+	// objects.
+	CAR Base64EncodedBytes `json:"car"`
+
+	Pre           *Preconditions  `json:"preconditions"`
+	ApplyMessages []Message       `json:"apply_messages"`
+	Post          *Postconditions `json:"postconditions"`
+}
+
+type Message struct {
+	Bytes Base64EncodedBytes `json:"bytes"`
+	Epoch *abi.ChainEpoch    `json:"epoch,omitempty"`
+}
+
+// Validate validates this test vector against the JSON schema, and applies
+// further validation rules that cannot be enforced through JSON Schema.
+func (tv *TestVector) Validate() error {
+	// TODO validate against JSON Schema.
+	if tv.Class == ClassMessage {
+		if len(tv.Post.Receipts) != len(tv.ApplyMessages) {
+			return fmt.Errorf("length of postcondition receipts must match length of messages to apply")
+		}
+	}
+	return nil
+}

conformance/stubs.go

@@ -0,0 +1,54 @@
+package conformance
+
+import (
+	"context"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/chain/state"
+	"github.com/filecoin-project/lotus/chain/vm"
+
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
+	"github.com/filecoin-project/specs-actors/actors/runtime"
+
+	cbor "github.com/ipfs/go-ipld-cbor"
+)
+
+type testRand struct{}
+
+var _ vm.Rand = (*testRand)(nil)
+
+func (r *testRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
+	return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes.
+}
+
+func (r *testRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
+	return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes.
+}
+
+type testSyscalls struct {
+	runtime.Syscalls
+}
+
+// TODO VerifySignature this will always succeed; but we want to be able to test failures too.
+func (fss *testSyscalls) VerifySignature(_ crypto.Signature, _ address.Address, _ []byte) error {
+	return nil
+}
+
+// TODO VerifySeal this will always succeed; but we want to be able to test failures too.
+func (fss *testSyscalls) VerifySeal(_ abi.SealVerifyInfo) error {
+	return nil
+}
+
+// TODO VerifyPoSt this will always succeed; but we want to be able to test failures too.
+func (fss *testSyscalls) VerifyPoSt(_ abi.WindowPoStVerifyInfo) error {
+	return nil
+}
+
+func mkFakedSigSyscalls(base vm.SyscallBuilder) vm.SyscallBuilder {
+	return func(ctx context.Context, cstate *state.StateTree, cst cbor.IpldStore) runtime.Syscalls {
+		return &testSyscalls{
+			base(ctx, cstate, cst),
+		}
+	}
+}

extern/conformance-vectors

@@ -0,0 +1 @@
+Subproject commit 6014f0705b91012b37ca5009b742b71a2423d529