diff --git a/.gitmodules b/.gitmodules index 709a28003..6cf1ccc72 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,3 +5,7 @@ [submodule "extern/serialization-vectors"] path = extern/serialization-vectors url = https://github.com/filecoin-project/serialization-vectors +[submodule "extern/conformance-vectors"] + path = extern/conformance-vectors + url = https://github.com/filecoin-project/conformance-vectors.git + branch = initial diff --git a/conformance/driver.go b/conformance/driver.go new file mode 100644 index 000000000..e25e5b9f2 --- /dev/null +++ b/conformance/driver.go @@ -0,0 +1,59 @@ +package conformance + +import ( + "context" + + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/puppet" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/lib/blockstore" + "github.com/filecoin-project/sector-storage/ffiwrapper" +) + +var ( + // BaseFee to use in the VM. + // TODO make parametrisable through vector. + BaseFee = abi.NewTokenAmount(100) +) + +type Driver struct { + ctx context.Context +} + +func NewDriver(ctx context.Context) *Driver { + return &Driver{ctx: ctx} +} + +// ExecuteMessage executes a conformance test vector message in a temporary VM. +func (d *Driver) ExecuteMessage(msg *types.Message, preroot cid.Cid, bs blockstore.Blockstore, epoch abi.ChainEpoch) (*vm.ApplyRet, cid.Cid, error) { + vmOpts := &vm.VMOpts{ + StateBase: preroot, + Epoch: epoch, + Rand: &testRand{}, // TODO always succeeds; need more flexibility. + Bstore: bs, + Syscalls: mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility. + CircSupplyCalc: nil, + BaseFee: BaseFee, + } + + lvm, err := vm.NewVM(vmOpts) + if err != nil { + return nil, cid.Undef, err + } + + // add support for the puppet actor. + invoker := vm.NewInvoker() + invoker.Register(puppet.PuppetActorCodeID, puppet.Actor{}, puppet.State{}) + lvm.SetInvoker(invoker) + + ret, err := lvm.ApplyMessage(d.ctx, msg) + if err != nil { + return nil, cid.Undef, err + } + + root, err := lvm.Flush(d.ctx) + return ret, root, err +} diff --git a/conformance/runner_test.go b/conformance/runner_test.go new file mode 100644 index 000000000..34d1a3c69 --- /dev/null +++ b/conformance/runner_test.go @@ -0,0 +1,180 @@ +package conformance + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "flag" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/lib/blockstore" + + "github.com/ipld/go-car" +) + +const ( + // defaultRoot is default root at where the message vectors are hosted. + // + // You can run this test with the -vectors.root flag to execute + // a custom corpus. + defaultRoot = "../extern/conformance-vectors" +) + +var ( + // root is the effective root path, taken from the `-vectors.root` CLI flag, + // falling back to defaultRoot if not provided. + root string + // ignore is a set of paths relative to root to skip. + ignore = map[string]struct{}{ + ".git": {}, + "schema.json": {}, + } +) + +func init() { + // read the alternative root from the -vectors.root CLI flag. + flag.StringVar(&root, "vectors.root", defaultRoot, "root directory containing test vectors") +} + +// TestConformance is the entrypoint test that runs all test vectors found +// in the root directory. +// +// It locates all json files via a recursive walk, skipping over the ignore set, +// as well as files beginning with _. It parses each file as a test vector, and +// runs it via the Driver. +func TestConformance(t *testing.T) { + var vectors []string + err := filepath.Walk(root+"/", func(path string, info os.FileInfo, err error) error { + if err != nil { + t.Fatal(err) + } + + filename := filepath.Base(path) + rel, err := filepath.Rel(root, path) + if err != nil { + t.Fatal(err) + } + + if _, ok := ignore[rel]; ok { + // skip over using the right error. + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + if info.IsDir() { + // dive into directories. + return nil + } + if filepath.Ext(path) != ".json" { + // skip if not .json. + return nil + } + if ignored := strings.HasPrefix(filename, "_"); ignored { + // ignore files starting with _. + t.Logf("ignoring: %s", rel) + return nil + } + vectors = append(vectors, rel) + return nil + }) + + if err != nil { + t.Fatal(err) + } + + if len(vectors) == 0 { + t.Fatalf("no test vectors found") + } + + // Run a test for each vector. + for _, v := range vectors { + v := v + t.Run(v, func(t *testing.T) { + path := filepath.Join(root, v) + raw, err := ioutil.ReadFile(path) + if err != nil { + t.Fatalf("failed to read test raw file: %s", path) + } + + var vector TestVector + err = json.Unmarshal(raw, &vector) + if err != nil { + t.Fatalf("failed to parse test raw: %s", err) + } + + // dispatch the execution depending on the vector class. + switch vector.Class { + case "message": + executeMessageVector(t, &vector) + default: + t.Fatalf("test vector class not supported: %s", vector.Class) + } + }) + } +} + +// executeMessageVector executes a message-class test vector. +func executeMessageVector(t *testing.T, vector *TestVector) { + var ( + ctx = context.Background() + epoch = vector.Pre.Epoch + root = vector.Pre.StateTree.RootCID + ) + + bs := blockstore.NewTemporary() + + // Read the base64-encoded CAR from the vector, and inflate the gzip. + buf := bytes.NewReader(vector.CAR) + r, err := gzip.NewReader(buf) + if err != nil { + t.Fatalf("failed to inflate gzipped CAR: %s", err) + } + defer r.Close() // nolint + + // Load the CAR embedded in the test vector into the Blockstore. + _, err = car.LoadCar(bs, r) + if err != nil { + t.Fatalf("failed to load state tree car from test vector: %s", err) + } + + // Create a new Driver. + driver := NewDriver(ctx) + + // Apply every message. + for i, m := range vector.ApplyMessages { + msg, err := types.DecodeMessage(m.Bytes) + if err != nil { + t.Fatalf("failed to deserialize message: %s", err) + } + + // add an epoch if one's set. + if m.Epoch != nil { + epoch = *m.Epoch + } + + var ret *vm.ApplyRet + ret, root, err = driver.ExecuteMessage(msg, root, bs, epoch) + if err != nil { + t.Fatalf("fatal failure when executing message: %s", err) + } + + receipt := vector.Post.Receipts[i] + if expected, actual := receipt.ExitCode, ret.ExitCode; expected != actual { + t.Errorf("exit code of msg %d did not match; expected: %s, got: %s", i, expected, actual) + } + if expected, actual := receipt.GasUsed, ret.GasUsed; expected != actual { + t.Errorf("gas used of msg %d did not match; expected: %d, got: %d", i, expected, actual) + } + } + if root != vector.Post.StateTree.RootCID { + t.Errorf("wrong post root cid; expected %vector , but got %vector", vector.Post.StateTree.RootCID, root) + } +} diff --git a/conformance/schema.go b/conformance/schema.go new file mode 100644 index 000000000..6c44f7ea1 --- /dev/null +++ b/conformance/schema.go @@ -0,0 +1,123 @@ +package conformance + +import ( + "encoding/base64" + "encoding/json" + "fmt" + + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/runtime/exitcode" + "github.com/ipfs/go-cid" +) + +// Class represents the type of test this instance is. +type Class string + +var ( + // ClassMessage tests the VM transition over a single message + ClassMessage Class = "message" + // ClassBlock tests the VM transition over a block of messages + ClassBlock Class = "block" + // ClassTipset tests the VM transition on a tipset update + ClassTipset Class = "tipset" + // ClassChain tests the VM transition across a chain segment + ClassChain Class = "chain" +) + +// Selector provides a filter to indicate what implementations this test is relevant for +type Selector string + +// Metadata provides information on the generation of this test case +type Metadata struct { + ID string `json:"id"` + Version string `json:"version,omitempty"` + Desc string `json:"description,omitempty"` + Comment string `json:"comment,omitempty"` + Gen GenerationData `json:"gen"` +} + +// GenerationData tags the source of this test case +type GenerationData struct { + Source string `json:"source,omitempty"` + Version string `json:"version,omitempty"` +} + +// StateTree represents a state tree within preconditions and postconditions. +type StateTree struct { + RootCID cid.Cid `json:"root_cid"` +} + +// Base64EncodedBytes is a base64-encoded binary value. +type Base64EncodedBytes []byte + +// Preconditions contain a representation of VM state at the beginning of the test +type Preconditions struct { + Epoch abi.ChainEpoch `json:"epoch"` + StateTree *StateTree `json:"state_tree"` +} + +// Receipt represents a receipt to match against. +type Receipt struct { + ExitCode exitcode.ExitCode `json:"exit_code"` + ReturnValue Base64EncodedBytes `json:"return"` + GasUsed int64 `json:"gas_used"` +} + +// Postconditions contain a representation of VM state at th end of the test +type Postconditions struct { + StateTree *StateTree `json:"state_tree"` + Receipts []*Receipt `json:"receipts"` +} + +// MarshalJSON implements json.Marshal for Base64EncodedBytes +func (beb Base64EncodedBytes) MarshalJSON() ([]byte, error) { + return json.Marshal(base64.StdEncoding.EncodeToString(beb)) +} + +// UnmarshalJSON implements json.Unmarshal for Base64EncodedBytes +func (beb *Base64EncodedBytes) UnmarshalJSON(v []byte) error { + var s string + if err := json.Unmarshal(v, &s); err != nil { + return err + } + + bytes, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return err + } + *beb = bytes + return nil +} + +// TestVector is a single test case +type TestVector struct { + Class `json:"class"` + Selector `json:"selector,omitempty"` + Meta *Metadata `json:"_meta"` + + // CAR binary data to be loaded into the test environment, usually a CAR + // containing multiple state trees, addressed by root CID from the relevant + // objects. + CAR Base64EncodedBytes `json:"car"` + + Pre *Preconditions `json:"preconditions"` + ApplyMessages []Message `json:"apply_messages"` + Post *Postconditions `json:"postconditions"` +} + +type Message struct { + Bytes Base64EncodedBytes `json:"bytes"` + Epoch *abi.ChainEpoch `json:"epoch,omitempty"` +} + +// Validate validates this test vector against the JSON schema, and applies +// further validation rules that cannot be enforced through JSON Schema. +func (tv *TestVector) Validate() error { + // TODO validate against JSON Schema. + if tv.Class == ClassMessage { + if len(tv.Post.Receipts) != len(tv.ApplyMessages) { + return fmt.Errorf("length of postcondition receipts must match length of messages to apply") + } + } + return nil +} diff --git a/conformance/stubs.go b/conformance/stubs.go new file mode 100644 index 000000000..2fd1e7b64 --- /dev/null +++ b/conformance/stubs.go @@ -0,0 +1,54 @@ +package conformance + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/state" + "github.com/filecoin-project/lotus/chain/vm" + + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/crypto" + "github.com/filecoin-project/specs-actors/actors/runtime" + + cbor "github.com/ipfs/go-ipld-cbor" +) + +type testRand struct{} + +var _ vm.Rand = (*testRand)(nil) + +func (r *testRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { + return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes. +} + +func (r *testRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { + return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes. +} + +type testSyscalls struct { + runtime.Syscalls +} + +// TODO VerifySignature this will always succeed; but we want to be able to test failures too. +func (fss *testSyscalls) VerifySignature(_ crypto.Signature, _ address.Address, _ []byte) error { + return nil +} + +// TODO VerifySeal this will always succeed; but we want to be able to test failures too. +func (fss *testSyscalls) VerifySeal(_ abi.SealVerifyInfo) error { + return nil +} + +// TODO VerifyPoSt this will always succeed; but we want to be able to test failures too. +func (fss *testSyscalls) VerifyPoSt(_ abi.WindowPoStVerifyInfo) error { + return nil +} + +func mkFakedSigSyscalls(base vm.SyscallBuilder) vm.SyscallBuilder { + return func(ctx context.Context, cstate *state.StateTree, cst cbor.IpldStore) runtime.Syscalls { + return &testSyscalls{ + base(ctx, cstate, cst), + } + } +} diff --git a/extern/conformance-vectors b/extern/conformance-vectors new file mode 160000 index 000000000..6014f0705 --- /dev/null +++ b/extern/conformance-vectors @@ -0,0 +1 @@ +Subproject commit 6014f0705b91012b37ca5009b742b71a2423d529