cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

unified tvx tool; end-to-end MVP for extracting+running msg-class vectors (#177)

Browse Source 
Co-authored-by: Will Scott <will@cypherpunk.email>
Co-authored-by: Anton Evangelatov <anton.evangelatov@gmail.com>
This commit is contained in:
Raúl Kripalani 2020-08-05 13:20:13 +01:00 committed by GitHub
parent 62a9982b76
commit b7e3b4ff77
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 3265 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,4 +1,4 @@
lotus
venv/ venv/
__pycache__/ __pycache__/
.ipynb_checkpoints/ .ipynb_checkpoints/
tvx/tvx

149
corpus/schema.json Normal file
View File

@ -0,0 +1,149 @@
{
"$id": "https://filecoin.io/oni/schemas/test-vector.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "a filecoin VM test vector",
"type": "object",
"definitions": {
"hex": {
"title": "hex value",
"description": "a hex value prefixed with 0x, and accepting only lowercase characters; 0x represents an empty byte array",
"type": "string",
"pattern": "0x[0-9a-f]*",
"examples": [
"0xa1b2c3",
"0x"
]
},
"meta": {
"title": "metadata",
"description": "metadata about this test vector, such as its id, version, data about its generation, etc.",
"type": "object",
"additionalProperties": false,
"properties": {
"id": {
"type": "string"
},
"version": {
"type": "string"
},
"gen": {
"title": "generation metadata",
"description": "metadata about how this test vector was generated",
"type": "object",
"additionalProperties": false,
"properties": {
"source": {
"type": "string",
"examples": [
"lotus",
"dsl"
]
},
"version": {
"type": "string",
"examples": [
"0.4.1+git.27d74337+api0.8.1"
]
}
}
}
}
},
"preconditions": {
"title": "execution preconditions",
"description": "preconditions that need to be applied and satisfied before this test vector can be executed",
"type": "object",
"additionalProperties": false,
"properties": {
"state_tree": {
"title": "state tree to seed",
"description": "state tree to seed before applying this test vector; mapping of actor addresses => serialized state",
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/hex"
},
"examples": [
{
"t01": "0x0123456789abcdef",
"t02": "0x0123456789abcdef",
"t03": "0x0123456789abcdef"
}
]
}
}
},
"postconditions": {
"title": "execution preconditions",
"description": "postconditions that need to be satisfied after execution for this test vector to pass",
"type": "object",
"additionalProperties": false,
"properties": {
"state_tree": {
"title": "state tree postconditions",
"description": "state tree postconditions that must be true for this test vector to pass",
"type": "object",
"properties": {
"car_bytes": {
"title": "the hex-encoded CAR containing the full state tree",
"description": "the hex-encoded CAR containing the full state tree, for debugging/diffing purposes",
"$ref": "#/definitions/hex"
}
}
}
}
},
"apply_message": {
"title": "message to apply, hex-encoded",
"$ref": "#/definitions/hex"
}
},
"required": [
"class"
],
"properties": {
"class": {
"title": "test vector class",
"description": "test vector class; depending on the value, the apply_* property to provide (and its schema) will vary; the relevant apply property is apply_[class]",
"type": "string",
"enum": [
"message",
"block",
"tipset",
"chain"
]
},
"selector": {
"title": "selector the driver can use to determine if this test vector applies",
"description": "format TBD",
"type": "string"
},
"_meta": {
"$ref": "#/definitions/meta"
},
"preconditions": {
"$ref": "#/definitions/preconditions"
},
"postconditions": {
"$ref": "#/definitions/postconditions"
}
},
"allOf": [
{
"if": {
"properties": {
"class": {
"const": "message"
}
}
},
"then": {
"required": ["apply_message"],
"properties": {
"apply_message": {
"$ref": "#/definitions/apply_message"
}
}
}
}
]
}

29
corpus/single_message.json Normal file
View File

@ -0,0 +1,29 @@
{
"class": "message",
"selector": "<some predicate for the driver to determine if this vector applies to the implementation, e.g. to deal with protocol upgrades>",
"_meta": {
"id": "test_vector_id",
"version": "version_id",
"gen": {
"source": "lotus",
"version": "0.4.1+git.27d74337+api0.8.1"
}
},
"preconditions": {
"state_tree": {
"t01": "0x0123456789abcdef",
"t02": "0x0123456789abcdef",
"t03": "0x0123456789abcdef"
}
},
"apply_message": "0x89004300ba20583103b85358cc1c968d826c0a0efcde99b6e4db138edfc051f188a4471cfc23ccf7884adbbac912e45f7363774bf16f83ca8f1902b44800505e1442e66b744200011a09d225c80758c68219016f58c099a447a8c80988e1c061dd3a8dd674033c3a6083c4b55cf035e6e8de3baf09436ad00adf6eb14d58ae6691d22ce82255874047e7aa0b9a86bbf424afc6d89dc29f7b253dd04c88a625b86345f05aa0bdee503971c1e9735ed5b81c31ad41a1f10b387d96151d1cbd0c4286267daf04087fd8d5272f870994c3732f3fc4f54287a485d6c8dafb6056df3fc9233f849027b4c1272850435126fef65a8363768a858c0cc5f9d9114ff378ad50cab25ab0a2f685ca63d9f1f7727da4d06db5b80b5f",
"postconditions": {
"state_tree": {
"car_bytes": "0x890043009920583103b5b1fab6769fcb464146d192af785915d8e1dacff264b2be3bcb7bf4064a77cb3b13e27063ea4825dc48cd0a7df77e211916e748002740f0c493a2824200011a09d9c4dc0758c68219091c58c083e7c59e748e070b382a813f4ffe8ead2a090469aaa37f0664b82c092bc78b43fac300335cd2d6e4a8c71ce138003c0fa8ee1707b41aa87f35c811613d59ac53449aef8263aa51c876f2e67c185118ec9628f73442f58e093909ce952e431e770ac693cdbe99f2f9404ad3ca754196ab443c6c66afa41310e705e453496bdcedf0f271529dc7374021d7b80d343397f996d06bf11e753a49eaed7b3ce80a8e9bbaa48bf83223527bd8a2933f002949fdb8058103336a4a979375d5e963cdd7b1"
}
}
}

112
tvx/examine.go Normal file
View File

@ -0,0 +1,112 @@
package main
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/specs-actors/actors/builtin"
init_ "github.com/filecoin-project/specs-actors/actors/builtin/init"
"github.com/filecoin-project/specs-actors/actors/util/adt"
"github.com/filecoin-project/oni/tvx/state"
)
func trimQuotes(s string) string {
if len(s) >= 2 {
if s[0] == '"' && s[len(s)-1] == '"' {
return s[1 : len(s)-1]
}
}
return s
}
var examineFlags struct {
file string
pre bool
post bool
}
var examineCmd = &cli.Command{
Name: "examine",
Description: "examine an exported state root as represented in a test vector",
Action: runExamineCmd,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "file",
Usage: "test vector file",
Required: true,
Destination: &examineFlags.file,
},
&cli.BoolFlag{
Name: "pre",
Usage: "examine the precondition state tree",
Destination: &examineFlags.pre,
},
&cli.BoolFlag{
Name: "post",
Usage: "examine the postcondition state tree",
Destination: &examineFlags.post,
},
},
}
func runExamineCmd(_ *cli.Context) error {
file, err := os.Open(examineFlags.file)
if err != nil {
return err
}
var tv TestVector
if err := json.NewDecoder(file).Decode(&tv); err != nil {
return err
}
examine := func(encoded []byte) error {
tree, err := state.RecoverStateTree(context.TODO(), encoded)
if err != nil {
return err
}
initActor, err := tree.GetActor(builtin.InitActorAddr)
if err != nil {
return fmt.Errorf("cannot recover init actor: %w", err)
}
var ias init_.State
if err := tree.Store.Get(context.TODO(), initActor.Head, &ias); err != nil {
return err
}
adtStore := adt.WrapStore(context.TODO(), tree.Store)
m, err := adt.AsMap(adtStore, ias.AddressMap)
if err != nil {
return err
}
actors, err := m.CollectKeys()
for _, actor := range actors {
fmt.Printf("%s\n", actor)
}
return nil
}
if examineFlags.pre {
log.Print("examining precondition tree")
if err := examine(tv.Pre.StateTree.CAR); err != nil {
return err
}
}
if examineFlags.post {
log.Print("examining postcondition tree")
if err := examine(tv.Post.StateTree.CAR); err != nil {
return err
}
}
return nil
}

96
tvx/exec_lotus.go Normal file
View File

@ -0,0 +1,96 @@
package main
import (
"bytes"
"compress/gzip"
"context"
"encoding/json"
"fmt"
"os"
"github.com/davecgh/go-spew/spew"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/ipld/go-car"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/oni/tvx/lotus"
)
var execLotusFlags struct {
file string
}
var execLotusCmd = &cli.Command{
Name: "exec-lotus",
Description: "execute a test vector against Lotus",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "file",
Usage: "input file",
Required: true,
Destination: &execLotusFlags.file,
},
},
Action: runExecLotus,
}
func runExecLotus(_ *cli.Context) error {
if execLotusFlags.file == "" {
return fmt.Errorf("test vector file cannot be empty")
}
file, err := os.Open(execLotusFlags.file)
if err != nil {
return fmt.Errorf("failed to open test vector: %w", err)
}
var (
dec = json.NewDecoder(file)
tv TestVector
)
if err = dec.Decode(&tv); err != nil {
return fmt.Errorf("failed to decode test vector: %w", err)
}
switch tv.Class {
case "message":
var (
ctx = context.Background()
epoch = tv.Pre.Epoch
)
bs := blockstore.NewTemporary()
buf := bytes.NewReader(tv.Pre.StateTree.CAR)
gr, err := gzip.NewReader(buf)
if err != nil {
return err
}
defer gr.Close()
header, err := car.LoadCar(bs, gr)
if err != nil {
return fmt.Errorf("failed to load state tree car from test vector: %w", err)
}
fmt.Println("roots: ", header.Roots)
fmt.Println("decoding message")
msg, err := types.DecodeMessage(tv.ApplyMessage)
if err != nil {
return err
}
driver := lotus.NewDriver(ctx)
fmt.Println("executing message")
spew.Dump(driver.ExecuteMessage(msg, header.Roots[0], bs, epoch))
return nil
default:
return fmt.Errorf("test vector class not supported")
}
}

204
tvx/extract_msg.go Normal file
View File

@ -0,0 +1,204 @@
package main
import (
"bytes"
"compress/gzip"
"context"
"encoding/json"
"fmt"
"os"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/ipfs/go-cid"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/oni/tvx/lotus"
"github.com/filecoin-project/oni/tvx/state"
)
var extractMsgFlags struct {
cid string
file string
}
var extractMsgCmd = &cli.Command{
Name: "extract-message",
Description: "generate a message-class test vector by extracting it from a network",
Action: runExtractMsg,
Flags: []cli.Flag{
&apiFlag,
&cli.StringFlag{
Name: "cid",
Usage: "message CID to generate test vector from",
Required: true,
Destination: &extractMsgFlags.cid,
},
&cli.StringFlag{
Name: "file",
Usage: "output file",
Required: true,
Destination: &extractMsgFlags.file,
},
},
}
func runExtractMsg(c *cli.Context) error {
// LOTUS_DISABLE_VM_BUF disables what's called "VM state tree buffering",
// which stashes write operations in a BufferedBlockstore
// (https://github.com/filecoin-project/lotus/blob/b7a4dbb07fd8332b4492313a617e3458f8003b2a/lib/bufbstore/buf_bstore.go#L21)
// such that they're not written until the VM is actually flushed.
//
// For some reason, the standard behaviour was not working for me (raulk),
// and disabling it (such that the state transformations are written immediately
// to the blockstore) worked.
_ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea")
ctx := context.Background()
// get the output file.
if extractMsgFlags.file == "" {
return fmt.Errorf("output file required")
}
mid, err := cid.Decode(extractMsgFlags.cid)
if err != nil {
return err
}
// Make the client.
api, err := makeClient(c)
if err != nil {
return err
}
// locate the message.
msgInfo, err := api.StateSearchMsg(ctx, mid)
if err != nil {
return fmt.Errorf("failed to locate message: %w", err)
}
// Extract the serialized message.
msg, err := api.ChainGetMessage(ctx, mid)
if err != nil {
return err
}
// create a read through store that uses ChainGetObject to fetch unknown CIDs.
pst := state.NewProxyingStore(ctx, api)
g := state.NewSurgeon(ctx, api, pst)
// Get actors accessed by message.
retain, err := g.GetAccessedActors(ctx, api, mid)
if err != nil {
return err
}
retain = append(retain, builtin.RewardActorAddr)
fmt.Println("accessed actors:")
for _, k := range retain {
fmt.Println("\t", k.String())
}
// get the tipset on which this message was mined.
ts, err := api.ChainGetTipSet(ctx, msgInfo.TipSet)
if err != nil {
return err
}
// get the previous tipset, on top of which the message was executed.
prevTs, err := api.ChainGetTipSet(ctx, ts.Parents())
if err != nil {
return err
}
fmt.Println("getting the _before_ filtered state tree")
preroot, err := g.GetMaskedStateTree(prevTs.Parents(), retain)
if err != nil {
return err
}
driver := lotus.NewDriver(ctx)
_, postroot, err := driver.ExecuteMessage(msg, preroot, pst.Blockstore, ts.Height())
if err != nil {
return fmt.Errorf("failed to execute message: %w", err)
}
msgBytes, err := msg.Serialize()
if err != nil {
return err
}
getZippedCAR := func(root cid.Cid) ([]byte, error) {
out := new(bytes.Buffer)
gw := gzip.NewWriter(out)
if err := g.WriteCAR(gw, root); err != nil {
return nil, err
}
if err = gw.Flush(); err != nil {
return nil, err
}
if err = gw.Close(); err != nil {
return nil, err
}
return out.Bytes(), nil
}
pretree, err := getZippedCAR(preroot)
if err != nil {
return err
}
posttree, err := getZippedCAR(postroot)
if err != nil {
return err
}
version, err := api.Version(ctx)
if err != nil {
return err
}
// Write out the test vector.
vector := TestVector{
Class: ClassMessage,
Selector: "",
Meta: &Metadata{
ID: "TK",
Version: "TK",
Gen: GenerationData{
Source: "TK",
Version: version.String(),
},
},
Pre: &Preconditions{
Epoch: ts.Height(),
StateTree: &StateTree{
CAR: pretree,
},
},
ApplyMessage: msgBytes,
Post: &Postconditions{
StateTree: &StateTree{
CAR: posttree,
},
},
}
file, err := os.Create(extractMsgFlags.file)
if err != nil {
return err
}
defer file.Close()
enc := json.NewEncoder(file)
enc.SetIndent("", " ")
if err := enc.Encode(&vector); err != nil {
return err
}
return nil
}

29
tvx/go.mod Normal file
View File

@ -0,0 +1,29 @@
module github.com/filecoin-project/oni/tvx
go 1.14
require (
github.com/davecgh/go-spew v1.1.1
github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef
github.com/filecoin-project/lotus v0.4.3-0.20200801235920-43491cb7edfd
github.com/filecoin-project/sector-storage v0.0.0-20200730203805-7153e1dd05b5
github.com/filecoin-project/specs-actors v0.8.6
github.com/ipfs/go-block-format v0.0.2
github.com/ipfs/go-blockservice v0.1.4-0.20200624145336-a978cec6e834
github.com/ipfs/go-cid v0.0.7
github.com/ipfs/go-datastore v0.4.4
github.com/ipfs/go-hamt-ipld v0.1.1
github.com/ipfs/go-ipfs-exchange-interface v0.0.1
github.com/ipfs/go-ipfs-exchange-offline v0.0.1
github.com/ipfs/go-ipld-cbor v0.0.5-0.20200428170625-a0bd04d3cbdf
github.com/ipfs/go-ipld-format v0.2.0
github.com/ipfs/go-merkledag v0.3.1
github.com/ipld/go-car v0.1.1-0.20200526133713-1c7508d55aae
github.com/multiformats/go-multiaddr v0.2.2
github.com/multiformats/go-multiaddr-net v0.1.5
github.com/multiformats/go-multihash v0.0.14
github.com/urfave/cli/v2 v2.2.0
github.com/whyrusleeping/cbor-gen v0.0.0-20200723185710-6a3894a6352b
)
replace github.com/filecoin-project/filecoin-ffi => ../extra/filecoin-ffi

1794
tvx/go.sum Normal file
View File

File diff suppressed because it is too large Load Diff

58
tvx/list_accessed.go Normal file
View File

@ -0,0 +1,58 @@
package main
import (
"context"
"fmt"
"github.com/ipfs/go-cid"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/oni/tvx/state"
)
var listAccessedFlags struct {
cid string
}
var listAccessedCmd = &cli.Command{
Name: "list-accessed",
Description: "extract actors accessed during the execution of a message",
Action: runListAccessed,
Flags: []cli.Flag{
&apiFlag,
&cli.StringFlag{
Name: "cid",
Usage: "message CID",
Required: true,
Destination: &listAccessedFlags.cid,
},
},
}
func runListAccessed(c *cli.Context) error {
ctx := context.Background()
node, err := makeClient(c)
if err != nil {
return err
}
mid, err := cid.Decode(listAccessedFlags.cid)
if err != nil {
return err
}
rtst := state.NewProxyingStore(ctx, node)
sg := state.NewSurgeon(ctx, node, rtst)
actors, err := sg.GetAccessedActors(context.TODO(), node, mid)
if err != nil {
return err
}
for k := range actors {
fmt.Printf("%v\n", k)
}
return nil
}

57
tvx/lotus/driver.go Normal file
View File

@ -0,0 +1,57 @@
package lotus
import (
"context"
"fmt"
"github.com/filecoin-project/lotus/chain/state"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/sector-storage/ffiwrapper"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/ipfs/go-cid"
cbor "github.com/ipfs/go-ipld-cbor"
)
type Driver struct {
ctx context.Context
}
func NewDriver(ctx context.Context) *Driver {
return &Driver{ctx: ctx}
}
func (d *Driver) ExecuteMessage(msg *types.Message, preroot cid.Cid, bs blockstore.Blockstore, epoch abi.ChainEpoch) (*vm.ApplyRet, cid.Cid, error) {
fmt.Println("execution sanity check")
cst := cbor.NewCborStore(bs)
st, err := state.LoadStateTree(cst, preroot)
if err != nil {
return nil, cid.Undef, err
}
actor, err := st.GetActor(msg.From)
if err != nil {
return nil, cid.Undef, err
}
fmt.Println("from actor found: ", actor)
fmt.Println("creating vm")
lvm, err := vm.NewVM(preroot, epoch, &vmRand{}, bs, mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), nil)
if err != nil {
return nil, cid.Undef, err
}
fmt.Println("applying message")
ret, err := lvm.ApplyMessage(d.ctx, msg)
if err != nil {
return nil, cid.Undef, err
}
fmt.Printf("applied message: %+v\n", ret)
fmt.Println("flushing")
root, err := lvm.Flush(d.ctx)
return ret, root, err
}

38
tvx/lotus/stubs.go Normal file
View File

@ -0,0 +1,38 @@
package lotus
import (
"context"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/chain/state"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/crypto"
"github.com/filecoin-project/specs-actors/actors/runtime"
cbor "github.com/ipfs/go-ipld-cbor"
)
type vmRand struct {
}
func (*vmRand) GetRandomness(ctx context.Context, dst crypto.DomainSeparationTag, h abi.ChainEpoch, input []byte) ([]byte, error) {
return []byte("i_am_random"), nil
}
type fakedSigSyscalls struct {
runtime.Syscalls
}
func (fss *fakedSigSyscalls) VerifySignature(_ crypto.Signature, _ address.Address, plaintext []byte) error {
return nil
}
func mkFakedSigSyscalls(base vm.SyscallBuilder) vm.SyscallBuilder {
return func(ctx context.Context, cstate *state.StateTree, cst cbor.IpldStore) runtime.Syscalls {
return &fakedSigSyscalls{
base(ctx, cstate, cst),
}
}
}

80
tvx/main.go Normal file
View File

@ -0,0 +1,80 @@
package main
import (
"fmt"
"log"
"net/http"
"os"
"sort"
"strings"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/client"
"github.com/multiformats/go-multiaddr"
manet "github.com/multiformats/go-multiaddr-net"
"github.com/urfave/cli/v2"
)
var apiFlag = cli.StringFlag{
Name: "api",
Usage: "api endpoint, formatted as token:multiaddr",
Value: "",
EnvVars: []string{"FULLNODE_API_INFO"},
}
func main() {
app := &cli.App{
Name: "tvx",
Description: "a toolbox for managing test vectors",
Usage: "a toolbox for managing test vectors",
Commands: []*cli.Command{
deltaCmd,
listAccessedCmd,
extractMsgCmd,
execLotusCmd,
examineCmd,
},
}
sort.Sort(cli.CommandsByName(app.Commands))
for _, c := range app.Commands {
sort.Sort(cli.FlagsByName(c.Flags))
}
err := app.Run(os.Args)
if err != nil {
log.Fatal(err)
}
}
func makeClient(c *cli.Context) (api.FullNode, error) {
api := c.String(apiFlag.Name)
sp := strings.SplitN(api, ":", 2)
if len(sp) != 2 {
return nil, fmt.Errorf("invalid api value, missing token or address: %s", api)
}
// TODO: discovery from filesystem
token := sp[0]
ma, err := multiaddr.NewMultiaddr(sp[1])
if err != nil {
return nil, fmt.Errorf("could not parse provided multiaddr: %w", err)
}
_, dialAddr, err := manet.DialArgs(ma)
if err != nil {
return nil, fmt.Errorf("invalid api multiAddr: %w", err)
}
addr := "ws://" + dialAddr + "/rpc/v0"
headers := http.Header{}
if len(token) != 0 {
headers.Add("Authorization", "Bearer "+token)
}
node, _, err := client.NewFullNodeRPC(addr, headers)
if err != nil {
return nil, fmt.Errorf("could not connect to api: %w", err)
}
return node, nil
}

90
tvx/schema.go Normal file
View File

@ -0,0 +1,90 @@
package main
import (
"encoding/hex"
"encoding/json"
"github.com/filecoin-project/specs-actors/actors/abi"
)
// Class represents the type of test this instance is.
type Class string
var (
// ClassMessage tests the VM transition over a single message
ClassMessage Class = "message"
// ClassBlock tests the VM transition over a block of messages
ClassBlock Class = "block"
// ClassTipset tests the VM transition on a tipset update
ClassTipset Class = "tipset"
// ClassChain tests the VM transition across a chain segment
ClassChain Class = "chain"
)
// Selector provides a filter to indicate what implementations this test is relevant for
type Selector string
// Metadata provides information on the generation of this test case
type Metadata struct {
ID string `json:"id"`
Version string `json:"version"`
Gen GenerationData `json:"gen"`
}
// GenerationData tags the source of this test case
type GenerationData struct {
Source string `json:"source"`
Version string `json:"version"`
}
// StateTree represents a state tree within preconditions and postconditions.
type StateTree struct {
// CAR is the car representation of a state tree
CAR HexEncodedBytes `json:"car_hex"`
}
// HexEncodedBytes is a hex-encoded binary value.
//
// TODO may switch to base64 or base85 for efficiency.
type HexEncodedBytes []byte
// Preconditions contain a representation of VM state at the beginning of the test
type Preconditions struct {
Epoch abi.ChainEpoch `json:"epoch"`
StateTree *StateTree `json:"state_tree"`
}
// Postconditions contain a representation of VM state at th end of the test
type Postconditions struct {
StateTree *StateTree `json:"state_tree"`
}
// MarshalJSON implements json.Marshal for HexEncodedBytes
func (heb HexEncodedBytes) MarshalJSON() ([]byte, error) {
return json.Marshal(hex.EncodeToString(heb))
}
// UnmarshalJSON implements json.Unmarshal for HexEncodedBytes
func (heb *HexEncodedBytes) UnmarshalJSON(v []byte) error {
var s string
if err := json.Unmarshal(v, &s); err != nil {
return err
}
bytes, err := hex.DecodeString(s)
if err != nil {
return err
}
*heb = bytes
return nil
}
// TestVector is a single test case
type TestVector struct {
Class `json:"class"`
Selector `json:"selector"`
Meta *Metadata `json:"_meta"`
Pre *Preconditions `json:"preconditions"`
ApplyMessage HexEncodedBytes `json:"apply_message"`
Post *Postconditions `json:"postconditions"`
}

55
tvx/state/serialize.go Normal file
View File

@ -0,0 +1,55 @@
package state
import (
"bytes"
"compress/gzip"
"context"
"fmt"
"github.com/filecoin-project/lotus/chain/state"
bs "github.com/filecoin-project/lotus/lib/blockstore"
"github.com/ipfs/go-hamt-ipld"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/ipfs/go-ipld-format"
"github.com/ipld/go-car"
)
// RecoverStateTree parses a car encoding of a state tree back to a structured format
func RecoverStateTree(ctx context.Context, raw []byte) (*state.StateTree, error) {
buf := bytes.NewBuffer(raw)
store := bs.NewTemporary()
gr, err := gzip.NewReader(buf)
if err != nil {
return nil, err
}
defer gr.Close()
ch, err := car.LoadCar(store, gr)
if err != nil {
return nil, err
}
if len(ch.Roots) != 1 {
return nil, fmt.Errorf("car should have 1 root, has %d", len(ch.Roots))
}
cborstore := cbor.NewCborStore(store)
fmt.Printf("root is %s\n", ch.Roots[0])
nd, err := hamt.LoadNode(ctx, cborstore, ch.Roots[0], hamt.UseTreeBitWidth(5))
if err != nil {
return nil, err
}
if err := nd.ForEach(ctx, func(k string, val interface{}) error {
n, ok := val.(format.Node)
if !ok {
fmt.Printf("hampt %s (not node): %+v\n", k, val)
} else {
fmt.Printf("%s: %#v\n", k, n)
}
return nil
}); err != nil {
return nil, err
}
return state.LoadStateTree(cborstore, ch.Roots[0])
}

93
tvx/state/store.go Normal file
View File

@ -0,0 +1,93 @@
package state
import (
"context"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/specs-actors/actors/util/adt"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
ds "github.com/ipfs/go-datastore"
exchange "github.com/ipfs/go-ipfs-exchange-interface"
offline "github.com/ipfs/go-ipfs-exchange-offline"
cbor "github.com/ipfs/go-ipld-cbor"
format "github.com/ipfs/go-ipld-format"
"github.com/ipfs/go-merkledag"
)
// ProxyingStores implements the ipld store where unknown items are fetched over the node API.
type ProxyingStores struct {
CBORStore cbor.IpldStore
ADTStore adt.Store
Datastore ds.Batching
Blockstore blockstore.Blockstore
BlockService blockservice.BlockService
Exchange exchange.Interface
DAGService format.DAGService
}
type proxyingBlockstore struct {
ctx context.Context
api api.FullNode
blockstore.Blockstore
}
func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
if block, err := pb.Blockstore.Get(cid); err == nil {
return block, err
}
// fmt.Printf("fetching cid via rpc: %v\n", cid)
item, err := pb.api.ChainReadObj(pb.ctx, cid)
if err != nil {
return nil, err
}
block, err := blocks.NewBlockWithCid(item, cid)
if err != nil {
return nil, err
}
err = pb.Blockstore.Put(block)
if err != nil {
return nil, err
}
return block, nil
}
// NewProxyingStore is a blockstore that proxies get requests for unknown CIDs
// to a Filecoin node, via the ChainReadObj RPC.
//
// It also contains all possible stores, services and gadget that IPLD
// requires (quite a handful).
func NewProxyingStore(ctx context.Context, api api.FullNode) *ProxyingStores {
ds := ds.NewMapDatastore()
bs := &proxyingBlockstore{
ctx: ctx,
api: api,
Blockstore: blockstore.NewBlockstore(ds),
}
var (
cborstore = cbor.NewCborStore(bs)
offl = offline.Exchange(bs)
blkserv = blockservice.New(bs, offl)
dserv = merkledag.NewDAGService(blkserv)
)
return &ProxyingStores{
CBORStore: cborstore,
ADTStore: adt.WrapStore(ctx, cborstore),
Datastore: ds,
Blockstore: bs,
Exchange: offl,
BlockService: blkserv,
DAGService: dserv,
}
}

283
tvx/state/surgeon.go Normal file
View File

@ -0,0 +1,283 @@
package state
import (
"bytes"
"context"
"fmt"
"io"
"log"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/specs-actors/actors/builtin"
init_ "github.com/filecoin-project/specs-actors/actors/builtin/init"
"github.com/filecoin-project/specs-actors/actors/util/adt"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-ipld-format"
"github.com/ipld/go-car"
cbg "github.com/whyrusleeping/cbor-gen"
)
// Surgeon is an object used to fetch and manipulate state.
type Surgeon struct {
ctx context.Context
api api.FullNode
stores *ProxyingStores
}
// NewSurgeon returns a state surgeon, an object used to fetch and manipulate
// state.
func NewSurgeon(ctx context.Context, api api.FullNode, stores *ProxyingStores) *Surgeon {
return &Surgeon{
ctx: ctx,
api: api,
stores: stores,
}
}
// GetMaskedStateTree trims the state tree at the supplied tipset to contain
// only the state of the actors in the retain set. It also "dives" into some
// singleton system actors, like the init actor, to trim the state so as to
// compute a minimal state tree. In the future, thid method will dive into
// other system actors like the power actor and the market actor.
func (sg *Surgeon) GetMaskedStateTree(tsk types.TipSetKey, retain []address.Address) (cid.Cid, error) {
stateMap := adt.MakeEmptyMap(sg.stores.ADTStore)
initState, err := sg.loadInitActor(tsk)
if err != nil {
return cid.Undef, err
}
resolved, err := sg.resolveAddresses(retain, initState)
if err != nil {
return cid.Undef, err
}
initState, err = sg.retainInitEntries(initState, retain)
if err != nil {
return cid.Undef, err
}
err = sg.saveInitActor(initState, stateMap)
if err != nil {
return cid.Undef, err
}
err = sg.pluckActorStates(tsk, resolved, stateMap)
if err != nil {
return cid.Undef, err
}
root, err := stateMap.Root()
if err != nil {
return cid.Undef, err
}
return root, nil
}
// GetAccessedActors identifies the actors that were accessed during the
// execution of a message.
func (sg *Surgeon) GetAccessedActors(ctx context.Context, a api.FullNode, mid cid.Cid) ([]address.Address, error) {
log.Printf("calculating accessed actors during execution of message: %s", mid)
msgInfo, err := a.StateSearchMsg(ctx, mid)
if err != nil {
return nil, err
}
ts, err := a.ChainGetTipSet(ctx, msgInfo.TipSet)
if err != nil {
return nil, err
}
trace, err := a.StateReplay(ctx, ts.Parents(), mid)
if err != nil {
return nil, fmt.Errorf("could not replay msg: %w", err)
}
accessed := make(map[address.Address]struct{})
var recur func(trace *types.ExecutionTrace)
recur = func(trace *types.ExecutionTrace) {
accessed[trace.Msg.To] = struct{}{}
accessed[trace.Msg.From] = struct{}{}
for _, s := range trace.Subcalls {
recur(&s)
}
}
recur(&trace.ExecutionTrace)
ret := make([]address.Address, 0, len(accessed))
for k := range accessed {
ret = append(ret, k)
}
return ret, nil
}
// WriteCAR recursively writes the tree referenced by the root as a CAR into the
// supplied io.Writer.
func (sg *Surgeon) WriteCAR(w io.Writer, root cid.Cid) error {
carWalkFn := func(nd format.Node) (out []*format.Link, err error) {
for _, link := range nd.Links() {
if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
continue
}
out = append(out, link)
}
return out, nil
}
return car.WriteCarWithWalker(sg.ctx, sg.stores.DAGService, []cid.Cid{root}, w, carWalkFn)
}
// pluckActorStates plucks the state from the supplied actors at the given
// tipset, and places it into the supplied state map.
func (sg *Surgeon) pluckActorStates(tsk types.TipSetKey, pluck []address.Address, stateMap *adt.Map) error {
for _, a := range pluck {
actor, err := sg.api.StateGetActor(sg.ctx, a, tsk)
if err != nil {
return err
}
err = stateMap.Put(adt.AddrKey(a), actor)
if err != nil {
return err
}
// recursive copy of the actor state so we can
err = vm.Copy(sg.stores.Blockstore, sg.stores.Blockstore, actor.Head)
if err != nil {
return err
}
actorState, err := sg.api.ChainReadObj(sg.ctx, actor.Head)
if err != nil {
return err
}
cid, err := sg.stores.CBORStore.Put(sg.ctx, &cbg.Deferred{Raw: actorState})
if err != nil {
return err
}
if cid != actor.Head {
panic("mismatched cids")
}
}
return nil
}
// saveInitActor saves the state of the init actor to the provided state map.
func (sg *Surgeon) saveInitActor(initState *init_.State, stateMap *adt.Map) error {
log.Printf("saving init actor into state tree")
// Store the state of the init actor.
cid, err := sg.stores.CBORStore.Put(sg.ctx, initState)
if err != nil {
return err
}
actor := &types.Actor{
Code: builtin.InitActorCodeID,
Head: cid,
}
err = stateMap.Put(adt.AddrKey(builtin.InitActorAddr), actor)
if err != nil {
return err
}
cid, _ = stateMap.Root()
log.Printf("saved init actor into state tree; new root: %s", cid)
return nil
}
// retainInitEntries takes an old init actor state, and retains only the
// entries in the retain set, returning a new init actor state.
func (sg *Surgeon) retainInitEntries(oldState *init_.State, retain []address.Address) (*init_.State, error) {
log.Printf("retaining init actor entries for addresses: %v", retain)
oldAddrs, err := adt.AsMap(sg.stores.ADTStore, oldState.AddressMap)
if err != nil {
return nil, err
}
newAddrs := adt.MakeEmptyMap(sg.stores.ADTStore)
for _, r := range retain {
if r.Protocol() == address.ID {
// skip over ID addresses; they don't need a mapping in the init actor.
continue
}
var d cbg.Deferred
if _, err := oldAddrs.Get(adt.AddrKey(r), &d); err != nil {
return nil, err
}
if err := newAddrs.Put(adt.AddrKey(r), &d); err != nil {
return nil, err
}
}
rootCid, err := newAddrs.Root()
if err != nil {
return nil, err
}
s := &init_.State{
NetworkName: oldState.NetworkName,
NextID: oldState.NextID,
AddressMap: rootCid,
}
log.Printf("new init actor state: %+v", s)
return s, nil
}
// resolveAddresses resolved the requested addresses from the provided
// InitActor state, returning a slice of length len(orig), where each index
// contains the resolved address.
func (sg *Surgeon) resolveAddresses(orig []address.Address, ist *init_.State) (ret []address.Address, err error) {
log.Printf("resolving addresses: %v", orig)
ret = make([]address.Address, len(orig))
for i, addr := range orig {
resolved, err := ist.ResolveAddress(sg.stores.ADTStore, addr)
if err != nil {
return nil, err
}
ret[i] = resolved
}
log.Printf("resolved addresses: %v", ret)
return ret, nil
}
// loadInitActor loads the init actor state from a given tipset.
func (sg *Surgeon) loadInitActor(tsk types.TipSetKey) (initState *init_.State, err error) {
log.Printf("loading the init actor for tipset: %s", tsk)
actor, err := sg.api.StateGetActor(sg.ctx, builtin.InitActorAddr, tsk)
if err != nil {
return initState, err
}
actorState, err := sg.api.ChainReadObj(sg.ctx, actor.Head)
if err != nil {
return initState, err
}
initState = new(init_.State)
err = initState.UnmarshalCBOR(bytes.NewReader(actorState))
if err != nil {
return initState, err
}
log.Printf("loaded init actor state: %+v", initState)
return initState, nil
}

97
tvx/state_delta.go Normal file
View File

@ -0,0 +1,97 @@
package main
import (
"context"
"fmt"
"github.com/filecoin-project/lotus/chain/types"
"github.com/ipfs/go-cid"
"github.com/urfave/cli/v2"
)
var deltaFlags struct {
from string
to string
}
var deltaCmd = &cli.Command{
Name: "state-delta",
Description: "collect affected state between two tipsets, addressed by blocks",
Action: runStateDelta,
Flags: []cli.Flag{
&apiFlag,
&cli.StringFlag{
Name: "from",
Usage: "block CID of initial state",
Required: true,
Destination: &deltaFlags.from,
},
&cli.StringFlag{
Name: "to",
Usage: "block CID of ending state",
Required: true,
Destination: &deltaFlags.to,
},
},
}
func runStateDelta(c *cli.Context) error {
node, err := makeClient(c)
if err != nil {
return err
}
from, err := cid.Decode(deltaFlags.from)
if err != nil {
return err
}
to, err := cid.Decode(deltaFlags.to)
if err != nil {
return err
}
currBlock, err := node.ChainGetBlock(context.TODO(), to)
if err != nil {
return err
}
srcBlock, err := node.ChainGetBlock(context.TODO(), from)
if err != nil {
return err
}
allMsgs := make(map[uint64][]*types.Message)
epochs := currBlock.Height - srcBlock.Height - 1
for epochs > 0 {
msgs, err := node.ChainGetBlockMessages(context.TODO(), to)
if err != nil {
return err
}
allMsgs[uint64(currBlock.Height)] = msgs.BlsMessages
currBlock, err = node.ChainGetBlock(context.TODO(), currBlock.Parents[0])
epochs--
}
if !hasParent(currBlock, from) {
return fmt.Errorf("from block was not a parent of `to` as expected")
}
m := 0
for _, msgs := range allMsgs {
m += len(msgs)
}
fmt.Printf("messages: %d\n", m)
fmt.Printf("initial state root: %v\n", currBlock.ParentStateRoot)
return nil
}
func hasParent(block *types.BlockHeader, parent cid.Cid) bool {
for _, p := range block.Parents {
if p.Equals(parent) {
return true
}
}
return false
}