cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

remove tvx and corpus from this repo; spun off elsewhere. (#268)

Browse Source
This commit is contained in:
Raúl Kripalani 2020-10-07 19:24:46 +01:00 committed by GitHub
parent 2a6e05b776
commit 44af7e43b6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 0 additions and 3374 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.circleci/config.yml
View File

@ -27,7 +27,6 @@ workflows:
version: 2 version: 2
main: main:
jobs: jobs:
- build-tvx-linux
- build-soup-linux - build-soup-linux
- trigger-testplans - trigger-testplans
nightly: nightly:
@ -42,13 +41,6 @@ workflows:
- trigger-testplans - trigger-testplans
jobs: jobs:
build-tvx-linux:
executor: golang
steps:
- setup
- run:
name: "build tvx"
command: pushd tvx && go build .
build-soup-linux: build-soup-linux:
executor: golang executor: golang

200
corpus/schema.json
View File

@ -1,200 +0,0 @@
{
"$id": "https://filecoin.io/oni/schemas/test-vector.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "a filecoin VM test vector",
"type": "object",
"definitions": {
"hex": {
"title": "hex value",
"description": "a hex value prefixed with 0x, and accepting only lowercase characters; 0x represents an empty byte array",
"type": "string",
"pattern": "0x[0-9a-f]*",
"examples": [
"0xa1b2c3",
"0x"
]
},
"meta": {
"title": "metadata",
"description": "metadata about this test vector, such as its id, version, data about its generation, etc.",
"type": "object",
"additionalProperties": false,
"properties": {
"id": {
"title": "a unique identifier that identifies this test vector",
"type": "string"
},
"version": {
"title": "the version of this test vector",
"type": "string"
},
"description": {
"title": "an optional description of the test vector",
"type": "string"
},
"comment": {
"title": "optional comments about this test vector, e.g. applicability, hints, rationale, etc.",
"type": "string"
},
"gen": {
"title": "generation metadata",
"description": "metadata about how this test vector was generated",
"type": "object",
"additionalProperties": false,
"properties": {
"source": {
"type": "string",
"examples": [
"lotus",
"dsl"
]
},
"version": {
"type": "string",
"examples": [
"0.4.1+git.27d74337+api0.8.1"
]
}
}
}
}
},
"state_tree": {
"additionalProperties": false,
"required": [
"root_cid"
],
"properties": {
"root_cid": {
"additionalProperties": false,
"required": [
"/"
],
"properties": {
"/": {
"type": "string"
}
}
}
}
},
"receipt": {
"type": "object",
"required": [
"exit_code",
"return",
"gas_used"
],
"additionalProperties": false,
"properties": {
"exit_code": {
"type": "number"
},
"return": {
"$ref": "#/definitions/hex"
},
"gas_used": {
"type": "number"
}
}
},
"preconditions": {
"title": "execution preconditions",
"description": "preconditions that need to be applied and satisfied before this test vector can be executed",
"additionalProperties": false,
"properties": {
"epoch": {
"type": "integer"
},
"state_tree": {
"title": "state tree to seed",
"description": "state tree to seed before applying this test vector; mapping of actor addresses => serialized state",
"$ref": "#/definitions/state_tree"
}
}
},
"postconditions": {
"title": "execution preconditions",
"description": "postconditions that need to be satisfied after execution for this test vector to pass",
"additionalProperties": false,
"properties": {
"state_tree": {
"title": "state tree postconditions",
"description": "state tree postconditions that must be true for this test vector to pass",
"$ref": "#/definitions/state_tree"
},
"receipts": {
"title": "receipts to match",
"description": "receipts to match, required when using messages-class test vectors; length of this array MUST be equal to length of apply_messages",
"type": "array",
"items": {
"$ref": "#/definitions/receipt"
}
}
}
},
"apply_messages": {
"title": "messages to apply, along with the receipt to expect for each",
"type": "array",
"items": {
"$ref": "#/definitions/hex"
}
}
},
"required": [
"class"
],
"properties": {
"class": {
"title": "test vector class",
"description": "test vector class; depending on the value, the apply_* property to provide (and its schema) will vary; the relevant apply property is apply_[class]",
"type": "string",
"enum": [
"messages",
"block",
"tipset",
"chain"
]
},
"selector": {
"title": "selector the driver can use to determine if this test vector applies",
"description": "format TBD",
"type": "string"
},
"_meta": {
"$ref": "#/definitions/meta"
},
"car_bytes": {
"title": "car containing state trees",
"description": "the gzipped, hex-encoded CAR containing the pre- and post-condition state trees for this test vector",
"$ref": "#/definitions/hex"
},
"preconditions": {
"$ref": "#/definitions/preconditions"
},
"postconditions": {
"$ref": "#/definitions/postconditions"
}
},
"allOf": [
{
"if": {
"properties": {
"class": {
"const": "messages"
}
}
},
"then": {
"required": [
"apply_messages"
],
"properties": {
"apply_messages": {
"$ref": "#/definitions/apply_messages"
}
}
}
}
]
}

40
corpus/single_message.json
View File

@ -1,40 +0,0 @@
{
"class": "messages",
"selector": "<some predicate for the driver to determine if this vector applies to the implementation, e.g. to deal with protocol upgrades>",
"_meta": {
"id": "test_vector_id",
"version": "version_id",
"description": "tests that the VM rejects a message containing...",
"comment": "this message should never actually be fed to the VM, because the block would be deemed invalid by the syncer",
"gen": {
"source": "lotus",
"version": "0.4.1+git.27d74337+api0.8.1"
}
},
"car_bytes": "0x890043009920583103b5b1fab6769fcb464146d192af785915d8e1dacff264b2be3bcb7bf4064a77cb3b13e27063ea4825dc48cd0a7df77e211916e748002740f0c493a2824200011a09d9c4dc0758c68219091c58c083e7c59e748e070b382a813f4ffe8ead2a090469aaa37f0664b82c092bc78b43fac300335cd2d6e4a8c71ce138003c0fa8ee1707b41aa87f35c811613d59ac53449aef8263aa51c876f2e67c185118ec9628f73442f58e093909ce952e431e770ac693cdbe99f2f9404ad3ca754196ab443c6c66afa41310e705e453496bdcedf0f271529dc7374021d7b80d343397f996d06bf11e753a49eaed7b3ce80a8e9bbaa48bf83223527bd8a2933f002949fdb8058103336a4a979375d5e963cdd7b1",
"preconditions": {
"epoch": 100,
"state_tree": {
"root_cid": {
"/": "bafy2bzacebbxsepazfgwepawspvzenb2x64pmqjyan3wgtfpxu5nxez33wzkc"
}
}
},
"apply_messages": [
"0x89004300ba20583103b85358cc1c968d826c0a0efcde99b6e4db138edfc051f188a4471cfc23ccf7884adbbac912e45f7363774bf16f83ca8f1902b44800505e1442e66b744200011a09d225c80758c68219016f58c099a447a8c80988e1c061dd3a8dd674033c3a6083c4b55cf035e6e8de3baf09436ad00adf6eb14d58ae6691d22ce82255874047e7aa0b9a86bbf424afc6d89dc29f7b253dd04c88a625b86345f05aa0bdee503971c1e9735ed5b81c31ad41a1f10b387d96151d1cbd0c4286267daf04087fd8d5272f870994c3732f3fc4f54287a485d6c8dafb6056df3fc9233f849027b4c1272850435126fef65a8363768a858c0cc5f9d9114ff378ad50cab25ab0a2f685ca63d9f1f7727da4d06db5b80b5f"
],
"postconditions": {
"state_tree": {
"root_cid": {
"/": "bafy2bzacebbxsepazfgwepawspvzenb2x64pmqjyan3wgtfpxu5nxez33wzkc"
}
},
"receipts": [
{
"exit_code": 10,
"gas_used": 1000,
"return": "0x"
}
]
}
}

116
tvx/examine.go
View File

@ -1,116 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"github.com/ipfs/go-cid"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/specs-actors/actors/builtin"
init_ "github.com/filecoin-project/specs-actors/actors/builtin/init"
"github.com/filecoin-project/specs-actors/actors/util/adt"
"github.com/filecoin-project/test-vectors/schema"
"github.com/filecoin-project/oni/tvx/state"
)
func trimQuotes(s string) string {
if len(s) >= 2 {
if s[0] == '"' && s[len(s)-1] == '"' {
return s[1 : len(s)-1]
}
}
return s
}
var examineFlags struct {
file string
pre bool
post bool
}
var examineCmd = &cli.Command{
Name: "examine",
Description: "examine an exported state root as represented in a test vector",
Action: runExamineCmd,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "file",
Usage: "test vector file",
Required: true,
Destination: &examineFlags.file,
},
&cli.BoolFlag{
Name: "pre",
Usage: "examine the precondition state tree",
Destination: &examineFlags.pre,
},
&cli.BoolFlag{
Name: "post",
Usage: "examine the postcondition state tree",
Destination: &examineFlags.post,
},
},
}
func runExamineCmd(_ *cli.Context) error {
file, err := os.Open(examineFlags.file)
if err != nil {
return err
}
var tv schema.TestVector
if err := json.NewDecoder(file).Decode(&tv); err != nil {
return err
}
examine := func(root cid.Cid) error {
encoded := tv.CAR
tree, err := state.RecoverStateTree(context.TODO(), encoded, root)
if err != nil {
return err
}
initActor, err := tree.GetActor(builtin.InitActorAddr)
if err != nil {
return fmt.Errorf("cannot recover init actor: %w", err)
}
var ias init_.State
if err := tree.Store.Get(context.TODO(), initActor.Head, &ias); err != nil {
return err
}
adtStore := adt.WrapStore(context.TODO(), tree.Store)
m, err := adt.AsMap(adtStore, ias.AddressMap)
if err != nil {
return err
}
actors, err := m.CollectKeys()
for _, actor := range actors {
fmt.Printf("%s\n", actor)
}
return nil
}
if examineFlags.pre {
log.Print("examining precondition tree")
if err := examine(tv.Pre.StateTree.RootCID); err != nil {
return err
}
}
if examineFlags.post {
log.Print("examining postcondition tree")
if err := examine(tv.Post.StateTree.RootCID); err != nil {
return err
}
}
return nil
}

146
tvx/exec_lotus.go
View File

@ -1,146 +0,0 @@
package main
import (
"bytes"
"compress/gzip"
"context"
"encoding/json"
"fmt"
"io"
"os"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/conformance"
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/ipld/go-car"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/test-vectors/schema"
)
var execLotusFlags struct {
file string
}
var execLotusCmd = &cli.Command{
Name: "exec-lotus",
Description: "execute a test vector against Lotus",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "file",
Usage: "input file",
Destination: &execLotusFlags.file,
},
},
Action: runExecLotus,
}
func runExecLotus(_ *cli.Context) error {
switch {
case execLotusFlags.file != "":
file, err := os.Open(execLotusFlags.file)
if err != nil {
return fmt.Errorf("failed to open test vector: %w", err)
}
var (
dec = json.NewDecoder(file)
tv schema.TestVector
)
if err = dec.Decode(&tv); err != nil {
return fmt.Errorf("failed to decode test vector: %w", err)
}
return executeTestVector(tv)
default:
dec := json.NewDecoder(os.Stdin)
for {
var tv schema.TestVector
err := dec.Decode(&tv)
if err == io.EOF {
return nil
}
if err != nil {
return err
}
err = executeTestVector(tv)
if err != nil {
return err
}
}
}
}
func executeTestVector(tv schema.TestVector) error {
fmt.Println("executing test vector:", tv.Meta.Desc)
switch tv.Class {
case "message":
var (
ctx = context.Background()
epoch = tv.Pre.Epoch
root = tv.Pre.StateTree.RootCID
)
bs := blockstore.NewTemporary()
buf := bytes.NewReader(tv.CAR)
gr, err := gzip.NewReader(buf)
if err != nil {
return err
}
defer gr.Close()
header, err := car.LoadCar(bs, gr)
if err != nil {
return fmt.Errorf("failed to load state tree car from test vector: %w", err)
}
fmt.Println("roots: ", header.Roots)
driver := conformance.NewDriver(ctx, tv.Selector)
for i, m := range tv.ApplyMessages {
fmt.Printf("decoding message %v\n", i)
msg, err := types.DecodeMessage(m.Bytes)
if err != nil {
return err
}
// add an epoch if we have set one
if m.Epoch != nil {
epoch = *m.Epoch
}
fmt.Printf("executing message %v\n", i)
var ret *vm.ApplyRet
ret, root, err = driver.ExecuteMessage(bs, root, abi.ChainEpoch(epoch), msg)
if err != nil {
return err
}
if expected, actual := tv.Post.Receipts[i].ExitCode, ret.ExitCode; expected != int64(actual) {
return fmt.Errorf("exit code of msg %d did not match; expected: %s, got: %s", i, expected, actual)
}
if expected, actual := tv.Post.Receipts[i].GasUsed, ret.GasUsed; expected != actual {
return fmt.Errorf("gas used of msg %d did not match; expected: %d, got: %d", i, expected, actual)
}
// TODO assert return value
fmt.Printf("✅ message %d passed expectations\n", i)
}
if root != tv.Post.StateTree.RootCID {
return fmt.Errorf("wrong post root cid; expected %v , but got %v", tv.Post.StateTree.RootCID, root)
}
return nil
default:
return fmt.Errorf("test vector class not supported")
}
}

349
tvx/extract_msg.go
View File

@ -1,349 +0,0 @@
package main
import (
"bytes"
"compress/gzip"
"context"
"encoding/json"
"fmt"
"io"
"log"
"os"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/api"
init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init"
"github.com/filecoin-project/lotus/chain/actors/builtin/reward"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/conformance"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/ipfs/go-cid"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/test-vectors/schema"
"github.com/filecoin-project/oni/tvx/state"
)
var extractMsgFlags struct {
cid string
file string
retain string
}
var extractMsgCmd = &cli.Command{
Name: "extract-message",
Description: "generate a message-class test vector by extracting it from a network",
Action: runExtractMsg,
Flags: []cli.Flag{
&apiFlag,
&cli.StringFlag{
Name: "cid",
Usage: "message CID to generate test vector from",
Required: true,
Destination: &extractMsgFlags.cid,
},
&cli.StringFlag{
Name: "file",
Usage: "output file",
Required: true,
Destination: &extractMsgFlags.file,
},
&cli.StringFlag{
Name: "state-retain",
Usage: "state retention policy; values: 'accessed-cids' (default), 'accessed-actors'",
Value: "accessed-cids",
Destination: &extractMsgFlags.retain,
},
},
}
func runExtractMsg(c *cli.Context) error {
// LOTUS_DISABLE_VM_BUF disables what's called "VM state tree buffering",
// which stashes write operations in a BufferedBlockstore
// (https://github.com/filecoin-project/lotus/blob/b7a4dbb07fd8332b4492313a617e3458f8003b2a/lib/bufbstore/buf_bstore.go#L21)
// such that they're not written until the VM is actually flushed.
//
// For some reason, the standard behaviour was not working for me (raulk),
// and disabling it (such that the state transformations are written immediately
// to the blockstore) worked.
_ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea")
ctx := context.Background()
// get the output file.
if extractMsgFlags.file == "" {
return fmt.Errorf("output file required")
}
mcid, err := cid.Decode(extractMsgFlags.cid)
if err != nil {
return err
}
// Make the client.
api, err := makeClient(c)
if err != nil {
return err
}
log.Printf("locating message with CID: %s...", mcid)
// Locate the message.
msgInfo, err := api.StateSearchMsg(ctx, mcid)
if err != nil {
return fmt.Errorf("failed to locate message: %w", err)
}
log.Printf("located message at tipset %s (height: %d) with exit code: %s", msgInfo.TipSet, msgInfo.Height, msgInfo.Receipt.ExitCode)
// Extract the full message.
msg, err := api.ChainGetMessage(ctx, mcid)
if err != nil {
return err
}
log.Printf("full message: %+v", msg)
execTs, incTs, err := findRelevantTipsets(ctx, api, msgInfo.TipSet)
if err != nil {
return err
}
log.Printf("message was executed in tipset: %s", execTs.Key())
log.Printf("message was included in tipset: %s", incTs.Key())
log.Printf("finding precursor messages...")
var allmsgs []*types.Message
for _, b := range incTs.Blocks() {
messages, err := api.ChainGetBlockMessages(ctx, b.Cid())
if err != nil {
return err
}
related, found, err := findMsgAndPrecursors(messages, msg)
if err != nil {
return fmt.Errorf("invariant failed while scanning messages in block %s: %w", b.Cid(), err)
}
if found {
var mcids []cid.Cid
for _, m := range related {
mcids = append(mcids, m.Cid())
}
log.Printf("found message in block %s; precursors: %v", b.Cid(), mcids[:len(mcids)-1])
allmsgs = related
break
}
log.Printf("message not found in block %s; precursors found: %d", b.Cid(), len(related))
}
if allmsgs == nil {
// Message was not found; abort.
return fmt.Errorf("did not find a block containing the message")
}
precursors := allmsgs[:len(allmsgs)-1]
var (
// create a read through store that uses ChainGetObject to fetch unknown CIDs.
pst = state.NewProxyingStores(ctx, api)
g = state.NewSurgeon(ctx, api, pst)
)
driver := conformance.NewDriver(ctx, schema.Selector{})
// this is the root of the state tree we start with.
root := incTs.ParentState()
log.Printf("base state tree root CID: %s", root)
// on top of that state tree, we apply all precursors.
log.Printf("precursors to apply: %d", len(precursors))
for i, m := range precursors {
log.Printf("applying precursor %d, cid: %s", i, m.Cid())
_, root, err = driver.ExecuteMessage(pst.Blockstore, root, execTs.Height(), m)
if err != nil {
return fmt.Errorf("failed to execute precursor message: %w", err)
}
}
var (
preroot cid.Cid
postroot cid.Cid
carWriter func(w io.Writer) error
)
switch retention := extractMsgFlags.retain; retention {
case "accessed-actors":
log.Printf("calculating accessed actors...")
// get actors accessed by message.
retain, err := g.GetAccessedActors(ctx, api, mcid)
if err != nil {
return fmt.Errorf("failed to calculate accessed actors: %w", err)
}
// also append the reward actor and the burnt funds actor.
retain = append(retain, reward.Address, builtin.BurntFundsActorAddr, init_.Address)
log.Printf("calculated accessed actors: %v", retain)
// get the masked state tree from the root,
preroot, err = g.GetMaskedStateTree(root, retain)
if err != nil {
return err
}
_, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg)
if err != nil {
return fmt.Errorf("failed to execute message: %w", err)
}
carWriter = func(w io.Writer) error {
return g.WriteCAR(w, preroot, postroot)
}
case "accessed-cids":
log.Printf("using state retention: %s", retention)
tbs, ok := pst.Blockstore.(state.TracingBlockstore)
if !ok {
return fmt.Errorf("requested 'accessed-cids' state retention, but no tracing blockstore was present")
}
tbs.StartTracing()
preroot = execTs.ParentState()
_, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg)
if err != nil {
return fmt.Errorf("failed to execute message: %w", err)
}
accessed := tbs.FinishTracing()
carWriter = func(w io.Writer) error {
return g.WriteCARIncluding(w, accessed, preroot, postroot)
}
default:
return fmt.Errorf("unknown state retention option: %s", retention)
}
msgBytes, err := msg.Serialize()
if err != nil {
return err
}
var (
out = new(bytes.Buffer)
gw = gzip.NewWriter(out)
)
if err := carWriter(gw); err != nil {
return err
}
if err = gw.Flush(); err != nil {
return err
}
if err = gw.Close(); err != nil {
return err
}
version, err := api.Version(ctx)
if err != nil {
return err
}
// Write out the test vector.
vector := schema.TestVector{
Class: schema.ClassMessage,
Selector: schema.Selector(map[string]string{}),
Meta: &schema.Metadata{
ID: "TK",
Version: "TK",
Gen: []schema.GenerationData{schema.GenerationData{
Source: msg.Cid().String(),
Version: version.String(),
}},
},
CAR: out.Bytes(),
Pre: &schema.Preconditions{
Epoch: int64(execTs.Height()),
StateTree: &schema.StateTree{
RootCID: preroot,
},
},
ApplyMessages: []schema.Message{{Bytes: msgBytes}},
Post: &schema.Postconditions{
StateTree: &schema.StateTree{
RootCID: postroot,
},
},
}
file, err := os.Create(extractMsgFlags.file)
if err != nil {
return err
}
defer file.Close()
enc := json.NewEncoder(file)
enc.SetIndent("", " ")
if err := enc.Encode(&vector); err != nil {
return err
}
return nil
}
func findRelevantTipsets(ctx context.Context, api api.FullNode, execTsk types.TipSetKey) (execTs *types.TipSet, incTs *types.TipSet, err error) {
// get the tipset on which this message was "executed" on.
// https://github.com/filecoin-project/lotus/issues/2847
execTs, err = api.ChainGetTipSet(ctx, execTsk)
if err != nil {
return nil, nil, err
}
// get the previous tipset, on which this message was mined,
// i.e. included on-chain.
incTs, err = api.ChainGetTipSet(ctx, execTs.Parents())
if err != nil {
return nil, nil, err
}
return execTs, incTs, nil
}
// findMsgAndPrecursors scans the messages in a block to locate the supplied
// message, looking into the BLS or SECP section depending on the sender's
// address type.
//
// It returns any precursors (if they exist), and the found message (if found),
// in a slice.
//
// It also returns a boolean indicating whether the message was actually found.
//
// This function also asserts invariants, and if those fail, it returns an error.
func findMsgAndPrecursors(messages *api.BlockMessages, target *types.Message) (related []*types.Message, found bool, err error) {
// Decide which block of messages to process, depending on whether the
// sender is a BLS or a SECP account.
input := messages.BlsMessages
if senderKind := target.From.Protocol(); senderKind == address.SECP256K1 {
input = make([]*types.Message, 0, len(messages.SecpkMessages))
for _, sm := range messages.SecpkMessages {
input = append(input, &sm.Message)
}
}
for _, other := range input {
if other.From != target.From {
continue
}
// this message is from the same sender, so it's related.
related = append(related, other)
if other.Nonce > target.Nonce {
return nil, false, fmt.Errorf("a message with nonce higher than the target was found before the target; offending mcid: %s", other.Cid())
}
// this message is the target; we're done.
if other.Cid() == target.Cid() {
return related, true, nil
}
}
// this could happen because a block contained related messages, but not
// the target (that is, messages with a lower nonce, but ultimately not the
// target).
return related, false, nil
}

34
tvx/go.mod
View File

@ -1,34 +0,0 @@
module github.com/filecoin-project/oni/tvx
go 1.14
require (
github.com/fatih/color v1.8.0
github.com/filecoin-project/go-address v0.0.3
github.com/filecoin-project/go-state-types v0.0.0-20200911004822-964d6c679cfc
github.com/filecoin-project/lotus v0.8.0
github.com/filecoin-project/specs-actors v0.9.11
github.com/filecoin-project/test-vectors/schema v0.0.1
github.com/ipfs/go-block-format v0.0.2
github.com/ipfs/go-blockservice v0.1.4-0.20200624145336-a978cec6e834
github.com/ipfs/go-cid v0.0.7
github.com/ipfs/go-datastore v0.4.4
github.com/ipfs/go-hamt-ipld v0.1.1
github.com/ipfs/go-ipfs-exchange-interface v0.0.1
github.com/ipfs/go-ipfs-exchange-offline v0.0.1
github.com/ipfs/go-ipld-cbor v0.0.5-0.20200428170625-a0bd04d3cbdf
github.com/ipfs/go-ipld-format v0.2.0
github.com/ipfs/go-merkledag v0.3.2
github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4
github.com/mattn/go-isatty v0.0.9 // indirect
github.com/multiformats/go-multiaddr v0.3.1
github.com/multiformats/go-multiaddr-net v0.2.0
github.com/urfave/cli/v2 v2.2.0
github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163
golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
honnef.co/go/tools v0.0.1-2020.1.3 // indirect
)
replace github.com/filecoin-project/filecoin-ffi => ../extra/filecoin-ffi
replace github.com/supranational/blst => github.com/supranational/blst v0.1.2-alpha.1

1754
tvx/go.sum
View File

File diff suppressed because it is too large Load Diff

58
tvx/list_accessed.go
View File

@ -1,58 +0,0 @@
package main
import (
"context"
"fmt"
"github.com/ipfs/go-cid"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/oni/tvx/state"
)
var listAccessedFlags struct {
cid string
}
var listAccessedCmd = &cli.Command{
Name: "list-accessed",
Description: "extract actors accessed during the execution of a message",
Action: runListAccessed,
Flags: []cli.Flag{
&apiFlag,
&cli.StringFlag{
Name: "cid",
Usage: "message CID",
Required: true,
Destination: &listAccessedFlags.cid,
},
},
}
func runListAccessed(c *cli.Context) error {
ctx := context.Background()
node, err := makeClient(c)
if err != nil {
return err
}
mid, err := cid.Decode(listAccessedFlags.cid)
if err != nil {
return err
}
rtst := state.NewProxyingStores(ctx, node)
sg := state.NewSurgeon(ctx, node, rtst)
actors, err := sg.GetAccessedActors(context.TODO(), node, mid)
if err != nil {
return err
}
for k := range actors {
fmt.Printf("%v\n", k)
}
return nil
}

81
tvx/main.go
View File

@ -1,81 +0,0 @@
package main
import (
"context"
"fmt"
"log"
"net/http"
"os"
"sort"
"strings"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/client"
"github.com/multiformats/go-multiaddr"
manet "github.com/multiformats/go-multiaddr-net"
"github.com/urfave/cli/v2"
)
var apiFlag = cli.StringFlag{
Name: "api",
Usage: "api endpoint, formatted as token:multiaddr",
Value: "",
EnvVars: []string{"FULLNODE_API_INFO"},
}
func main() {
app := &cli.App{
Name: "tvx",
Description: "a toolbox for managing test vectors",
Usage: "a toolbox for managing test vectors",
Commands: []*cli.Command{
deltaCmd,
listAccessedCmd,
extractMsgCmd,
execLotusCmd,
examineCmd,
},
}
sort.Sort(cli.CommandsByName(app.Commands))
for _, c := range app.Commands {
sort.Sort(cli.FlagsByName(c.Flags))
}
err := app.Run(os.Args)
if err != nil {
log.Fatal(err)
}
}
func makeClient(c *cli.Context) (api.FullNode, error) {
api := c.String(apiFlag.Name)
sp := strings.SplitN(api, ":", 2)
if len(sp) != 2 {
return nil, fmt.Errorf("invalid api value, missing token or address: %s", api)
}
// TODO: discovery from filesystem
token := sp[0]
ma, err := multiaddr.NewMultiaddr(sp[1])
if err != nil {
return nil, fmt.Errorf("could not parse provided multiaddr: %w", err)
}
_, dialAddr, err := manet.DialArgs(ma)
if err != nil {
return nil, fmt.Errorf("invalid api multiAddr: %w", err)
}
addr := "ws://" + dialAddr + "/rpc/v0"
headers := http.Header{}
if len(token) != 0 {
headers.Add("Authorization", "Bearer "+token)
}
node, _, err := client.NewFullNodeRPC(context.Background(), addr, headers)
if err != nil {
return nil, fmt.Errorf("could not connect to api: %w", err)
}
return node, nil
}

54
tvx/state/serialize.go
View File

@ -1,54 +0,0 @@
package state
import (
"bytes"
"compress/gzip"
"context"
"fmt"
"github.com/filecoin-project/lotus/chain/state"
bs "github.com/filecoin-project/lotus/lib/blockstore"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-hamt-ipld"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/ipfs/go-ipld-format"
"github.com/ipld/go-car"
)
// RecoverStateTree parses a car encoding of a state tree back to a structured format
func RecoverStateTree(ctx context.Context, raw []byte, root cid.Cid) (*state.StateTree, error) {
buf := bytes.NewBuffer(raw)
store := bs.NewTemporary()
gr, err := gzip.NewReader(buf)
if err != nil {
return nil, err
}
defer gr.Close()
ch, err := car.LoadCar(store, gr)
if err != nil {
return nil, err
}
cborstore := cbor.NewCborStore(store)
fmt.Printf("roots are %v\n", ch.Roots)
nd, err := hamt.LoadNode(ctx, cborstore, root, hamt.UseTreeBitWidth(5))
if err != nil {
return nil, err
}
if err := nd.ForEach(ctx, func(k string, val interface{}) error {
n, ok := val.(format.Node)
if !ok {
fmt.Printf("hampt %s (not node): %+v\n", k, val)
} else {
fmt.Printf("%s: %#v\n", k, n)
}
return nil
}); err != nil {
return nil, err
}
return state.LoadStateTree(cborstore, root)
}

136
tvx/state/store.go
View File

@ -1,136 +0,0 @@
package state
import (
"context"
"log"
"sync"
"github.com/fatih/color"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/specs-actors/actors/util/adt"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
ds "github.com/ipfs/go-datastore"
exchange "github.com/ipfs/go-ipfs-exchange-interface"
offline "github.com/ipfs/go-ipfs-exchange-offline"
cbor "github.com/ipfs/go-ipld-cbor"
format "github.com/ipfs/go-ipld-format"
"github.com/ipfs/go-merkledag"
)
// Stores is a collection of the different stores and services that are needed
// to deal with the data layer of Filecoin, conveniently interlinked with one
// another.
type Stores struct {
CBORStore cbor.IpldStore
ADTStore adt.Store
Datastore ds.Batching
Blockstore blockstore.Blockstore
BlockService blockservice.BlockService
Exchange exchange.Interface
DAGService format.DAGService
}
func newStores(ctx context.Context, ds ds.Batching, bs blockstore.Blockstore) *Stores {
var (
cborstore = cbor.NewCborStore(bs)
offl = offline.Exchange(bs)
blkserv = blockservice.New(bs, offl)
dserv = merkledag.NewDAGService(blkserv)
)
return &Stores{
CBORStore: cborstore,
ADTStore: adt.WrapStore(ctx, cborstore),
Datastore: ds,
Blockstore: bs,
Exchange: offl,
BlockService: blkserv,
DAGService: dserv,
}
}
type proxyingBlockstore struct {
ctx context.Context
api api.FullNode
lk sync.RWMutex
tracing bool
traced map[cid.Cid]struct{}
blockstore.Blockstore
}
type TracingBlockstore interface {
StartTracing()
FinishTracing() map[cid.Cid]struct{}
}
var _ TracingBlockstore = (*proxyingBlockstore)(nil)
// StartTracing starts tracing the CIDs that are effectively fetched during the
// processing of a message.
func (pb *proxyingBlockstore) StartTracing() {
pb.lk.Lock()
pb.tracing = true
pb.traced = map[cid.Cid]struct{}{}
pb.lk.Unlock()
}
// FinishTracing finishes tracing accessed CIDs, and returns a map of the
// CIDs that were traced.
func (pb *proxyingBlockstore) FinishTracing() map[cid.Cid]struct{} {
pb.lk.Lock()
ret := pb.traced
pb.tracing = false
pb.traced = map[cid.Cid]struct{}{}
pb.lk.Unlock()
return ret
}
func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
pb.lk.RLock()
if pb.tracing {
pb.traced[cid] = struct{}{}
}
pb.lk.RUnlock()
if block, err := pb.Blockstore.Get(cid); err == nil {
return block, err
}
log.Println(color.CyanString("fetching cid via rpc: %v", cid))
item, err := pb.api.ChainReadObj(pb.ctx, cid)
if err != nil {
return nil, err
}
block, err := blocks.NewBlockWithCid(item, cid)
if err != nil {
return nil, err
}
err = pb.Blockstore.Put(block)
if err != nil {
return nil, err
}
return block, nil
}
// NewProxyingStores is a Stores that proxies get requests for unknown CIDs
// to a Filecoin node, via the ChainReadObj RPC.
func NewProxyingStores(ctx context.Context, api api.FullNode) *Stores {
ds := ds.NewMapDatastore()
bs := &proxyingBlockstore{
ctx: ctx,
api: api,
Blockstore: blockstore.NewBlockstore(ds),
}
return newStores(ctx, ds, bs)
}

301
tvx/state/surgeon.go
View File

@ -1,301 +0,0 @@
package state
import (
"context"
"fmt"
"io"
"log"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/api"
init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init"
"github.com/filecoin-project/lotus/chain/state"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-ipld-format"
"github.com/ipld/go-car"
cbg "github.com/whyrusleeping/cbor-gen"
)
// Surgeon is an object used to fetch and manipulate state.
type Surgeon struct {
ctx context.Context
api api.FullNode
stores *Stores
}
// NewSurgeon returns a state surgeon, an object used to fetch and manipulate
// state.
func NewSurgeon(ctx context.Context, api api.FullNode, stores *Stores) *Surgeon {
return &Surgeon{
ctx: ctx,
api: api,
stores: stores,
}
}
func (sg *Surgeon) GetStateTreeRootFromTipset(tsk types.TipSetKey) (cid.Cid, error) {
ts, err := sg.api.ChainGetTipSet(sg.ctx, tsk)
if err != nil {
return cid.Undef, err
}
return ts.ParentState(), nil
}
// GetMaskedStateTree trims the state tree at the supplied tipset to contain
// only the state of the actors in the retain set. It also "dives" into some
// singleton system actors, like the init actor, to trim the state so as to
// compute a minimal state tree. In the future, thid method will dive into
// other system actors like the power actor and the market actor.
func (sg *Surgeon) GetMaskedStateTree(previousRoot cid.Cid, retain []address.Address) (cid.Cid, error) {
// TODO: this will need to be parameterized on network version.
st, err := state.LoadStateTree(sg.stores.CBORStore, previousRoot)
if err != nil {
return cid.Undef, err
}
initActor, initState, err := sg.loadInitActor(st)
if err != nil {
return cid.Undef, err
}
err = sg.retainInitEntries(initState, retain)
if err != nil {
return cid.Undef, err
}
err = sg.saveInitActor(initActor, initState, st)
if err != nil {
return cid.Undef, err
}
// resolve all addresses to ID addresses.
resolved, err := sg.resolveAddresses(retain, initState)
if err != nil {
return cid.Undef, err
}
st, err = sg.transplantActors(st, resolved)
if err != nil {
return cid.Undef, err
}
root, err := st.Flush(sg.ctx)
if err != nil {
return cid.Undef, err
}
return root, nil
}
// GetAccessedActors identifies the actors that were accessed during the
// execution of a message.
func (sg *Surgeon) GetAccessedActors(ctx context.Context, a api.FullNode, mid cid.Cid) ([]address.Address, error) {
log.Printf("calculating accessed actors during execution of message: %s", mid)
msgInfo, err := a.StateSearchMsg(ctx, mid)
if err != nil {
return nil, err
}
if msgInfo == nil {
return nil, fmt.Errorf("message info is nil")
}
msgObj, err := a.ChainGetMessage(ctx, mid)
if err != nil {
return nil, err
}
ts, err := a.ChainGetTipSet(ctx, msgInfo.TipSet)
if err != nil {
return nil, err
}
trace, err := a.StateCall(ctx, msgObj, ts.Parents())
if err != nil {
return nil, fmt.Errorf("could not replay msg: %w", err)
}
accessed := make(map[address.Address]struct{})
var recur func(trace *types.ExecutionTrace)
recur = func(trace *types.ExecutionTrace) {
accessed[trace.Msg.To] = struct{}{}
accessed[trace.Msg.From] = struct{}{}
for _, s := range trace.Subcalls {
recur(&s)
}
}
recur(&trace.ExecutionTrace)
ret := make([]address.Address, 0, len(accessed))
for k := range accessed {
ret = append(ret, k)
}
return ret, nil
}
// WriteCAR recursively writes the tree referenced by the root as a CAR into the
// supplied io.Writer.
func (sg *Surgeon) WriteCAR(w io.Writer, roots ...cid.Cid) error {
carWalkFn := func(nd format.Node) (out []*format.Link, err error) {
for _, link := range nd.Links() {
if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
continue
}
out = append(out, link)
}
return out, nil
}
return car.WriteCarWithWalker(sg.ctx, sg.stores.DAGService, roots, w, carWalkFn)
}
// WriteCARIncluding writes a CAR including only the CIDs that are listed in
// the include set. This leads to an intentially sparse tree with dangling links.
func (sg *Surgeon) WriteCARIncluding(w io.Writer, include map[cid.Cid]struct{}, roots ...cid.Cid) error {
carWalkFn := func(nd format.Node) (out []*format.Link, err error) {
for _, link := range nd.Links() {
if _, ok := include[link.Cid]; !ok {
continue
}
if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
continue
}
out = append(out, link)
}
return out, nil
}
return car.WriteCarWithWalker(sg.ctx, sg.stores.DAGService, roots, w, carWalkFn)
}
// transplantActors plucks the state from the supplied actors at the given
// tipset, and places it into the supplied state map.
func (sg *Surgeon) transplantActors(src *state.StateTree, pluck []address.Address) (*state.StateTree, error) {
log.Printf("transplanting actor states: %v", pluck)
dst, err := state.NewStateTree(sg.stores.CBORStore, src.Version())
if err != nil {
return nil, err
}
for _, a := range pluck {
actor, err := src.GetActor(a)
if err != nil {
return nil, fmt.Errorf("get actor %s failed: %w", a, err)
}
err = dst.SetActor(a, actor)
if err != nil {
return nil, err
}
// recursive copy of the actor state.
err = vm.Copy(context.TODO(), sg.stores.Blockstore, sg.stores.Blockstore, actor.Head)
if err != nil {
return nil, err
}
actorState, err := sg.api.ChainReadObj(sg.ctx, actor.Head)
if err != nil {
return nil, err
}
cid, err := sg.stores.CBORStore.Put(sg.ctx, &cbg.Deferred{Raw: actorState})
if err != nil {
return nil, err
}
if cid != actor.Head {
panic("mismatched cids")
}
}
return dst, nil
}
// saveInitActor saves the state of the init actor to the provided state map.
func (sg *Surgeon) saveInitActor(initActor *types.Actor, initState init_.State, st *state.StateTree) error {
log.Printf("saving init actor into state tree")
// Store the state of the init actor.
cid, err := sg.stores.CBORStore.Put(sg.ctx, initState)
if err != nil {
return err
}
actor := *initActor
actor.Head = cid
err = st.SetActor(init_.Address, &actor)
if err != nil {
return err
}
cid, _ = st.Flush(sg.ctx)
log.Printf("saved init actor into state tree; new root: %s", cid)
return nil
}
// retainInitEntries takes an old init actor state, and retains only the
// entries in the retain set, returning a new init actor state.
func (sg *Surgeon) retainInitEntries(state init_.State, retain []address.Address) error {
log.Printf("retaining init actor entries for addresses: %v", retain)
m := make(map[address.Address]struct{}, len(retain))
for _, a := range retain {
m[a] = struct{}{}
}
var remove []address.Address
_ = state.ForEachActor(func(id abi.ActorID, address address.Address) error {
if _, ok := m[address]; !ok {
remove = append(remove, address)
}
return nil
})
err := state.Remove(remove...)
log.Printf("new init actor state: %+v", state)
return err
}
// resolveAddresses resolved the requested addresses from the provided
// InitActor state, returning a slice of length len(orig), where each index
// contains the resolved address.
func (sg *Surgeon) resolveAddresses(orig []address.Address, ist init_.State) (ret []address.Address, err error) {
log.Printf("resolving addresses: %v", orig)
ret = make([]address.Address, len(orig))
for i, addr := range orig {
resolved, found, err := ist.ResolveAddress(addr)
if err != nil {
return nil, err
}
if !found {
return nil, fmt.Errorf("address not found: %s", addr)
}
ret[i] = resolved
}
log.Printf("resolved addresses: %v", ret)
return ret, nil
}
// loadInitActor loads the init actor state from a given tipset.
func (sg *Surgeon) loadInitActor(st *state.StateTree) (*types.Actor, init_.State, error) {
actor, err := st.GetActor(init_.Address)
if err != nil {
return nil, nil, err
}
initState, err := init_.Load(sg.stores.ADTStore, actor)
if err != nil {
return nil, nil, err
}
log.Printf("loaded init actor state: %+v", initState)
return actor, initState, nil
}

97
tvx/state_delta.go
View File

@ -1,97 +0,0 @@
package main
import (
"context"
"fmt"
"github.com/filecoin-project/lotus/chain/types"
"github.com/ipfs/go-cid"
"github.com/urfave/cli/v2"
)
var deltaFlags struct {
from string
to string
}
var deltaCmd = &cli.Command{
Name: "state-delta",
Description: "collect affected state between two tipsets, addressed by blocks",
Action: runStateDelta,
Flags: []cli.Flag{
&apiFlag,
&cli.StringFlag{
Name: "from",
Usage: "block CID of initial state",
Required: true,
Destination: &deltaFlags.from,
},
&cli.StringFlag{
Name: "to",
Usage: "block CID of ending state",
Required: true,
Destination: &deltaFlags.to,
},
},
}
func runStateDelta(c *cli.Context) error {
node, err := makeClient(c)
if err != nil {
return err
}
from, err := cid.Decode(deltaFlags.from)
if err != nil {
return err
}
to, err := cid.Decode(deltaFlags.to)
if err != nil {
return err
}
currBlock, err := node.ChainGetBlock(context.TODO(), to)
if err != nil {
return err
}
srcBlock, err := node.ChainGetBlock(context.TODO(), from)
if err != nil {
return err
}
allMsgs := make(map[uint64][]*types.Message)
epochs := currBlock.Height - srcBlock.Height - 1
for epochs > 0 {
msgs, err := node.ChainGetBlockMessages(context.TODO(), to)
if err != nil {
return err
}
allMsgs[uint64(currBlock.Height)] = msgs.BlsMessages
currBlock, err = node.ChainGetBlock(context.TODO(), currBlock.Parents[0])
epochs--
}
if !hasParent(currBlock, from) {
return fmt.Errorf("from block was not a parent of `to` as expected")
}
m := 0
for _, msgs := range allMsgs {
m += len(msgs)
}
fmt.Printf("messages: %d\n", m)
fmt.Printf("initial state root: %v\n", currBlock.ParentStateRoot)
return nil
}
func hasParent(block *types.BlockHeader, parent cid.Cid) bool {
for _, p := range block.Parents {
if p.Equals(parent) {
return true
}
}
return false
}