Merge pull request #4064 from filecoin-project/tvx
tvx: a test vector extraction and execution tool
This commit is contained in:
Łukasz Magiera
GitHub
commit
5bffea6f54
1
.github/CODEOWNERS
vendored
1
.github/CODEOWNERS
vendored
@ -13,3 +13,4 @@
|
||||
### Conformance testing.
|
||||
conformance/ @raulk
|
||||
extern/test-vectors @raulk
|
||||
cmd/tvx @raulk
|
||||
44
cmd/tvx/actor_mapping.go
Normal file
44
cmd/tvx/actor_mapping.go
Normal file
@ -0,0 +1,44 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
|
||||
"github.com/filecoin-project/specs-actors/actors/builtin"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/multiformats/go-multihash"
|
||||
)
|
||||
|
||||
var ActorMethodTable = make(map[string][]string, 64)
|
||||
|
||||
var Actors = map[cid.Cid]interface{}{
|
||||
builtin.InitActorCodeID: builtin.MethodsInit,
|
||||
builtin.CronActorCodeID: builtin.MethodsCron,
|
||||
builtin.AccountActorCodeID: builtin.MethodsAccount,
|
||||
builtin.StoragePowerActorCodeID: builtin.MethodsPower,
|
||||
builtin.StorageMinerActorCodeID: builtin.MethodsMiner,
|
||||
builtin.StorageMarketActorCodeID: builtin.MethodsMarket,
|
||||
builtin.PaymentChannelActorCodeID: builtin.MethodsPaych,
|
||||
builtin.MultisigActorCodeID: builtin.MethodsMultisig,
|
||||
builtin.RewardActorCodeID: builtin.MethodsReward,
|
||||
builtin.VerifiedRegistryActorCodeID: builtin.MethodsVerifiedRegistry,
|
||||
}
|
||||
|
||||
func init() {
|
||||
for code, methods := range Actors {
|
||||
cmh, err := multihash.Decode(code.Hash()) // identity hash.
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
var (
|
||||
aname = string(cmh.Digest)
|
||||
rt = reflect.TypeOf(methods)
|
||||
nf = rt.NumField()
|
||||
)
|
||||
|
||||
ActorMethodTable[aname] = append(ActorMethodTable[aname], "Send")
|
||||
for i := 0; i < nf; i++ {
|
||||
ActorMethodTable[aname] = append(ActorMethodTable[aname], rt.Field(i).Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
92
cmd/tvx/exec.go
Normal file
92
cmd/tvx/exec.go
Normal file
@ -0,0 +1,92 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/urfave/cli/v2"
|
||||
|
||||
"github.com/filecoin-project/lotus/conformance"
|
||||
|
||||
"github.com/filecoin-project/test-vectors/schema"
|
||||
)
|
||||
|
||||
var execFlags struct {
|
||||
file string
|
||||
}
|
||||
|
||||
var execCmd = &cli.Command{
|
||||
Name: "exec",
|
||||
Description: "execute one or many test vectors against Lotus; supplied as a single JSON file, or a ndjson stdin stream",
|
||||
Action: runExecLotus,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "file",
|
||||
Usage: "input file; if not supplied, the vector will be read from stdin",
|
||||
TakesFile: true,
|
||||
Destination: &execFlags.file,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func runExecLotus(_ *cli.Context) error {
|
||||
if file := execFlags.file; file != "" {
|
||||
// we have a single test vector supplied as a file.
|
||||
file, err := os.Open(file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open test vector: %w", err)
|
||||
}
|
||||
|
||||
var (
|
||||
dec = json.NewDecoder(file)
|
||||
tv schema.TestVector
|
||||
)
|
||||
|
||||
if err = dec.Decode(&tv); err != nil {
|
||||
return fmt.Errorf("failed to decode test vector: %w", err)
|
||||
}
|
||||
|
||||
return executeTestVector(tv)
|
||||
}
|
||||
|
||||
for dec := json.NewDecoder(os.Stdin); ; {
|
||||
var tv schema.TestVector
|
||||
switch err := dec.Decode(&tv); err {
|
||||
case nil:
|
||||
if err = executeTestVector(tv); err != nil {
|
||||
return err
|
||||
}
|
||||
case io.EOF:
|
||||
// we're done.
|
||||
return nil
|
||||
default:
|
||||
// something bad happened.
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func executeTestVector(tv schema.TestVector) error {
|
||||
log.Println("executing test vector:", tv.Meta.ID)
|
||||
r := new(conformance.LogReporter)
|
||||
switch class := tv.Class; class {
|
||||
case "message":
|
||||
conformance.ExecuteMessageVector(r, &tv)
|
||||
case "tipset":
|
||||
conformance.ExecuteTipsetVector(r, &tv)
|
||||
default:
|
||||
return fmt.Errorf("test vector class %s not supported", class)
|
||||
}
|
||||
|
||||
if r.Failed() {
|
||||
log.Println(color.HiRedString("❌ test vector failed"))
|
||||
} else {
|
||||
log.Println(color.GreenString("✅ test vector succeeded"))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
503
cmd/tvx/extract.go
Normal file
503
cmd/tvx/extract.go
Normal file
@ -0,0 +1,503 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/fatih/color"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/reward"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/chain/vm"
|
||||
lcli "github.com/filecoin-project/lotus/cli"
|
||||
"github.com/filecoin-project/lotus/conformance"
|
||||
|
||||
"github.com/filecoin-project/specs-actors/actors/builtin"
|
||||
|
||||
"github.com/filecoin-project/test-vectors/schema"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
const (
|
||||
PrecursorSelectAll = "all"
|
||||
PrecursorSelectSender = "sender"
|
||||
)
|
||||
|
||||
type extractOpts struct {
|
||||
id string
|
||||
block string
|
||||
class string
|
||||
cid string
|
||||
file string
|
||||
retain string
|
||||
precursor string
|
||||
}
|
||||
|
||||
var extractFlags extractOpts
|
||||
|
||||
var extractCmd = &cli.Command{
|
||||
Name: "extract",
|
||||
Description: "generate a test vector by extracting it from a live chain",
|
||||
Action: runExtract,
|
||||
Flags: []cli.Flag{
|
||||
&repoFlag,
|
||||
&cli.StringFlag{
|
||||
Name: "class",
|
||||
Usage: "class of vector to extract; other required flags depend on the; values: 'message'",
|
||||
Value: "message",
|
||||
Destination: &extractFlags.class,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "id",
|
||||
Usage: "identifier to name this test vector with",
|
||||
Value: "(undefined)",
|
||||
Destination: &extractFlags.id,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "block",
|
||||
Usage: "optionally, the block CID the message was included in, to avoid expensive chain scanning",
|
||||
Destination: &extractFlags.block,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "cid",
|
||||
Usage: "message CID to generate test vector from",
|
||||
Required: true,
|
||||
Destination: &extractFlags.cid,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "out",
|
||||
Aliases: []string{"o"},
|
||||
Usage: "file to write test vector to",
|
||||
Destination: &extractFlags.file,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "state-retain",
|
||||
Usage: "state retention policy; values: 'accessed-cids', 'accessed-actors'",
|
||||
Value: "accessed-cids",
|
||||
Destination: &extractFlags.retain,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "precursor-select",
|
||||
Usage: "precursors to apply; values: 'all', 'sender'; 'all' selects all preceding" +
|
||||
"messages in the canonicalised tipset, 'sender' selects only preceding messages from the same" +
|
||||
"sender. Usually, 'sender' is a good tradeoff and gives you sufficient accuracy. If the receipt sanity" +
|
||||
"check fails due to gas reasons, switch to 'all', as previous messages in the tipset may have" +
|
||||
"affected state in a disruptive way",
|
||||
Value: "sender",
|
||||
Destination: &extractFlags.precursor,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func runExtract(c *cli.Context) error {
|
||||
// LOTUS_DISABLE_VM_BUF disables what's called "VM state tree buffering",
|
||||
// which stashes write operations in a BufferedBlockstore
|
||||
// (https://github.com/filecoin-project/lotus/blob/b7a4dbb07fd8332b4492313a617e3458f8003b2a/lib/bufbstore/buf_bstore.go#L21)
|
||||
// such that they're not written until the VM is actually flushed.
|
||||
//
|
||||
// For some reason, the standard behaviour was not working for me (raulk),
|
||||
// and disabling it (such that the state transformations are written immediately
|
||||
// to the blockstore) worked.
|
||||
_ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea")
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Make the API client.
|
||||
fapi, closer, err := lcli.GetFullNodeAPI(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
|
||||
return doExtract(ctx, fapi, extractFlags)
|
||||
}
|
||||
|
||||
func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
|
||||
mcid, err := cid.Decode(opts.cid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
msg, execTs, incTs, err := resolveFromChain(ctx, fapi, mcid, opts.block)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to resolve message and tipsets from chain: %w", err)
|
||||
}
|
||||
|
||||
// get the circulating supply before the message was executed.
|
||||
circSupplyDetail, err := fapi.StateCirculatingSupply(ctx, incTs.Key())
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed while fetching circulating supply: %w", err)
|
||||
}
|
||||
|
||||
circSupply := circSupplyDetail.FilCirculating
|
||||
|
||||
log.Printf("message was executed in tipset: %s", execTs.Key())
|
||||
log.Printf("message was included in tipset: %s", incTs.Key())
|
||||
log.Printf("circulating supply at inclusion tipset: %d", circSupply)
|
||||
log.Printf("finding precursor messages using mode: %s", opts.precursor)
|
||||
|
||||
// Fetch messages in canonical order from inclusion tipset.
|
||||
msgs, err := fapi.ChainGetParentMessages(ctx, execTs.Blocks()[0].Cid())
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to fetch messages in canonical order from inclusion tipset: %w", err)
|
||||
}
|
||||
|
||||
related, found, err := findMsgAndPrecursors(opts.precursor, msg, msgs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed while finding message and precursors: %w", err)
|
||||
}
|
||||
|
||||
if !found {
|
||||
return fmt.Errorf("message not found; precursors found: %d", len(related))
|
||||
}
|
||||
|
||||
var (
|
||||
precursors = related[:len(related)-1]
|
||||
precursorsCids []cid.Cid
|
||||
)
|
||||
|
||||
for _, p := range precursors {
|
||||
precursorsCids = append(precursorsCids, p.Cid())
|
||||
}
|
||||
|
||||
log.Println(color.GreenString("found message; precursors (count: %d): %v", len(precursors), precursorsCids))
|
||||
|
||||
var (
|
||||
// create a read-through store that uses ChainGetObject to fetch unknown CIDs.
|
||||
pst = NewProxyingStores(ctx, fapi)
|
||||
g = NewSurgeon(ctx, fapi, pst)
|
||||
)
|
||||
|
||||
driver := conformance.NewDriver(ctx, schema.Selector{}, conformance.DriverOpts{
|
||||
DisableVMFlush: true,
|
||||
})
|
||||
|
||||
// this is the root of the state tree we start with.
|
||||
root := incTs.ParentState()
|
||||
log.Printf("base state tree root CID: %s", root)
|
||||
|
||||
basefee := incTs.Blocks()[0].ParentBaseFee
|
||||
log.Printf("basefee: %s", basefee)
|
||||
|
||||
// on top of that state tree, we apply all precursors.
|
||||
log.Printf("number of precursors to apply: %d", len(precursors))
|
||||
for i, m := range precursors {
|
||||
log.Printf("applying precursor %d, cid: %s", i, m.Cid())
|
||||
_, root, err = driver.ExecuteMessage(pst.Blockstore, conformance.ExecuteMessageParams{
|
||||
Preroot: root,
|
||||
Epoch: execTs.Height(),
|
||||
Message: m,
|
||||
CircSupply: &circSupplyDetail.FilCirculating,
|
||||
BaseFee: &basefee,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to execute precursor message: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
preroot cid.Cid
|
||||
postroot cid.Cid
|
||||
applyret *vm.ApplyRet
|
||||
carWriter func(w io.Writer) error
|
||||
retention = opts.retain
|
||||
)
|
||||
|
||||
log.Printf("using state retention strategy: %s", retention)
|
||||
switch retention {
|
||||
case "accessed-cids":
|
||||
tbs, ok := pst.Blockstore.(TracingBlockstore)
|
||||
if !ok {
|
||||
return fmt.Errorf("requested 'accessed-cids' state retention, but no tracing blockstore was present")
|
||||
}
|
||||
|
||||
tbs.StartTracing()
|
||||
|
||||
preroot = root
|
||||
applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, conformance.ExecuteMessageParams{
|
||||
Preroot: preroot,
|
||||
Epoch: execTs.Height(),
|
||||
Message: msg,
|
||||
CircSupply: &circSupplyDetail.FilCirculating,
|
||||
BaseFee: &basefee,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to execute message: %w", err)
|
||||
}
|
||||
accessed := tbs.FinishTracing()
|
||||
carWriter = func(w io.Writer) error {
|
||||
return g.WriteCARIncluding(w, accessed, preroot, postroot)
|
||||
}
|
||||
|
||||
case "accessed-actors":
|
||||
log.Printf("calculating accessed actors")
|
||||
// get actors accessed by message.
|
||||
retain, err := g.GetAccessedActors(ctx, fapi, mcid)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to calculate accessed actors: %w", err)
|
||||
}
|
||||
// also append the reward actor and the burnt funds actor.
|
||||
retain = append(retain, reward.Address, builtin.BurntFundsActorAddr, init_.Address)
|
||||
log.Printf("calculated accessed actors: %v", retain)
|
||||
|
||||
// get the masked state tree from the root,
|
||||
preroot, err = g.GetMaskedStateTree(root, retain)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, conformance.ExecuteMessageParams{
|
||||
Preroot: preroot,
|
||||
Epoch: execTs.Height(),
|
||||
Message: msg,
|
||||
CircSupply: &circSupplyDetail.FilCirculating,
|
||||
BaseFee: &basefee,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to execute message: %w", err)
|
||||
}
|
||||
carWriter = func(w io.Writer) error {
|
||||
return g.WriteCAR(w, preroot, postroot)
|
||||
}
|
||||
|
||||
default:
|
||||
return fmt.Errorf("unknown state retention option: %s", retention)
|
||||
}
|
||||
|
||||
log.Printf("message applied; preroot: %s, postroot: %s", preroot, postroot)
|
||||
log.Println("performing sanity check on receipt")
|
||||
|
||||
// TODO sometimes this returns a nil receipt and no error ¯\_(ツ)_/¯
|
||||
// ex: https://filfox.info/en/message/bafy2bzacebpxw3yiaxzy2bako62akig46x3imji7fewszen6fryiz6nymu2b2
|
||||
// This code is lenient and skips receipt comparison in case of a nil receipt.
|
||||
rec, err := fapi.StateGetReceipt(ctx, mcid, execTs.Key())
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to find receipt on chain: %w", err)
|
||||
}
|
||||
log.Printf("found receipt: %+v", rec)
|
||||
|
||||
// generate the schema receipt; if we got
|
||||
var receipt *schema.Receipt
|
||||
if rec != nil {
|
||||
receipt = &schema.Receipt{
|
||||
ExitCode: int64(rec.ExitCode),
|
||||
ReturnValue: rec.Return,
|
||||
GasUsed: rec.GasUsed,
|
||||
}
|
||||
reporter := new(conformance.LogReporter)
|
||||
conformance.AssertMsgResult(reporter, receipt, applyret, "as locally executed")
|
||||
if reporter.Failed() {
|
||||
log.Println(color.RedString("receipt sanity check failed; aborting"))
|
||||
return fmt.Errorf("vector generation aborted")
|
||||
}
|
||||
log.Println(color.GreenString("receipt sanity check succeeded"))
|
||||
} else {
|
||||
receipt = &schema.Receipt{
|
||||
ExitCode: int64(applyret.ExitCode),
|
||||
ReturnValue: applyret.Return,
|
||||
GasUsed: applyret.GasUsed,
|
||||
}
|
||||
log.Println(color.YellowString("skipping receipts comparison; we got back a nil receipt from lotus"))
|
||||
}
|
||||
|
||||
log.Println("generating vector")
|
||||
msgBytes, err := msg.Serialize()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var (
|
||||
out = new(bytes.Buffer)
|
||||
gw = gzip.NewWriter(out)
|
||||
)
|
||||
if err := carWriter(gw); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = gw.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = gw.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
version, err := fapi.Version(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ntwkName, err := fapi.StateNetworkName(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Write out the test vector.
|
||||
vector := schema.TestVector{
|
||||
Class: schema.ClassMessage,
|
||||
Meta: &schema.Metadata{
|
||||
ID: opts.id,
|
||||
// TODO need to replace schema.GenerationData with a more flexible
|
||||
// data structure that makes no assumption about the traceability
|
||||
// data that's being recorded; a flexible map[string]string
|
||||
// would do.
|
||||
Gen: []schema.GenerationData{
|
||||
{Source: fmt.Sprintf("network:%s", ntwkName)},
|
||||
{Source: fmt.Sprintf("message:%s", msg.Cid().String())},
|
||||
{Source: fmt.Sprintf("inclusion_tipset:%s", incTs.Key().String())},
|
||||
{Source: fmt.Sprintf("execution_tipset:%s", execTs.Key().String())},
|
||||
{Source: "github.com/filecoin-project/lotus", Version: version.String()}},
|
||||
},
|
||||
CAR: out.Bytes(),
|
||||
Pre: &schema.Preconditions{
|
||||
Epoch: int64(execTs.Height()),
|
||||
CircSupply: circSupply.Int,
|
||||
BaseFee: basefee.Int,
|
||||
StateTree: &schema.StateTree{
|
||||
RootCID: preroot,
|
||||
},
|
||||
},
|
||||
ApplyMessages: []schema.Message{{Bytes: msgBytes}},
|
||||
Post: &schema.Postconditions{
|
||||
StateTree: &schema.StateTree{
|
||||
RootCID: postroot,
|
||||
},
|
||||
Receipts: []*schema.Receipt{
|
||||
{
|
||||
ExitCode: int64(applyret.ExitCode),
|
||||
ReturnValue: applyret.Return,
|
||||
GasUsed: applyret.GasUsed,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
output := io.WriteCloser(os.Stdout)
|
||||
if file := opts.file; file != "" {
|
||||
dir := filepath.Dir(file)
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return fmt.Errorf("unable to create directory %s: %w", dir, err)
|
||||
}
|
||||
output, err = os.Create(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer output.Close() //nolint:errcheck
|
||||
defer log.Printf("wrote test vector to file: %s", file)
|
||||
}
|
||||
|
||||
enc := json.NewEncoder(output)
|
||||
enc.SetIndent("", " ")
|
||||
if err := enc.Encode(&vector); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolveFromChain queries the chain for the provided message, using the block CID to
|
||||
// speed up the query, if provided
|
||||
func resolveFromChain(ctx context.Context, api api.FullNode, mcid cid.Cid, block string) (msg *types.Message, execTs *types.TipSet, incTs *types.TipSet, err error) {
|
||||
// Extract the full message.
|
||||
msg, err = api.ChainGetMessage(ctx, mcid)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
log.Printf("found message with CID %s: %+v", mcid, msg)
|
||||
|
||||
if block == "" {
|
||||
log.Printf("locating message in blockchain")
|
||||
|
||||
// Locate the message.
|
||||
msgInfo, err := api.StateSearchMsg(ctx, mcid)
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("failed to locate message: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("located message at tipset %s (height: %d) with exit code: %s", msgInfo.TipSet, msgInfo.Height, msgInfo.Receipt.ExitCode)
|
||||
|
||||
execTs, incTs, err = fetchThisAndPrevTipset(ctx, api, msgInfo.TipSet)
|
||||
return msg, execTs, incTs, err
|
||||
}
|
||||
|
||||
bcid, err := cid.Decode(block)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
log.Printf("message inclusion block CID was provided; scanning around it: %s", bcid)
|
||||
|
||||
blk, err := api.ChainGetBlock(ctx, bcid)
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("failed to get block: %w", err)
|
||||
}
|
||||
|
||||
// types.EmptyTSK hints to use the HEAD.
|
||||
execTs, err = api.ChainGetTipSetByHeight(ctx, blk.Height+1, types.EmptyTSK)
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("failed to get message execution tipset: %w", err)
|
||||
}
|
||||
|
||||
// walk back from the execTs instead of HEAD, to save time.
|
||||
incTs, err = api.ChainGetTipSetByHeight(ctx, blk.Height, execTs.Key())
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("failed to get message inclusion tipset: %w", err)
|
||||
}
|
||||
|
||||
return msg, execTs, incTs, nil
|
||||
}
|
||||
|
||||
// fetchThisAndPrevTipset returns the full tipset identified by the key, as well
|
||||
// as the previous tipset. In the context of vector generation, the target
|
||||
// tipset is the one where a message was executed, and the previous tipset is
|
||||
// the one where the message was included.
|
||||
func fetchThisAndPrevTipset(ctx context.Context, api api.FullNode, target types.TipSetKey) (targetTs *types.TipSet, prevTs *types.TipSet, err error) {
|
||||
// get the tipset on which this message was "executed" on.
|
||||
// https://github.com/filecoin-project/lotus/issues/2847
|
||||
targetTs, err = api.ChainGetTipSet(ctx, target)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// get the previous tipset, on which this message was mined,
|
||||
// i.e. included on-chain.
|
||||
prevTs, err = api.ChainGetTipSet(ctx, targetTs.Parents())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return targetTs, prevTs, nil
|
||||
}
|
||||
|
||||
// findMsgAndPrecursors ranges through the canonical messages slice, locating
|
||||
// the target message and returning precursors in accordance to the supplied
|
||||
// mode.
|
||||
func findMsgAndPrecursors(mode string, target *types.Message, msgs []api.Message) (related []*types.Message, found bool, err error) {
|
||||
// Range through canonicalised messages, selecting only the precursors based
|
||||
// on selection mode.
|
||||
for _, other := range msgs {
|
||||
switch {
|
||||
case mode == PrecursorSelectAll:
|
||||
fallthrough
|
||||
case mode == PrecursorSelectSender && other.Message.From == target.From:
|
||||
related = append(related, other.Message)
|
||||
}
|
||||
|
||||
// this message is the target; we're done.
|
||||
if other.Cid == target.Cid() {
|
||||
return related, true, nil
|
||||
}
|
||||
}
|
||||
|
||||
// this could happen because a block contained related messages, but not
|
||||
// the target (that is, messages with a lower nonce, but ultimately not the
|
||||
// target).
|
||||
return related, false, nil
|
||||
}
|
||||
232
cmd/tvx/extract_many.go
Normal file
232
cmd/tvx/extract_many.go
Normal file
@ -0,0 +1,232 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/filecoin-project/go-state-types/exitcode"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/urfave/cli/v2"
|
||||
|
||||
lcli "github.com/filecoin-project/lotus/cli"
|
||||
)
|
||||
|
||||
var extractManyFlags struct {
|
||||
in string
|
||||
outdir string
|
||||
batchId string
|
||||
}
|
||||
|
||||
var extractManyCmd = &cli.Command{
|
||||
Name: "extract-many",
|
||||
Description: `generate many test vectors by repeatedly calling tvx extract, using a csv file as input.
|
||||
|
||||
The CSV file must have a format just like the following:
|
||||
|
||||
message_cid,receiver_code,method_num,exit_code,height,block_cid,seq
|
||||
bafy2bzacedvuvgpsnwq7i7kltfap6hnp7fdmzf6lr4w34zycjrthb3v7k6zi6,fil/1/account,0,0,67972,bafy2bzacebthpxzlk7zhlkz3jfzl4qw7mdoswcxlf3rkof3b4mbxfj3qzfk7w,1
|
||||
bafy2bzacedwicofymn4imgny2hhbmcm4o5bikwnv3qqgohyx73fbtopiqlro6,fil/1/account,0,0,67860,bafy2bzacebj7beoxyzll522o6o76mt7von4psn3tlvunokhv4zhpwmfpipgti,2
|
||||
...
|
||||
|
||||
The first row MUST be a header row. At the bare minimum, those seven fields
|
||||
must appear, in the order specified. Extra fields are accepted, but always
|
||||
after these compulsory seven.
|
||||
`,
|
||||
Action: runExtractMany,
|
||||
Flags: []cli.Flag{
|
||||
&repoFlag,
|
||||
&cli.StringFlag{
|
||||
Name: "batch-id",
|
||||
Usage: "batch id; a four-digit left-zero-padded sequential number (e.g. 0041)",
|
||||
Required: true,
|
||||
Destination: &extractManyFlags.batchId,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "in",
|
||||
Usage: "path to input file (csv)",
|
||||
Destination: &extractManyFlags.in,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "outdir",
|
||||
Usage: "output directory",
|
||||
Destination: &extractManyFlags.outdir,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func runExtractMany(c *cli.Context) error {
|
||||
// LOTUS_DISABLE_VM_BUF disables what's called "VM state tree buffering",
|
||||
// which stashes write operations in a BufferedBlockstore
|
||||
// (https://github.com/filecoin-project/lotus/blob/b7a4dbb07fd8332b4492313a617e3458f8003b2a/lib/bufbstore/buf_bstore.go#L21)
|
||||
// such that they're not written until the VM is actually flushed.
|
||||
//
|
||||
// For some reason, the standard behaviour was not working for me (raulk),
|
||||
// and disabling it (such that the state transformations are written immediately
|
||||
// to the blockstore) worked.
|
||||
_ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea")
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Make the API client.
|
||||
fapi, closer, err := lcli.GetFullNodeAPI(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
|
||||
var (
|
||||
in = extractManyFlags.in
|
||||
outdir = extractManyFlags.outdir
|
||||
)
|
||||
|
||||
if in == "" {
|
||||
return fmt.Errorf("input file not provided")
|
||||
}
|
||||
|
||||
if outdir == "" {
|
||||
return fmt.Errorf("output dir not provided")
|
||||
}
|
||||
|
||||
// Open the CSV file for reading.
|
||||
f, err := os.Open(in)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open file %s: %w", in, err)
|
||||
}
|
||||
|
||||
// Ensure the output directory exists.
|
||||
if err := os.MkdirAll(outdir, 0755); err != nil {
|
||||
return fmt.Errorf("could not create output dir %s: %w", outdir, err)
|
||||
}
|
||||
|
||||
// Create a CSV reader and validate the header row.
|
||||
reader := csv.NewReader(f)
|
||||
if header, err := reader.Read(); err != nil {
|
||||
return fmt.Errorf("failed to read header from csv: %w", err)
|
||||
} else if l := len(header); l < 7 {
|
||||
return fmt.Errorf("insufficient number of fields: %d", l)
|
||||
} else if f := header[0]; f != "message_cid" {
|
||||
return fmt.Errorf("csv sanity check failed: expected first field in header to be 'message_cid'; was: %s", f)
|
||||
} else {
|
||||
log.Println(color.GreenString("csv sanity check succeeded; header contains fields: %v", header))
|
||||
}
|
||||
|
||||
var (
|
||||
generated []string
|
||||
merr = new(multierror.Error)
|
||||
retry []extractOpts // to retry with 'canonical' precursor selection mode
|
||||
)
|
||||
|
||||
// Read each row and extract the requested message.
|
||||
for {
|
||||
row, err := reader.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("failed to read row: %w", err)
|
||||
}
|
||||
var (
|
||||
cid = row[0]
|
||||
actorcode = row[1]
|
||||
methodnumstr = row[2]
|
||||
exitcodestr = row[3]
|
||||
_ = row[4]
|
||||
block = row[5]
|
||||
seq = row[6]
|
||||
|
||||
exit int
|
||||
methodnum int
|
||||
methodname string
|
||||
)
|
||||
|
||||
// Parse the exit code.
|
||||
if exit, err = strconv.Atoi(exitcodestr); err != nil {
|
||||
return fmt.Errorf("invalid exitcode number: %d", exit)
|
||||
}
|
||||
// Parse the method number.
|
||||
if methodnum, err = strconv.Atoi(methodnumstr); err != nil {
|
||||
return fmt.Errorf("invalid method number: %s", methodnumstr)
|
||||
}
|
||||
|
||||
// Lookup the method in actor method table.
|
||||
if m, ok := ActorMethodTable[actorcode]; !ok {
|
||||
return fmt.Errorf("unrecognized actor: %s", actorcode)
|
||||
} else if methodnum >= len(m) {
|
||||
return fmt.Errorf("unrecognized method number for actor %s: %d", actorcode, methodnum)
|
||||
} else {
|
||||
methodname = m[methodnum]
|
||||
}
|
||||
|
||||
// exitcode string representations are of kind ErrType(0); strip out
|
||||
// the number portion.
|
||||
exitcodename := strings.Split(exitcode.ExitCode(exit).String(), "(")[0]
|
||||
// replace the slashes in the actor code name with underscores.
|
||||
actorcodename := strings.ReplaceAll(actorcode, "/", "_")
|
||||
|
||||
// Compute the ID of the vector.
|
||||
id := fmt.Sprintf("ext-%s-%s-%s-%s-%s", extractManyFlags.batchId, actorcodename, methodname, exitcodename, seq)
|
||||
// Vector filename, using a base of outdir.
|
||||
file := filepath.Join(outdir, actorcodename, methodname, exitcodename, id) + ".json"
|
||||
|
||||
log.Println(color.YellowString("processing message cid with 'sender' precursor mode: %s", id))
|
||||
|
||||
opts := extractOpts{
|
||||
id: id,
|
||||
block: block,
|
||||
class: "message",
|
||||
cid: cid,
|
||||
file: file,
|
||||
retain: "accessed-cids",
|
||||
precursor: PrecursorSelectSender,
|
||||
}
|
||||
|
||||
if err := doExtract(ctx, fapi, opts); err != nil {
|
||||
log.Println(color.RedString("failed to extract vector for message %s: %s; queuing for 'canonical' precursor selection", cid, err))
|
||||
retry = append(retry, opts)
|
||||
continue
|
||||
}
|
||||
|
||||
log.Println(color.MagentaString("generated file: %s", file))
|
||||
|
||||
generated = append(generated, file)
|
||||
}
|
||||
|
||||
log.Printf("extractions to try with canonical precursor selection mode: %d", len(retry))
|
||||
|
||||
for _, r := range retry {
|
||||
log.Printf("retrying %s: %s", r.cid, r.id)
|
||||
|
||||
r.precursor = PrecursorSelectAll
|
||||
if err := doExtract(ctx, fapi, r); err != nil {
|
||||
merr = multierror.Append(merr, fmt.Errorf("failed to extract vector for message %s: %w", r.cid, err))
|
||||
continue
|
||||
}
|
||||
|
||||
log.Println(color.MagentaString("generated file: %s", r.file))
|
||||
generated = append(generated, r.file)
|
||||
}
|
||||
|
||||
if len(generated) == 0 {
|
||||
log.Println("no files generated")
|
||||
} else {
|
||||
log.Println("files generated:")
|
||||
for _, g := range generated {
|
||||
log.Println(g)
|
||||
}
|
||||
}
|
||||
|
||||
if merr.ErrorOrNil() != nil {
|
||||
log.Println(color.YellowString("done processing with errors: %v", merr))
|
||||
} else {
|
||||
log.Println(color.GreenString("done processing with no errors"))
|
||||
}
|
||||
|
||||
return merr.ErrorOrNil()
|
||||
}
|
||||
71
cmd/tvx/main.go
Normal file
71
cmd/tvx/main.go
Normal file
@ -0,0 +1,71 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
"sort"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
// DefaultLotusRepoPath is where the fallback path where to look for a Lotus
|
||||
// client repo. It is expanded with mitchellh/go-homedir, so it'll work with all
|
||||
// OSes despite the Unix twiddle notation.
|
||||
const DefaultLotusRepoPath = "~/.lotus"
|
||||
|
||||
var repoFlag = cli.StringFlag{
|
||||
Name: "repo",
|
||||
EnvVars: []string{"LOTUS_PATH"},
|
||||
Value: DefaultLotusRepoPath,
|
||||
TakesFile: true,
|
||||
}
|
||||
|
||||
func main() {
|
||||
app := &cli.App{
|
||||
Name: "tvx",
|
||||
Description: `tvx is a tool for extracting and executing test vectors. It has three subcommands.
|
||||
|
||||
tvx extract extracts a test vector from a live network. It requires access to
|
||||
a Filecoin client that exposes the standard JSON-RPC API endpoint. Only
|
||||
message class test vectors are supported at this time.
|
||||
|
||||
tvx exec executes test vectors against Lotus. Either you can supply one in a
|
||||
file, or many as an ndjson stdin stream.
|
||||
|
||||
tvx extract-many performs a batch extraction of many messages, supplied in a
|
||||
CSV file. Refer to the help of that subcommand for more info.
|
||||
|
||||
SETTING THE JSON-RPC API ENDPOINT
|
||||
|
||||
You can set the JSON-RPC API endpoint through one of the following methods.
|
||||
|
||||
1. Directly set the API endpoint on the FULLNODE_API_INFO env variable.
|
||||
The format is [token]:multiaddr, where token is optional for commands not
|
||||
accessing privileged operations.
|
||||
|
||||
2. If you're running tvx against a local Lotus client, you can set the REPO
|
||||
env variable to have the API endpoint and token extracted from the repo.
|
||||
Alternatively, you can pass the --repo CLI flag.
|
||||
|
||||
3. Rely on the default fallback, which inspects ~/.lotus and extracts the
|
||||
API endpoint string if the location is a Lotus repo.
|
||||
|
||||
tvx will apply these methods in the same order of precedence they're listed.
|
||||
`,
|
||||
Usage: "tvx is a tool for extracting and executing test vectors",
|
||||
Commands: []*cli.Command{
|
||||
extractCmd,
|
||||
execCmd,
|
||||
extractManyCmd,
|
||||
},
|
||||
}
|
||||
|
||||
sort.Sort(cli.CommandsByName(app.Commands))
|
||||
for _, c := range app.Commands {
|
||||
sort.Sort(cli.FlagsByName(c.Flags))
|
||||
}
|
||||
|
||||
if err := app.Run(os.Args); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
293
cmd/tvx/state.go
Normal file
293
cmd/tvx/state.go
Normal file
@ -0,0 +1,293 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/ipfs/go-cid"
|
||||
format "github.com/ipfs/go-ipld-format"
|
||||
"github.com/ipld/go-car"
|
||||
cbg "github.com/whyrusleeping/cbor-gen"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init"
|
||||
"github.com/filecoin-project/lotus/chain/state"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/chain/vm"
|
||||
)
|
||||
|
||||
// StateSurgeon is an object used to fetch and manipulate state.
|
||||
type StateSurgeon struct {
|
||||
ctx context.Context
|
||||
api api.FullNode
|
||||
stores *Stores
|
||||
}
|
||||
|
||||
// NewSurgeon returns a state surgeon, an object used to fetch and manipulate
|
||||
// state.
|
||||
func NewSurgeon(ctx context.Context, api api.FullNode, stores *Stores) *StateSurgeon {
|
||||
return &StateSurgeon{
|
||||
ctx: ctx,
|
||||
api: api,
|
||||
stores: stores,
|
||||
}
|
||||
}
|
||||
|
||||
// GetMaskedStateTree trims the state tree at the supplied tipset to contain
|
||||
// only the state of the actors in the retain set. It also "dives" into some
|
||||
// singleton system actors, like the init actor, to trim the state so as to
|
||||
// compute a minimal state tree. In the future, thid method will dive into
|
||||
// other system actors like the power actor and the market actor.
|
||||
func (sg *StateSurgeon) GetMaskedStateTree(previousRoot cid.Cid, retain []address.Address) (cid.Cid, error) {
|
||||
// TODO: this will need to be parameterized on network version.
|
||||
st, err := state.LoadStateTree(sg.stores.CBORStore, previousRoot)
|
||||
if err != nil {
|
||||
return cid.Undef, err
|
||||
}
|
||||
|
||||
initActor, initState, err := sg.loadInitActor(st)
|
||||
if err != nil {
|
||||
return cid.Undef, err
|
||||
}
|
||||
|
||||
err = sg.retainInitEntries(initState, retain)
|
||||
if err != nil {
|
||||
return cid.Undef, err
|
||||
}
|
||||
|
||||
err = sg.saveInitActor(initActor, initState, st)
|
||||
if err != nil {
|
||||
return cid.Undef, err
|
||||
}
|
||||
|
||||
// resolve all addresses to ID addresses.
|
||||
resolved, err := sg.resolveAddresses(retain, initState)
|
||||
if err != nil {
|
||||
return cid.Undef, err
|
||||
}
|
||||
|
||||
st, err = sg.transplantActors(st, resolved)
|
||||
if err != nil {
|
||||
return cid.Undef, err
|
||||
}
|
||||
|
||||
root, err := st.Flush(sg.ctx)
|
||||
if err != nil {
|
||||
return cid.Undef, err
|
||||
}
|
||||
|
||||
return root, nil
|
||||
}
|
||||
|
||||
// GetAccessedActors identifies the actors that were accessed during the
|
||||
// execution of a message.
|
||||
func (sg *StateSurgeon) GetAccessedActors(ctx context.Context, a api.FullNode, mid cid.Cid) ([]address.Address, error) {
|
||||
log.Printf("calculating accessed actors during execution of message: %s", mid)
|
||||
msgInfo, err := a.StateSearchMsg(ctx, mid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if msgInfo == nil {
|
||||
return nil, fmt.Errorf("message info is nil")
|
||||
}
|
||||
|
||||
msgObj, err := a.ChainGetMessage(ctx, mid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ts, err := a.ChainGetTipSet(ctx, msgInfo.TipSet)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
trace, err := a.StateCall(ctx, msgObj, ts.Parents())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not replay msg: %w", err)
|
||||
}
|
||||
|
||||
accessed := make(map[address.Address]struct{})
|
||||
|
||||
var recur func(trace *types.ExecutionTrace)
|
||||
recur = func(trace *types.ExecutionTrace) {
|
||||
accessed[trace.Msg.To] = struct{}{}
|
||||
accessed[trace.Msg.From] = struct{}{}
|
||||
for i := range trace.Subcalls {
|
||||
recur(&trace.Subcalls[i])
|
||||
}
|
||||
}
|
||||
recur(&trace.ExecutionTrace)
|
||||
|
||||
ret := make([]address.Address, 0, len(accessed))
|
||||
for k := range accessed {
|
||||
ret = append(ret, k)
|
||||
}
|
||||
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
// WriteCAR recursively writes the tree referenced by the root as a CAR into the
|
||||
// supplied io.Writer.
|
||||
func (sg *StateSurgeon) WriteCAR(w io.Writer, roots ...cid.Cid) error {
|
||||
carWalkFn := func(nd format.Node) (out []*format.Link, err error) {
|
||||
for _, link := range nd.Links() {
|
||||
if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
|
||||
continue
|
||||
}
|
||||
out = append(out, link)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
return car.WriteCarWithWalker(sg.ctx, sg.stores.DAGService, roots, w, carWalkFn)
|
||||
}
|
||||
|
||||
// WriteCARIncluding writes a CAR including only the CIDs that are listed in
|
||||
// the include set. This leads to an intentially sparse tree with dangling links.
|
||||
func (sg *StateSurgeon) WriteCARIncluding(w io.Writer, include map[cid.Cid]struct{}, roots ...cid.Cid) error {
|
||||
carWalkFn := func(nd format.Node) (out []*format.Link, err error) {
|
||||
for _, link := range nd.Links() {
|
||||
if _, ok := include[link.Cid]; !ok {
|
||||
continue
|
||||
}
|
||||
if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
|
||||
continue
|
||||
}
|
||||
out = append(out, link)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
return car.WriteCarWithWalker(sg.ctx, sg.stores.DAGService, roots, w, carWalkFn)
|
||||
}
|
||||
|
||||
// transplantActors plucks the state from the supplied actors at the given
|
||||
// tipset, and places it into the supplied state map.
|
||||
func (sg *StateSurgeon) transplantActors(src *state.StateTree, pluck []address.Address) (*state.StateTree, error) {
|
||||
log.Printf("transplanting actor states: %v", pluck)
|
||||
|
||||
dst, err := state.NewStateTree(sg.stores.CBORStore, src.Version())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, a := range pluck {
|
||||
actor, err := src.GetActor(a)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get actor %s failed: %w", a, err)
|
||||
}
|
||||
|
||||
err = dst.SetActor(a, actor)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// recursive copy of the actor state.
|
||||
err = vm.Copy(context.TODO(), sg.stores.Blockstore, sg.stores.Blockstore, actor.Head)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
actorState, err := sg.api.ChainReadObj(sg.ctx, actor.Head)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cid, err := sg.stores.CBORStore.Put(sg.ctx, &cbg.Deferred{Raw: actorState})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if cid != actor.Head {
|
||||
panic("mismatched cids")
|
||||
}
|
||||
}
|
||||
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
// saveInitActor saves the state of the init actor to the provided state map.
|
||||
func (sg *StateSurgeon) saveInitActor(initActor *types.Actor, initState init_.State, st *state.StateTree) error {
|
||||
log.Printf("saving init actor into state tree")
|
||||
|
||||
// Store the state of the init actor.
|
||||
cid, err := sg.stores.CBORStore.Put(sg.ctx, initState)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
actor := *initActor
|
||||
actor.Head = cid
|
||||
|
||||
err = st.SetActor(init_.Address, &actor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cid, _ = st.Flush(sg.ctx)
|
||||
log.Printf("saved init actor into state tree; new root: %s", cid)
|
||||
return nil
|
||||
}
|
||||
|
||||
// retainInitEntries takes an old init actor state, and retains only the
|
||||
// entries in the retain set, returning a new init actor state.
|
||||
func (sg *StateSurgeon) retainInitEntries(state init_.State, retain []address.Address) error {
|
||||
log.Printf("retaining init actor entries for addresses: %v", retain)
|
||||
|
||||
m := make(map[address.Address]struct{}, len(retain))
|
||||
for _, a := range retain {
|
||||
m[a] = struct{}{}
|
||||
}
|
||||
|
||||
var remove []address.Address
|
||||
_ = state.ForEachActor(func(id abi.ActorID, address address.Address) error {
|
||||
if _, ok := m[address]; !ok {
|
||||
remove = append(remove, address)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
err := state.Remove(remove...)
|
||||
log.Printf("new init actor state: %+v", state)
|
||||
return err
|
||||
}
|
||||
|
||||
// resolveAddresses resolved the requested addresses from the provided
|
||||
// InitActor state, returning a slice of length len(orig), where each index
|
||||
// contains the resolved address.
|
||||
func (sg *StateSurgeon) resolveAddresses(orig []address.Address, ist init_.State) (ret []address.Address, err error) {
|
||||
log.Printf("resolving addresses: %v", orig)
|
||||
|
||||
ret = make([]address.Address, len(orig))
|
||||
for i, addr := range orig {
|
||||
resolved, found, err := ist.ResolveAddress(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("address not found: %s", addr)
|
||||
}
|
||||
ret[i] = resolved
|
||||
}
|
||||
|
||||
log.Printf("resolved addresses: %v", ret)
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
// loadInitActor loads the init actor state from a given tipset.
|
||||
func (sg *StateSurgeon) loadInitActor(st *state.StateTree) (*types.Actor, init_.State, error) {
|
||||
actor, err := st.GetActor(init_.Address)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
initState, err := init_.Load(sg.stores.ADTStore, actor)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
log.Printf("loaded init actor state: %+v", initState)
|
||||
|
||||
return actor, initState, nil
|
||||
}
|
||||
142
cmd/tvx/stores.go
Normal file
142
cmd/tvx/stores.go
Normal file
@ -0,0 +1,142 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
|
||||
"github.com/fatih/color"
|
||||
dssync "github.com/ipfs/go-datastore/sync"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/lib/blockstore"
|
||||
|
||||
"github.com/filecoin-project/lotus/chain/actors/adt"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-blockservice"
|
||||
"github.com/ipfs/go-cid"
|
||||
ds "github.com/ipfs/go-datastore"
|
||||
exchange "github.com/ipfs/go-ipfs-exchange-interface"
|
||||
offline "github.com/ipfs/go-ipfs-exchange-offline"
|
||||
cbor "github.com/ipfs/go-ipld-cbor"
|
||||
format "github.com/ipfs/go-ipld-format"
|
||||
"github.com/ipfs/go-merkledag"
|
||||
)
|
||||
|
||||
// Stores is a collection of the different stores and services that are needed
|
||||
// to deal with the data layer of Filecoin, conveniently interlinked with one
|
||||
// another.
|
||||
type Stores struct {
|
||||
CBORStore cbor.IpldStore
|
||||
ADTStore adt.Store
|
||||
Datastore ds.Batching
|
||||
Blockstore blockstore.Blockstore
|
||||
BlockService blockservice.BlockService
|
||||
Exchange exchange.Interface
|
||||
DAGService format.DAGService
|
||||
}
|
||||
|
||||
// NewProxyingStores is a set of Stores backed by a proxying Blockstore that
|
||||
// proxies Get requests for unknown CIDs to a Filecoin node, via the
|
||||
// ChainReadObj RPC.
|
||||
func NewProxyingStores(ctx context.Context, api api.FullNode) *Stores {
|
||||
ds := dssync.MutexWrap(ds.NewMapDatastore())
|
||||
bs := &proxyingBlockstore{
|
||||
ctx: ctx,
|
||||
api: api,
|
||||
Blockstore: blockstore.NewBlockstore(ds),
|
||||
}
|
||||
return NewStores(ctx, ds, bs)
|
||||
}
|
||||
|
||||
// NewStores creates a non-proxying set of Stores.
|
||||
func NewStores(ctx context.Context, ds ds.Batching, bs blockstore.Blockstore) *Stores {
|
||||
var (
|
||||
cborstore = cbor.NewCborStore(bs)
|
||||
offl = offline.Exchange(bs)
|
||||
blkserv = blockservice.New(bs, offl)
|
||||
dserv = merkledag.NewDAGService(blkserv)
|
||||
)
|
||||
|
||||
return &Stores{
|
||||
CBORStore: cborstore,
|
||||
ADTStore: adt.WrapStore(ctx, cborstore),
|
||||
Datastore: ds,
|
||||
Blockstore: bs,
|
||||
Exchange: offl,
|
||||
BlockService: blkserv,
|
||||
DAGService: dserv,
|
||||
}
|
||||
}
|
||||
|
||||
// TracingBlockstore is a Blockstore trait that records CIDs that were accessed
|
||||
// through Get.
|
||||
type TracingBlockstore interface {
|
||||
// StartTracing starts tracing CIDs accessed through the this Blockstore.
|
||||
StartTracing()
|
||||
|
||||
// FinishTracing finishes tracing accessed CIDs, and returns a map of the
|
||||
// CIDs that were traced.
|
||||
FinishTracing() map[cid.Cid]struct{}
|
||||
}
|
||||
|
||||
// proxyingBlockstore is a Blockstore wrapper that fetches unknown CIDs from
|
||||
// a Filecoin node via JSON-RPC.
|
||||
type proxyingBlockstore struct {
|
||||
ctx context.Context
|
||||
api api.FullNode
|
||||
|
||||
lk sync.RWMutex
|
||||
tracing bool
|
||||
traced map[cid.Cid]struct{}
|
||||
|
||||
blockstore.Blockstore
|
||||
}
|
||||
|
||||
var _ TracingBlockstore = (*proxyingBlockstore)(nil)
|
||||
|
||||
func (pb *proxyingBlockstore) StartTracing() {
|
||||
pb.lk.Lock()
|
||||
pb.tracing = true
|
||||
pb.traced = map[cid.Cid]struct{}{}
|
||||
pb.lk.Unlock()
|
||||
}
|
||||
|
||||
func (pb *proxyingBlockstore) FinishTracing() map[cid.Cid]struct{} {
|
||||
pb.lk.Lock()
|
||||
ret := pb.traced
|
||||
pb.tracing = false
|
||||
pb.traced = map[cid.Cid]struct{}{}
|
||||
pb.lk.Unlock()
|
||||
return ret
|
||||
}
|
||||
|
||||
func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
|
||||
pb.lk.RLock()
|
||||
if pb.tracing {
|
||||
pb.traced[cid] = struct{}{}
|
||||
}
|
||||
pb.lk.RUnlock()
|
||||
|
||||
if block, err := pb.Blockstore.Get(cid); err == nil {
|
||||
return block, err
|
||||
}
|
||||
|
||||
log.Println(color.CyanString("fetching cid via rpc: %v", cid))
|
||||
item, err := pb.api.ChainReadObj(pb.ctx, cid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
block, err := blocks.NewBlockWithCid(item, cid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = pb.Blockstore.Put(block)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return block, nil
|
||||
}
|
||||
133
conformance/corpus_test.go
Normal file
133
conformance/corpus_test.go
Normal file
@ -0,0 +1,133 @@
|
||||
package conformance
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/filecoin-project/test-vectors/schema"
|
||||
)
|
||||
|
||||
const (
|
||||
// EnvSkipConformance, if 1, skips the conformance test suite.
|
||||
EnvSkipConformance = "SKIP_CONFORMANCE"
|
||||
|
||||
// EnvCorpusRootDir is the name of the environment variable where the path
|
||||
// to an alternative corpus location can be provided.
|
||||
//
|
||||
// The default is defaultCorpusRoot.
|
||||
EnvCorpusRootDir = "CORPUS_DIR"
|
||||
|
||||
// defaultCorpusRoot is the directory where the test vector corpus is hosted.
|
||||
// It is mounted on the Lotus repo as a git submodule.
|
||||
//
|
||||
// When running this test, the corpus root can be overridden through the
|
||||
// -conformance.corpus CLI flag to run an alternate corpus.
|
||||
defaultCorpusRoot = "../extern/test-vectors/corpus"
|
||||
)
|
||||
|
||||
// ignore is a set of paths relative to root to skip.
|
||||
var ignore = map[string]struct{}{
|
||||
".git": {},
|
||||
"schema.json": {},
|
||||
}
|
||||
|
||||
// TestConformance is the entrypoint test that runs all test vectors found
|
||||
// in the corpus root directory.
|
||||
//
|
||||
// It locates all json files via a recursive walk, skipping over the ignore set,
|
||||
// as well as files beginning with _. It parses each file as a test vector, and
|
||||
// runs it via the Driver.
|
||||
func TestConformance(t *testing.T) {
|
||||
if skip := strings.TrimSpace(os.Getenv(EnvSkipConformance)); skip == "1" {
|
||||
t.SkipNow()
|
||||
}
|
||||
// corpusRoot is the effective corpus root path, taken from the `-conformance.corpus` CLI flag,
|
||||
// falling back to defaultCorpusRoot if not provided.
|
||||
corpusRoot := defaultCorpusRoot
|
||||
if dir := strings.TrimSpace(os.Getenv(EnvCorpusRootDir)); dir != "" {
|
||||
corpusRoot = dir
|
||||
}
|
||||
|
||||
var vectors []string
|
||||
err := filepath.Walk(corpusRoot+"/", func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
filename := filepath.Base(path)
|
||||
rel, err := filepath.Rel(corpusRoot, path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, ok := ignore[rel]; ok {
|
||||
// skip over using the right error.
|
||||
if info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if info.IsDir() {
|
||||
// dive into directories.
|
||||
return nil
|
||||
}
|
||||
if filepath.Ext(path) != ".json" {
|
||||
// skip if not .json.
|
||||
return nil
|
||||
}
|
||||
if ignored := strings.HasPrefix(filename, "_"); ignored {
|
||||
// ignore files starting with _.
|
||||
t.Logf("ignoring: %s", rel)
|
||||
return nil
|
||||
}
|
||||
vectors = append(vectors, rel)
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(vectors) == 0 {
|
||||
t.Fatalf("no test vectors found")
|
||||
}
|
||||
|
||||
// Run a test for each vector.
|
||||
for _, v := range vectors {
|
||||
path := filepath.Join(corpusRoot, v)
|
||||
raw, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test raw file: %s", path)
|
||||
}
|
||||
|
||||
var vector schema.TestVector
|
||||
err = json.Unmarshal(raw, &vector)
|
||||
if err != nil {
|
||||
t.Errorf("failed to parse test vector %s: %s; skipping", path, err)
|
||||
continue
|
||||
}
|
||||
|
||||
t.Run(v, func(t *testing.T) {
|
||||
for _, h := range vector.Hints {
|
||||
if h == schema.HintIncorrect {
|
||||
t.Logf("skipping vector marked as incorrect: %s", vector.Meta.ID)
|
||||
t.SkipNow()
|
||||
}
|
||||
}
|
||||
|
||||
// dispatch the execution depending on the vector class.
|
||||
switch vector.Class {
|
||||
case "message":
|
||||
ExecuteMessageVector(t, &vector)
|
||||
case "tipset":
|
||||
ExecuteTipsetVector(t, &vector)
|
||||
default:
|
||||
t.Fatalf("test vector class not supported: %s", vector.Class)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -2,9 +2,9 @@ package conformance
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/crypto"
|
||||
|
||||
"github.com/filecoin-project/lotus/chain/state"
|
||||
"github.com/filecoin-project/lotus/chain/stmgr"
|
||||
"github.com/filecoin-project/lotus/chain/store"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
@ -14,6 +14,7 @@ import (
|
||||
"github.com/filecoin-project/lotus/lib/blockstore"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/crypto"
|
||||
|
||||
"github.com/filecoin-project/test-vectors/schema"
|
||||
|
||||
@ -24,18 +25,36 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
// BaseFee to use in the VM.
|
||||
// TODO make parametrisable through vector.
|
||||
BaseFee = abi.NewTokenAmount(100)
|
||||
// DefaultCirculatingSupply is the fallback circulating supply returned by
|
||||
// the driver's CircSupplyCalculator function, used if the vector specifies
|
||||
// no circulating supply.
|
||||
DefaultCirculatingSupply = types.TotalFilecoinInt
|
||||
|
||||
// DefaultBaseFee to use in the VM, if one is not supplied in the vector.
|
||||
DefaultBaseFee = abi.NewTokenAmount(100)
|
||||
)
|
||||
|
||||
type Driver struct {
|
||||
ctx context.Context
|
||||
selector schema.Selector
|
||||
vmFlush bool
|
||||
}
|
||||
|
||||
func NewDriver(ctx context.Context, selector schema.Selector) *Driver {
|
||||
return &Driver{ctx: ctx, selector: selector}
|
||||
type DriverOpts struct {
|
||||
// DisableVMFlush, when true, avoids calling VM.Flush(), forces a blockstore
|
||||
// recursive copy, from the temporary buffer blockstore, to the real
|
||||
// system's blockstore. Disabling VM flushing is useful when extracting test
|
||||
// vectors and trimming state, as we don't want to force an accidental
|
||||
// deep copy of the state tree.
|
||||
//
|
||||
// Disabling VM flushing almost always should go hand-in-hand with
|
||||
// LOTUS_DISABLE_VM_BUF=iknowitsabadidea. That way, state tree writes are
|
||||
// immediately committed to the blockstore.
|
||||
DisableVMFlush bool
|
||||
}
|
||||
|
||||
func NewDriver(ctx context.Context, selector schema.Selector, opts DriverOpts) *Driver {
|
||||
return &Driver{ctx: ctx, selector: selector, vmFlush: !opts.DisableVMFlush}
|
||||
}
|
||||
|
||||
type ExecuteTipsetResult struct {
|
||||
@ -120,19 +139,47 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
type ExecuteMessageParams struct {
|
||||
Preroot cid.Cid
|
||||
Epoch abi.ChainEpoch
|
||||
Message *types.Message
|
||||
CircSupply *abi.TokenAmount
|
||||
BaseFee *abi.TokenAmount
|
||||
}
|
||||
|
||||
// ExecuteMessage executes a conformance test vector message in a temporary VM.
|
||||
func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch abi.ChainEpoch, msg *types.Message) (*vm.ApplyRet, cid.Cid, error) {
|
||||
// dummy state manager; only to reference the GetNetworkVersion method, which does not depend on state.
|
||||
func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, params ExecuteMessageParams) (*vm.ApplyRet, cid.Cid, error) {
|
||||
if !d.vmFlush {
|
||||
// do not flush the VM, just the state tree; this should be used with
|
||||
// LOTUS_DISABLE_VM_BUF enabled, so writes will anyway be visible.
|
||||
_ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea")
|
||||
}
|
||||
|
||||
basefee := DefaultBaseFee
|
||||
if params.BaseFee != nil {
|
||||
basefee = *params.BaseFee
|
||||
}
|
||||
|
||||
circSupply := DefaultCirculatingSupply
|
||||
if params.CircSupply != nil {
|
||||
circSupply = *params.CircSupply
|
||||
}
|
||||
|
||||
// dummy state manager; only to reference the GetNetworkVersion method,
|
||||
// which does not depend on state.
|
||||
sm := new(stmgr.StateManager)
|
||||
|
||||
vmOpts := &vm.VMOpts{
|
||||
StateBase: preroot,
|
||||
Epoch: epoch,
|
||||
Rand: &testRand{}, // TODO always succeeds; need more flexibility.
|
||||
Bstore: bs,
|
||||
Syscalls: mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility.
|
||||
CircSupplyCalc: nil,
|
||||
BaseFee: BaseFee,
|
||||
NtwkVersion: sm.GetNtwkVersion,
|
||||
StateBase: params.Preroot,
|
||||
Epoch: params.Epoch,
|
||||
Rand: &testRand{}, // TODO always succeeds; need more flexibility.
|
||||
Bstore: bs,
|
||||
Syscalls: mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility.
|
||||
CircSupplyCalc: func(_ context.Context, _ abi.ChainEpoch, _ *state.StateTree) (abi.TokenAmount, error) {
|
||||
return circSupply, nil
|
||||
},
|
||||
BaseFee: basefee,
|
||||
NtwkVersion: sm.GetNtwkVersion,
|
||||
}
|
||||
|
||||
lvm, err := vm.NewVM(context.TODO(), vmOpts)
|
||||
@ -149,12 +196,20 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch
|
||||
|
||||
lvm.SetInvoker(invoker)
|
||||
|
||||
ret, err := lvm.ApplyMessage(d.ctx, toChainMsg(msg))
|
||||
ret, err := lvm.ApplyMessage(d.ctx, toChainMsg(params.Message))
|
||||
if err != nil {
|
||||
return nil, cid.Undef, err
|
||||
}
|
||||
|
||||
root, err := lvm.Flush(d.ctx)
|
||||
var root cid.Cid
|
||||
if d.vmFlush {
|
||||
// flush the VM, committing the state tree changes and forcing a
|
||||
// recursive copoy from the temporary blcokstore to the real blockstore.
|
||||
root, err = lvm.Flush(d.ctx)
|
||||
} else {
|
||||
root, err = lvm.StateTree().(*state.StateTree).Flush(d.ctx)
|
||||
}
|
||||
|
||||
return ret, root, err
|
||||
}
|
||||
|
||||
|
||||
62
conformance/reporter.go
Normal file
62
conformance/reporter.go
Normal file
@ -0,0 +1,62 @@
|
||||
package conformance
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/fatih/color"
|
||||
)
|
||||
|
||||
// Reporter is a contains a subset of the testing.T methods, so that the
|
||||
// Execute* functions in this package can be used inside or outside of
|
||||
// go test runs.
|
||||
type Reporter interface {
|
||||
Helper()
|
||||
|
||||
Log(args ...interface{})
|
||||
Errorf(format string, args ...interface{})
|
||||
Fatalf(format string, args ...interface{})
|
||||
Logf(format string, args ...interface{})
|
||||
FailNow()
|
||||
Failed() bool
|
||||
}
|
||||
|
||||
var _ Reporter = (*testing.T)(nil)
|
||||
|
||||
// LogReporter wires the Reporter methods to the log package. It is appropriate
|
||||
// to use when calling the Execute* functions from a standalone CLI program.
|
||||
type LogReporter struct {
|
||||
failed int32
|
||||
}
|
||||
|
||||
var _ Reporter = (*LogReporter)(nil)
|
||||
|
||||
func (*LogReporter) Helper() {}
|
||||
|
||||
func (*LogReporter) Log(args ...interface{}) {
|
||||
log.Println(args...)
|
||||
}
|
||||
|
||||
func (*LogReporter) Logf(format string, args ...interface{}) {
|
||||
log.Printf(format, args...)
|
||||
}
|
||||
|
||||
func (*LogReporter) FailNow() {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
func (l *LogReporter) Failed() bool {
|
||||
return atomic.LoadInt32(&l.failed) == 1
|
||||
}
|
||||
|
||||
func (l *LogReporter) Errorf(format string, args ...interface{}) {
|
||||
atomic.StoreInt32(&l.failed, 1)
|
||||
log.Println(color.HiRedString("❌ "+format, args...))
|
||||
}
|
||||
|
||||
func (l *LogReporter) Fatalf(format string, args ...interface{}) {
|
||||
atomic.StoreInt32(&l.failed, 1)
|
||||
log.Fatal(color.HiRedString("❌ "+format, args...))
|
||||
}
|
||||
272
conformance/runner.go
Normal file
272
conformance/runner.go
Normal file
@ -0,0 +1,272 @@
|
||||
package conformance
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/big"
|
||||
"github.com/filecoin-project/go-state-types/exitcode"
|
||||
"github.com/filecoin-project/test-vectors/schema"
|
||||
"github.com/ipfs/go-blockservice"
|
||||
"github.com/ipfs/go-cid"
|
||||
ds "github.com/ipfs/go-datastore"
|
||||
offline "github.com/ipfs/go-ipfs-exchange-offline"
|
||||
format "github.com/ipfs/go-ipld-format"
|
||||
"github.com/ipfs/go-merkledag"
|
||||
"github.com/ipld/go-car"
|
||||
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/chain/vm"
|
||||
"github.com/filecoin-project/lotus/lib/blockstore"
|
||||
)
|
||||
|
||||
// ExecuteMessageVector executes a message-class test vector.
|
||||
func ExecuteMessageVector(r Reporter, vector *schema.TestVector) {
|
||||
var (
|
||||
ctx = context.Background()
|
||||
epoch = vector.Pre.Epoch
|
||||
root = vector.Pre.StateTree.RootCID
|
||||
)
|
||||
|
||||
// Load the CAR into a new temporary Blockstore.
|
||||
bs, err := LoadVectorCAR(vector.CAR)
|
||||
if err != nil {
|
||||
r.Fatalf("failed to load the vector CAR: %w", err)
|
||||
}
|
||||
|
||||
// Create a new Driver.
|
||||
driver := NewDriver(ctx, vector.Selector, DriverOpts{DisableVMFlush: true})
|
||||
|
||||
var circSupply *abi.TokenAmount
|
||||
if cs := vector.Pre.CircSupply; cs != nil {
|
||||
ta := big.NewFromGo(cs)
|
||||
circSupply = &ta
|
||||
}
|
||||
|
||||
var basefee *abi.TokenAmount
|
||||
if bf := vector.Pre.BaseFee; bf != nil {
|
||||
ta := big.NewFromGo(bf)
|
||||
basefee = &ta
|
||||
}
|
||||
|
||||
// Apply every message.
|
||||
for i, m := range vector.ApplyMessages {
|
||||
msg, err := types.DecodeMessage(m.Bytes)
|
||||
if err != nil {
|
||||
r.Fatalf("failed to deserialize message: %s", err)
|
||||
}
|
||||
|
||||
// add an epoch if one's set.
|
||||
if m.Epoch != nil {
|
||||
epoch = *m.Epoch
|
||||
}
|
||||
|
||||
// Execute the message.
|
||||
var ret *vm.ApplyRet
|
||||
ret, root, err = driver.ExecuteMessage(bs, ExecuteMessageParams{
|
||||
Preroot: root,
|
||||
Epoch: abi.ChainEpoch(epoch),
|
||||
Message: msg,
|
||||
CircSupply: circSupply,
|
||||
BaseFee: basefee,
|
||||
})
|
||||
if err != nil {
|
||||
r.Fatalf("fatal failure when executing message: %s", err)
|
||||
}
|
||||
|
||||
// Assert that the receipt matches what the test vector expects.
|
||||
AssertMsgResult(r, vector.Post.Receipts[i], ret, strconv.Itoa(i))
|
||||
}
|
||||
|
||||
// Once all messages are applied, assert that the final state root matches
|
||||
// the expected postcondition root.
|
||||
if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual {
|
||||
r.Errorf("wrong post root cid; expected %v, but got %v", expected, actual)
|
||||
dumpThreeWayStateDiff(r, vector, bs, root)
|
||||
r.FailNow()
|
||||
}
|
||||
}
|
||||
|
||||
// ExecuteTipsetVector executes a tipset-class test vector.
|
||||
func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) {
|
||||
var (
|
||||
ctx = context.Background()
|
||||
prevEpoch = vector.Pre.Epoch
|
||||
root = vector.Pre.StateTree.RootCID
|
||||
tmpds = ds.NewMapDatastore()
|
||||
)
|
||||
|
||||
// Load the vector CAR into a new temporary Blockstore.
|
||||
bs, err := LoadVectorCAR(vector.CAR)
|
||||
if err != nil {
|
||||
r.Fatalf("failed to load the vector CAR: %w", err)
|
||||
}
|
||||
|
||||
// Create a new Driver.
|
||||
driver := NewDriver(ctx, vector.Selector, DriverOpts{})
|
||||
|
||||
// Apply every tipset.
|
||||
var receiptsIdx int
|
||||
for i, ts := range vector.ApplyTipsets {
|
||||
ts := ts // capture
|
||||
ret, err := driver.ExecuteTipset(bs, tmpds, root, abi.ChainEpoch(prevEpoch), &ts)
|
||||
if err != nil {
|
||||
r.Fatalf("failed to apply tipset %d message: %s", i, err)
|
||||
}
|
||||
|
||||
for j, v := range ret.AppliedResults {
|
||||
AssertMsgResult(r, vector.Post.Receipts[receiptsIdx], v, fmt.Sprintf("%d of tipset %d", j, i))
|
||||
receiptsIdx++
|
||||
}
|
||||
|
||||
// Compare the receipts root.
|
||||
if expected, actual := vector.Post.ReceiptsRoots[i], ret.ReceiptsRoot; expected != actual {
|
||||
r.Errorf("post receipts root doesn't match; expected: %s, was: %s", expected, actual)
|
||||
}
|
||||
|
||||
prevEpoch = ts.Epoch
|
||||
root = ret.PostStateRoot
|
||||
}
|
||||
|
||||
// Once all messages are applied, assert that the final state root matches
|
||||
// the expected postcondition root.
|
||||
if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual {
|
||||
r.Errorf("wrong post root cid; expected %v, but got %v", expected, actual)
|
||||
dumpThreeWayStateDiff(r, vector, bs, root)
|
||||
r.FailNow()
|
||||
}
|
||||
}
|
||||
|
||||
// AssertMsgResult compares a message result. It takes the expected receipt
|
||||
// encoded in the vector, the actual receipt returned by Lotus, and a message
|
||||
// label to log in the assertion failure message to facilitate debugging.
|
||||
func AssertMsgResult(r Reporter, expected *schema.Receipt, actual *vm.ApplyRet, label string) {
|
||||
r.Helper()
|
||||
|
||||
if expected, actual := exitcode.ExitCode(expected.ExitCode), actual.ExitCode; expected != actual {
|
||||
r.Errorf("exit code of msg %s did not match; expected: %s, got: %s", label, expected, actual)
|
||||
}
|
||||
if expected, actual := expected.GasUsed, actual.GasUsed; expected != actual {
|
||||
r.Errorf("gas used of msg %s did not match; expected: %d, got: %d", label, expected, actual)
|
||||
}
|
||||
if expected, actual := []byte(expected.ReturnValue), actual.Return; !bytes.Equal(expected, actual) {
|
||||
r.Errorf("return value of msg %s did not match; expected: %s, got: %s", label, base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(actual))
|
||||
}
|
||||
}
|
||||
|
||||
func dumpThreeWayStateDiff(r Reporter, vector *schema.TestVector, bs blockstore.Blockstore, actual cid.Cid) {
|
||||
// check if statediff exists; if not, skip.
|
||||
if err := exec.Command("statediff", "--help").Run(); err != nil {
|
||||
r.Log("could not dump 3-way state tree diff upon test failure: statediff command not found")
|
||||
r.Log("install statediff with:")
|
||||
r.Log("$ git clone https://github.com/filecoin-project/statediff.git")
|
||||
r.Log("$ cd statediff")
|
||||
r.Log("$ go generate ./...")
|
||||
r.Log("$ go install ./cmd/statediff")
|
||||
return
|
||||
}
|
||||
|
||||
tmpCar, err := writeStateToTempCAR(bs,
|
||||
vector.Pre.StateTree.RootCID,
|
||||
vector.Post.StateTree.RootCID,
|
||||
actual,
|
||||
)
|
||||
if err != nil {
|
||||
r.Fatalf("failed to write temporary state CAR: %s", err)
|
||||
}
|
||||
defer os.RemoveAll(tmpCar) //nolint:errcheck
|
||||
|
||||
color.NoColor = false // enable colouring.
|
||||
|
||||
var (
|
||||
a = color.New(color.FgMagenta, color.Bold).Sprint("(A) expected final state")
|
||||
b = color.New(color.FgYellow, color.Bold).Sprint("(B) actual final state")
|
||||
c = color.New(color.FgCyan, color.Bold).Sprint("(C) initial state")
|
||||
d1 = color.New(color.FgGreen, color.Bold).Sprint("[Δ1]")
|
||||
d2 = color.New(color.FgGreen, color.Bold).Sprint("[Δ2]")
|
||||
d3 = color.New(color.FgGreen, color.Bold).Sprint("[Δ3]")
|
||||
)
|
||||
|
||||
printDiff := func(left, right cid.Cid) {
|
||||
cmd := exec.Command("statediff", "car", "--file", tmpCar, left.String(), right.String())
|
||||
b, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
r.Fatalf("statediff failed: %s", err)
|
||||
}
|
||||
r.Log(string(b))
|
||||
}
|
||||
|
||||
bold := color.New(color.Bold).SprintfFunc()
|
||||
|
||||
// run state diffs.
|
||||
r.Log(bold("=== dumping 3-way diffs between %s, %s, %s ===", a, b, c))
|
||||
|
||||
r.Log(bold("--- %s left: %s; right: %s ---", d1, a, b))
|
||||
printDiff(vector.Post.StateTree.RootCID, actual)
|
||||
|
||||
r.Log(bold("--- %s left: %s; right: %s ---", d2, c, b))
|
||||
printDiff(vector.Pre.StateTree.RootCID, actual)
|
||||
|
||||
r.Log(bold("--- %s left: %s; right: %s ---", d3, c, a))
|
||||
printDiff(vector.Pre.StateTree.RootCID, vector.Post.StateTree.RootCID)
|
||||
}
|
||||
|
||||
// writeStateToTempCAR writes the provided roots to a temporary CAR that'll be
|
||||
// cleaned up via t.Cleanup(). It returns the full path of the temp file.
|
||||
func writeStateToTempCAR(bs blockstore.Blockstore, roots ...cid.Cid) (string, error) {
|
||||
tmp, err := ioutil.TempFile("", "lotus-tests-*.car")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create temp file to dump CAR for diffing: %w", err)
|
||||
}
|
||||
|
||||
carWalkFn := func(nd format.Node) (out []*format.Link, err error) {
|
||||
for _, link := range nd.Links() {
|
||||
if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
|
||||
continue
|
||||
}
|
||||
out = append(out, link)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
var (
|
||||
offl = offline.Exchange(bs)
|
||||
blkserv = blockservice.New(bs, offl)
|
||||
dserv = merkledag.NewDAGService(blkserv)
|
||||
)
|
||||
|
||||
err = car.WriteCarWithWalker(context.Background(), dserv, roots, tmp, carWalkFn)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to dump CAR for diffing: %w", err)
|
||||
}
|
||||
_ = tmp.Close()
|
||||
return tmp.Name(), nil
|
||||
}
|
||||
|
||||
func LoadVectorCAR(vectorCAR schema.Base64EncodedBytes) (blockstore.Blockstore, error) {
|
||||
bs := blockstore.NewTemporary()
|
||||
|
||||
// Read the base64-encoded CAR from the vector, and inflate the gzip.
|
||||
buf := bytes.NewReader(vectorCAR)
|
||||
r, err := gzip.NewReader(buf)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to inflate gzipped CAR: %s", err)
|
||||
}
|
||||
defer r.Close() // nolint
|
||||
|
||||
// Load the CAR embedded in the test vector into the Blockstore.
|
||||
_, err = car.LoadCar(bs, r)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load state tree car from test vector: %s", err)
|
||||
}
|
||||
return bs, nil
|
||||
}
|
||||
@ -1,376 +0,0 @@
|
||||
package conformance
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/exitcode"
|
||||
"github.com/ipfs/go-blockservice"
|
||||
"github.com/ipfs/go-cid"
|
||||
ds "github.com/ipfs/go-datastore"
|
||||
offline "github.com/ipfs/go-ipfs-exchange-offline"
|
||||
format "github.com/ipfs/go-ipld-format"
|
||||
"github.com/ipfs/go-merkledag"
|
||||
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/chain/vm"
|
||||
"github.com/filecoin-project/lotus/lib/blockstore"
|
||||
|
||||
"github.com/filecoin-project/test-vectors/schema"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/ipld/go-car"
|
||||
)
|
||||
|
||||
const (
|
||||
// EnvSkipConformance, if 1, skips the conformance test suite.
|
||||
EnvSkipConformance = "SKIP_CONFORMANCE"
|
||||
|
||||
// EnvCorpusRootDir is the name of the environment variable where the path
|
||||
// to an alternative corpus location can be provided.
|
||||
//
|
||||
// The default is defaultCorpusRoot.
|
||||
EnvCorpusRootDir = "CORPUS_DIR"
|
||||
|
||||
// defaultCorpusRoot is the directory where the test vector corpus is hosted.
|
||||
// It is mounted on the Lotus repo as a git submodule.
|
||||
//
|
||||
// When running this test, the corpus root can be overridden through the
|
||||
// -conformance.corpus CLI flag to run an alternate corpus.
|
||||
defaultCorpusRoot = "../extern/test-vectors/corpus"
|
||||
)
|
||||
|
||||
// ignore is a set of paths relative to root to skip.
|
||||
var ignore = map[string]struct{}{
|
||||
".git": {},
|
||||
"schema.json": {},
|
||||
}
|
||||
|
||||
// TestConformance is the entrypoint test that runs all test vectors found
|
||||
// in the corpus root directory.
|
||||
//
|
||||
// It locates all json files via a recursive walk, skipping over the ignore set,
|
||||
// as well as files beginning with _. It parses each file as a test vector, and
|
||||
// runs it via the Driver.
|
||||
func TestConformance(t *testing.T) {
|
||||
if skip := strings.TrimSpace(os.Getenv(EnvSkipConformance)); skip == "1" {
|
||||
t.SkipNow()
|
||||
}
|
||||
// corpusRoot is the effective corpus root path, taken from the `-conformance.corpus` CLI flag,
|
||||
// falling back to defaultCorpusRoot if not provided.
|
||||
corpusRoot := defaultCorpusRoot
|
||||
if dir := strings.TrimSpace(os.Getenv(EnvCorpusRootDir)); dir != "" {
|
||||
corpusRoot = dir
|
||||
}
|
||||
|
||||
var vectors []string
|
||||
err := filepath.Walk(corpusRoot+"/", func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
filename := filepath.Base(path)
|
||||
rel, err := filepath.Rel(corpusRoot, path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, ok := ignore[rel]; ok {
|
||||
// skip over using the right error.
|
||||
if info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if info.IsDir() {
|
||||
// dive into directories.
|
||||
return nil
|
||||
}
|
||||
if filepath.Ext(path) != ".json" {
|
||||
// skip if not .json.
|
||||
return nil
|
||||
}
|
||||
if ignored := strings.HasPrefix(filename, "_"); ignored {
|
||||
// ignore files starting with _.
|
||||
t.Logf("ignoring: %s", rel)
|
||||
return nil
|
||||
}
|
||||
vectors = append(vectors, rel)
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(vectors) == 0 {
|
||||
t.Fatalf("no test vectors found")
|
||||
}
|
||||
|
||||
// Run a test for each vector.
|
||||
for _, v := range vectors {
|
||||
path := filepath.Join(corpusRoot, v)
|
||||
raw, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test raw file: %s", path)
|
||||
}
|
||||
|
||||
var vector schema.TestVector
|
||||
err = json.Unmarshal(raw, &vector)
|
||||
if err != nil {
|
||||
t.Errorf("failed to parse test vector %s: %s; skipping", path, err)
|
||||
continue
|
||||
}
|
||||
|
||||
t.Run(v, func(t *testing.T) {
|
||||
for _, h := range vector.Hints {
|
||||
if h == schema.HintIncorrect {
|
||||
t.Logf("skipping vector marked as incorrect: %s", vector.Meta.ID)
|
||||
t.SkipNow()
|
||||
}
|
||||
}
|
||||
|
||||
// dispatch the execution depending on the vector class.
|
||||
switch vector.Class {
|
||||
case "message":
|
||||
executeMessageVector(t, &vector)
|
||||
case "tipset":
|
||||
executeTipsetVector(t, &vector)
|
||||
default:
|
||||
t.Fatalf("test vector class not supported: %s", vector.Class)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// executeMessageVector executes a message-class test vector.
|
||||
func executeMessageVector(t *testing.T, vector *schema.TestVector) {
|
||||
var (
|
||||
ctx = context.Background()
|
||||
epoch = vector.Pre.Epoch
|
||||
root = vector.Pre.StateTree.RootCID
|
||||
)
|
||||
|
||||
// Load the CAR into a new temporary Blockstore.
|
||||
bs := loadCAR(t, vector.CAR)
|
||||
|
||||
// Create a new Driver.
|
||||
driver := NewDriver(ctx, vector.Selector)
|
||||
|
||||
// Apply every message.
|
||||
for i, m := range vector.ApplyMessages {
|
||||
msg, err := types.DecodeMessage(m.Bytes)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to deserialize message: %s", err)
|
||||
}
|
||||
|
||||
// add an epoch if one's set.
|
||||
if m.Epoch != nil {
|
||||
epoch = *m.Epoch
|
||||
}
|
||||
|
||||
// Execute the message.
|
||||
var ret *vm.ApplyRet
|
||||
ret, root, err = driver.ExecuteMessage(bs, root, abi.ChainEpoch(epoch), msg)
|
||||
if err != nil {
|
||||
t.Fatalf("fatal failure when executing message: %s", err)
|
||||
}
|
||||
|
||||
// Assert that the receipt matches what the test vector expects.
|
||||
assertMsgResult(t, vector.Post.Receipts[i], ret, strconv.Itoa(i))
|
||||
}
|
||||
|
||||
// Once all messages are applied, assert that the final state root matches
|
||||
// the expected postcondition root.
|
||||
if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual {
|
||||
t.Logf("actual state root CID doesn't match expected one; expected: %s, actual: %s", expected, actual)
|
||||
dumpThreeWayStateDiff(t, vector, bs, root)
|
||||
t.FailNow()
|
||||
}
|
||||
}
|
||||
|
||||
// executeTipsetVector executes a tipset-class test vector.
|
||||
func executeTipsetVector(t *testing.T, vector *schema.TestVector) {
|
||||
var (
|
||||
ctx = context.Background()
|
||||
prevEpoch = vector.Pre.Epoch
|
||||
root = vector.Pre.StateTree.RootCID
|
||||
tmpds = ds.NewMapDatastore()
|
||||
)
|
||||
|
||||
// Load the CAR into a new temporary Blockstore.
|
||||
bs := loadCAR(t, vector.CAR)
|
||||
|
||||
// Create a new Driver.
|
||||
driver := NewDriver(ctx, vector.Selector)
|
||||
|
||||
// Apply every tipset.
|
||||
var receiptsIdx int
|
||||
for i, ts := range vector.ApplyTipsets {
|
||||
ts := ts // capture
|
||||
ret, err := driver.ExecuteTipset(bs, tmpds, root, abi.ChainEpoch(prevEpoch), &ts)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to apply tipset %d message: %s", i, err)
|
||||
}
|
||||
|
||||
for j, v := range ret.AppliedResults {
|
||||
assertMsgResult(t, vector.Post.Receipts[receiptsIdx], v, fmt.Sprintf("%d of tipset %d", j, i))
|
||||
receiptsIdx++
|
||||
}
|
||||
|
||||
// Compare the receipts root.
|
||||
if expected, actual := vector.Post.ReceiptsRoots[i], ret.ReceiptsRoot; expected != actual {
|
||||
t.Errorf("post receipts root doesn't match; expected: %s, was: %s", expected, actual)
|
||||
}
|
||||
|
||||
prevEpoch = ts.Epoch
|
||||
root = ret.PostStateRoot
|
||||
}
|
||||
|
||||
// Once all messages are applied, assert that the final state root matches
|
||||
// the expected postcondition root.
|
||||
if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual {
|
||||
t.Logf("actual state root CID doesn't match expected one; expected: %s, actual: %s", expected, actual)
|
||||
dumpThreeWayStateDiff(t, vector, bs, root)
|
||||
t.FailNow()
|
||||
}
|
||||
}
|
||||
|
||||
// assertMsgResult compares a message result. It takes the expected receipt
|
||||
// encoded in the vector, the actual receipt returned by Lotus, and a message
|
||||
// label to log in the assertion failure message to facilitate debugging.
|
||||
func assertMsgResult(t *testing.T, expected *schema.Receipt, actual *vm.ApplyRet, label string) {
|
||||
t.Helper()
|
||||
|
||||
if expected, actual := exitcode.ExitCode(expected.ExitCode), actual.ExitCode; expected != actual {
|
||||
t.Errorf("exit code of msg %s did not match; expected: %s, got: %s", label, expected, actual)
|
||||
}
|
||||
if expected, actual := expected.GasUsed, actual.GasUsed; expected != actual {
|
||||
t.Errorf("gas used of msg %s did not match; expected: %d, got: %d", label, expected, actual)
|
||||
}
|
||||
if expected, actual := []byte(expected.ReturnValue), actual.Return; !bytes.Equal(expected, actual) {
|
||||
t.Errorf("return value of msg %s did not match; expected: %s, got: %s", label, base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(actual))
|
||||
}
|
||||
}
|
||||
|
||||
func dumpThreeWayStateDiff(t *testing.T, vector *schema.TestVector, bs blockstore.Blockstore, actual cid.Cid) {
|
||||
// check if statediff exists; if not, skip.
|
||||
if err := exec.Command("statediff", "--help").Run(); err != nil {
|
||||
t.Log("could not dump 3-way state tree diff upon test failure: statediff command not found")
|
||||
t.Log("install statediff with:")
|
||||
t.Log("$ git clone https://github.com/filecoin-project/statediff.git")
|
||||
t.Log("$ cd statediff")
|
||||
t.Log("$ go generate ./...")
|
||||
t.Log("$ go install ./cmd/statediff")
|
||||
return
|
||||
}
|
||||
|
||||
tmpCar := writeStateToTempCAR(t, bs,
|
||||
vector.Pre.StateTree.RootCID,
|
||||
vector.Post.StateTree.RootCID,
|
||||
actual,
|
||||
)
|
||||
|
||||
color.NoColor = false // enable colouring.
|
||||
|
||||
t.Errorf("wrong post root cid; expected %v, but got %v", vector.Post.StateTree.RootCID, actual)
|
||||
|
||||
var (
|
||||
a = color.New(color.FgMagenta, color.Bold).Sprint("(A) expected final state")
|
||||
b = color.New(color.FgYellow, color.Bold).Sprint("(B) actual final state")
|
||||
c = color.New(color.FgCyan, color.Bold).Sprint("(C) initial state")
|
||||
d1 = color.New(color.FgGreen, color.Bold).Sprint("[Δ1]")
|
||||
d2 = color.New(color.FgGreen, color.Bold).Sprint("[Δ2]")
|
||||
d3 = color.New(color.FgGreen, color.Bold).Sprint("[Δ3]")
|
||||
)
|
||||
|
||||
printDiff := func(left, right cid.Cid) {
|
||||
cmd := exec.Command("statediff", "car", "--file", tmpCar, left.String(), right.String())
|
||||
b, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("statediff failed: %s", err)
|
||||
}
|
||||
t.Log(string(b))
|
||||
}
|
||||
|
||||
bold := color.New(color.Bold).SprintfFunc()
|
||||
|
||||
// run state diffs.
|
||||
t.Log(bold("=== dumping 3-way diffs between %s, %s, %s ===", a, b, c))
|
||||
|
||||
t.Log(bold("--- %s left: %s; right: %s ---", d1, a, b))
|
||||
printDiff(vector.Post.StateTree.RootCID, actual)
|
||||
|
||||
t.Log(bold("--- %s left: %s; right: %s ---", d2, c, b))
|
||||
printDiff(vector.Pre.StateTree.RootCID, actual)
|
||||
|
||||
t.Log(bold("--- %s left: %s; right: %s ---", d3, c, a))
|
||||
printDiff(vector.Pre.StateTree.RootCID, vector.Post.StateTree.RootCID)
|
||||
}
|
||||
|
||||
// writeStateToTempCAR writes the provided roots to a temporary CAR that'll be
|
||||
// cleaned up via t.Cleanup(). It returns the full path of the temp file.
|
||||
func writeStateToTempCAR(t *testing.T, bs blockstore.Blockstore, roots ...cid.Cid) string {
|
||||
tmp, err := ioutil.TempFile("", "lotus-tests-*.car")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create temp file to dump CAR for diffing: %s", err)
|
||||
}
|
||||
// register a cleanup function to delete the CAR.
|
||||
t.Cleanup(func() {
|
||||
_ = os.Remove(tmp.Name())
|
||||
})
|
||||
|
||||
carWalkFn := func(nd format.Node) (out []*format.Link, err error) {
|
||||
for _, link := range nd.Links() {
|
||||
if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
|
||||
continue
|
||||
}
|
||||
out = append(out, link)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
var (
|
||||
offl = offline.Exchange(bs)
|
||||
blkserv = blockservice.New(bs, offl)
|
||||
dserv = merkledag.NewDAGService(blkserv)
|
||||
)
|
||||
|
||||
err = car.WriteCarWithWalker(context.Background(), dserv, roots, tmp, carWalkFn)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to dump CAR for diffing: %s", err)
|
||||
}
|
||||
_ = tmp.Close()
|
||||
return tmp.Name()
|
||||
}
|
||||
|
||||
func loadCAR(t *testing.T, vectorCAR schema.Base64EncodedBytes) blockstore.Blockstore {
|
||||
bs := blockstore.NewTemporary()
|
||||
|
||||
// Read the base64-encoded CAR from the vector, and inflate the gzip.
|
||||
buf := bytes.NewReader(vectorCAR)
|
||||
r, err := gzip.NewReader(buf)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to inflate gzipped CAR: %s", err)
|
||||
}
|
||||
defer r.Close() // nolint
|
||||
|
||||
// Load the CAR embedded in the test vector into the Blockstore.
|
||||
_, err = car.LoadCar(bs, r)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load state tree car from test vector: %s", err)
|
||||
}
|
||||
return bs
|
||||
}
|
||||
2
extern/test-vectors
vendored
2
extern/test-vectors
vendored
@ -1 +1 @@
|
||||
Subproject commit 6bea015edddde116001a4251dce3c4a9966c25d9
|
||||
Subproject commit 3a6e0b5e069b1452ce1a032aa315354d645f3ec4
|
||||
2
go.mod
2
go.mod
@ -39,7 +39,7 @@ require (
|
||||
github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
|
||||
github.com/filecoin-project/specs-actors v0.9.11
|
||||
github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796
|
||||
github.com/filecoin-project/test-vectors/schema v0.0.1
|
||||
github.com/filecoin-project/test-vectors/schema v0.0.3
|
||||
github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
|
||||
github.com/go-kit/kit v0.10.0
|
||||
github.com/go-ole/go-ole v1.2.4 // indirect
|
||||
|
||||
4
go.sum
4
go.sum
@ -272,8 +272,8 @@ github.com/filecoin-project/specs-actors v0.9.11 h1:TnpG7HAeiUrfj0mJM7UaPW0P2137
|
||||
github.com/filecoin-project/specs-actors v0.9.11/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys=
|
||||
github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk=
|
||||
github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g=
|
||||
github.com/filecoin-project/test-vectors/schema v0.0.1 h1:5fNF76nl4qolEvcIsjc0kUADlTMVHO73tW4kXXPnsus=
|
||||
github.com/filecoin-project/test-vectors/schema v0.0.1/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E=
|
||||
github.com/filecoin-project/test-vectors/schema v0.0.3 h1:1zuBo25B3016inbygYLgYFdpJ2m1BDTbAOCgABRleiU=
|
||||
github.com/filecoin-project/test-vectors/schema v0.0.3/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
|
||||
github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as=
|
||||
github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ=
|
||||
|
||||
Loading…
Reference in New Issue
Block a user