cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

add extract-many command.

Browse Source
This commit is contained in:
Raúl Kripalani 2020-09-28 17:02:56 +01:00
parent 9f6862a456
commit 96f882860f
4 changed files with 283 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
cmd/tvx/actor_mapping.go Normal file
View File

@ -0,0 +1,44 @@
package main
import (
"reflect"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
)
var ActorMethodTable = make(map[string][]string, 64)
var Actors = map[cid.Cid]interface{}{
builtin.InitActorCodeID: builtin.MethodsInit,
builtin.CronActorCodeID: builtin.MethodsCron,
builtin.AccountActorCodeID: builtin.MethodsAccount,
builtin.StoragePowerActorCodeID: builtin.MethodsPower,
builtin.StorageMinerActorCodeID: builtin.MethodsMiner,
builtin.StorageMarketActorCodeID: builtin.MethodsMarket,
builtin.PaymentChannelActorCodeID: builtin.MethodsPaych,
builtin.MultisigActorCodeID: builtin.MethodsMultisig,
builtin.RewardActorCodeID: builtin.MethodsReward,
builtin.VerifiedRegistryActorCodeID: builtin.MethodsVerifiedRegistry,
}
func init() {
for code, methods := range Actors {
cmh, err := multihash.Decode(code.Hash()) // identity hash.
if err != nil {
panic(err)
}
var (
aname = string(cmh.Digest)
rt = reflect.TypeOf(methods)
nf = rt.NumField()
)
ActorMethodTable[aname] = append(ActorMethodTable[aname], "Send")
for i := 0; i < nf; i++ {
ActorMethodTable[aname] = append(ActorMethodTable[aname], rt.Field(i).Name)
}
}
}

50
cmd/tvx/extract.go
View File

@ -9,6 +9,7 @@ import (
"io"
"log"
"os"
"path/filepath"
"github.com/fatih/color"
@ -28,7 +29,7 @@ import (
"github.com/urfave/cli/v2"
)
var extractFlags struct {
type extractOpts struct {
id string
block string
class string
@ -37,6 +38,8 @@ var extractFlags struct {
retain string
}
var extractFlags extractOpts
var extractCmd = &cli.Command{
Name: "extract",
Description: "generate a test vector by extracting it from a live chain",
@ -94,19 +97,23 @@ func runExtract(c *cli.Context) error {
ctx := context.Background()
mcid, err := cid.Decode(extractFlags.cid)
if err != nil {
return err
}
// Make the API client.
api, closer, err := lcli.GetFullNodeAPI(c)
fapi, closer, err := lcli.GetFullNodeAPI(c)
if err != nil {
return err
}
defer closer()
msg, execTs, incTs, err := resolveFromChain(ctx, api, mcid)
return doExtract(ctx, fapi, extractFlags)
}
func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
mcid, err := cid.Decode(opts.cid)
if err != nil {
return err
}
msg, execTs, incTs, err := resolveFromChain(ctx, fapi, mcid, opts.block)
if err != nil {
return fmt.Errorf("failed to resolve message and tipsets from chain: %w", err)
}
@ -119,7 +126,7 @@ func runExtract(c *cli.Context) error {
// precursors, if any.
var allmsgs []*types.Message
for _, b := range incTs.Blocks() {
messages, err := api.ChainGetBlockMessages(ctx, b.Cid())
messages, err := fapi.ChainGetBlockMessages(ctx, b.Cid())
if err != nil {
return err
}
@ -139,7 +146,7 @@ func runExtract(c *cli.Context) error {
break
}
log.Printf("message not found in block %s; precursors found: %v; ignoring block", b.Cid(), related)
log.Printf("message not found in block %s; number of precursors found: %d; ignoring block", b.Cid(), len(related))
}
if allmsgs == nil {
@ -151,8 +158,8 @@ func runExtract(c *cli.Context) error {
var (
// create a read-through store that uses ChainGetObject to fetch unknown CIDs.
pst = NewProxyingStores(ctx, api)
g = NewSurgeon(ctx, api, pst)
pst = NewProxyingStores(ctx, fapi)
g = NewSurgeon(ctx, fapi, pst)
)
driver := conformance.NewDriver(ctx, schema.Selector{}, conformance.DriverOpts{
@ -178,7 +185,7 @@ func runExtract(c *cli.Context) error {
postroot cid.Cid
applyret *vm.ApplyRet
carWriter func(w io.Writer) error
retention = extractFlags.retain
retention = opts.retain
)
log.Printf("using state retention strategy: %s", retention)
@ -204,7 +211,7 @@ func runExtract(c *cli.Context) error {
case "accessed-actors":
log.Printf("calculating accessed actors")
// get actors accessed by message.
retain, err := g.GetAccessedActors(ctx, api, mcid)
retain, err := g.GetAccessedActors(ctx, fapi, mcid)
if err != nil {
return fmt.Errorf("failed to calculate accessed actors: %w", err)
}
@ -267,12 +274,12 @@ func runExtract(c *cli.Context) error {
return err
}
version, err := api.Version(ctx)
version, err := fapi.Version(ctx)
if err != nil {
return err
}
ntwkName, err := api.StateNetworkName(ctx)
ntwkName, err := fapi.StateNetworkName(ctx)
if err != nil {
return err
}
@ -281,7 +288,7 @@ func runExtract(c *cli.Context) error {
vector := schema.TestVector{
Class: schema.ClassMessage,
Meta: &schema.Metadata{
ID: extractFlags.id,
ID: opts.id,
// TODO need to replace schema.GenerationData with a more flexible
// data structure that makes no assumption about the traceability
// data that's being recorded; a flexible map[string]string
@ -316,7 +323,11 @@ func runExtract(c *cli.Context) error {
}
output := io.WriteCloser(os.Stdout)
if file := extractFlags.file; file != "" {
if file := opts.file; file != "" {
dir := filepath.Dir(file)
if err := os.MkdirAll(dir, 0755); err != nil {
return fmt.Errorf("unable to create directory %s: %w", dir, err)
}
output, err = os.Create(file)
if err != nil {
return err
@ -336,7 +347,7 @@ func runExtract(c *cli.Context) error {
// resolveFromChain queries the chain for the provided message, using the block CID to
// speed up the query, if provided
func resolveFromChain(ctx context.Context, api api.FullNode, mcid cid.Cid) (msg *types.Message, execTs *types.TipSet, incTs *types.TipSet, err error) {
func resolveFromChain(ctx context.Context, api api.FullNode, mcid cid.Cid, block string) (msg *types.Message, execTs *types.TipSet, incTs *types.TipSet, err error) {
// Extract the full message.
msg, err = api.ChainGetMessage(ctx, mcid)
if err != nil {
@ -345,7 +356,6 @@ func resolveFromChain(ctx context.Context, api api.FullNode, mcid cid.Cid) (msg
log.Printf("found message with CID %s: %+v", mcid, msg)
block := extractFlags.block
if block == "" {
log.Printf("locating message in blockchain")

204
cmd/tvx/extract_many.go Normal file
View File

@ -0,0 +1,204 @@
package main
import (
"context"
"encoding/csv"
"fmt"
"io"
"log"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/fatih/color"
"github.com/filecoin-project/go-state-types/exitcode"
"github.com/hashicorp/go-multierror"
"github.com/urfave/cli/v2"
lcli "github.com/filecoin-project/lotus/cli"
)
var extractManyFlags struct {
in string
outdir string
}
var extractManyCmd = &cli.Command{
Name: "extract-many",
Description: `generate many test vectors by repeateadly calling tvx extract, using a csv file as input.
The CSV file must have a format just like the following:
message_cid,receiver_code,method_num,exit_code,height,block_cid,seq
bafy2bzacedvuvgpsnwq7i7kltfap6hnp7fdmzf6lr4w34zycjrthb3v7k6zi6,fil/1/account,0,0,67972,bafy2bzacebthpxzlk7zhlkz3jfzl4qw7mdoswcxlf3rkof3b4mbxfj3qzfk7w,1
bafy2bzacedwicofymn4imgny2hhbmcm4o5bikwnv3qqgohyx73fbtopiqlro6,fil/1/account,0,0,67860,bafy2bzacebj7beoxyzll522o6o76mt7von4psn3tlvunokhv4zhpwmfpipgti,2
...
The first row MUST be a header row. At the bare minimum, those seven fields
must appear, in the order specified. Extra fields are accepted, but always
after these compulsory six.
`,
Action: runExtractMany,
Flags: []cli.Flag{
&repoFlag,
&cli.StringFlag{
Name: "in",
Usage: "path to input file (csv)",
Destination: &extractManyFlags.in,
},
&cli.StringFlag{
Name: "out-dir",
Usage: "output directory",
Destination: &extractManyFlags.outdir,
},
},
}
func runExtractMany(c *cli.Context) error {
// LOTUS_DISABLE_VM_BUF disables what's called "VM state tree buffering",
// which stashes write operations in a BufferedBlockstore
// (https://github.com/filecoin-project/lotus/blob/b7a4dbb07fd8332b4492313a617e3458f8003b2a/lib/bufbstore/buf_bstore.go#L21)
// such that they're not written until the VM is actually flushed.
//
// For some reason, the standard behaviour was not working for me (raulk),
// and disabling it (such that the state transformations are written immediately
// to the blockstore) worked.
_ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea")
ctx := context.Background()
// Make the API client.
fapi, closer, err := lcli.GetFullNodeAPI(c)
if err != nil {
return err
}
defer closer()
var (
in = extractManyFlags.in
outdir = extractManyFlags.outdir
)
if in == "" {
return fmt.Errorf("input file not provided")
}
if outdir == "" {
return fmt.Errorf("output dir not provided")
}
// Open the CSV file for reading.
f, err := os.Open(in)
if err != nil {
return fmt.Errorf("could not open file %s: %w", in, err)
}
// Ensure the output directory exists.
if err := os.MkdirAll(outdir, 0755); err != nil {
return fmt.Errorf("could not create output dir %s: %w", outdir, err)
}
// Create a CSV reader and validate the header row.
reader := csv.NewReader(f)
if header, err := reader.Read(); err != nil {
return fmt.Errorf("failed to read header from csv: %w", err)
} else if l := len(header); l < 7 {
return fmt.Errorf("insufficient number of fields: %d", l)
} else if f := header[0]; f != "message_cid" {
return fmt.Errorf("csv sanity check failed: expected first field in header to be 'message_cid'; was: %s", f)
} else {
log.Println(color.GreenString("csv sanity check succeeded; header contains fields: %v", header))
}
var generated []string
merr := new(multierror.Error)
// Read each row and extract the requested message.
for {
row, err := reader.Read()
if err == io.EOF {
break
} else if err != nil {
return fmt.Errorf("failed to read row: %w", err)
}
var (
cid = row[0]
actorcode = row[1]
methodnumstr = row[2]
exitcodestr = row[3]
_ = row[4]
block = row[5]
seq = row[6]
exit int
methodnum int
methodname string
)
// Parse the exit code.
if exit, err = strconv.Atoi(exitcodestr); err != nil {
return fmt.Errorf("invalid exitcode number: %d", exit)
}
// Parse the method number.
if methodnum, err = strconv.Atoi(methodnumstr); err != nil {
return fmt.Errorf("invalid method number: %s", methodnumstr)
}
// Lookup the method in actor method table.
if m, ok := ActorMethodTable[actorcode]; !ok {
return fmt.Errorf("unrecognized actor: %s", actorcode)
} else if methodnum >= len(m) {
return fmt.Errorf("unrecognized method number for actor %s: %d", actorcode, methodnum)
} else {
methodname = m[methodnum]
}
// exitcode string representations are of kind ErrType(0); strip out
// the number portion.
exitcodename := strings.Split(exitcode.ExitCode(exit).String(), "(")[0]
// replace the slashes in the actor code name with underscores.
actorcodename := strings.ReplaceAll(actorcode, "/", "_")
// Compute the ID of the vector.
id := fmt.Sprintf("extracted-msg-%s-%s-%s-%s", actorcodename, methodname, exitcodename, seq)
// Vector filename, using a base of outdir.
file := filepath.Join(outdir, actorcodename, methodname, exitcodename, id) + ".json"
log.Println(color.YellowString("processing message id: %s", id))
opts := extractOpts{
id: id,
block: block,
class: "message",
cid: cid,
file: file,
retain: "accessed-cids",
}
if err := doExtract(ctx, fapi, opts); err != nil {
merr = multierror.Append(err, fmt.Errorf("failed to extract vector for message %s: %w", cid, err))
continue
}
log.Println(color.MagentaString("generated file: %s", file))
generated = append(generated, file)
}
if len(generated) == 0 {
log.Println("no files generated")
} else {
log.Println("files generated:")
for _, g := range generated {
log.Println(g)
}
}
if merr.ErrorOrNil() != nil {
log.Println(color.YellowString("done processing with errors: %s"))
} else {
log.Println(color.GreenString("done processing with no errors"))
}
return merr.ErrorOrNil()
}

6
cmd/tvx/main.go
View File

@ -23,7 +23,7 @@ var repoFlag = cli.StringFlag{
func main() {
app := &cli.App{
Name: "tvx",
Description: `tvx is a tool for extracting and executing test vectors. It has two subcommands.
Description: `tvx is a tool for extracting and executing test vectors. It has three subcommands.
tvx extract extracts a test vector from a live network. It requires access to
a Filecoin client that exposes the standard JSON-RPC API endpoint. Only
@ -32,6 +32,9 @@ func main() {
tvx exec executes test vectors against Lotus. Either you can supply one in a
file, or many as an ndjson stdin stream.
tvx extract-many performs a batch extraction of many messages, supplied in a
CSV file. Refer to the help of that subcommand for more info.
SETTING THE JSON-RPC API ENDPOINT
You can set the JSON-RPC API endpoint through one of the following methods.
@ -53,6 +56,7 @@ func main() {
Commands: []*cli.Command{
extractCmd,
execCmd,
extractManyCmd,
},
}