tvx: precursor selection modes; canonical message fetching; basefee.

cmd/tvx/extract.go

@@ -21,21 +21,27 @@ import (
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/filecoin-project/lotus/conformance"
 
-	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
+
 	"github.com/filecoin-project/test-vectors/schema"
 
 	"github.com/ipfs/go-cid"
 	"github.com/urfave/cli/v2"
 )
 
+const (
+	PrecursorSelectAll    = "all"
+	PrecursorSelectSender = "sender"
+)
+
 type extractOpts struct {
-	id     string
+	id        string
-	block  string
+	block     string
-	class  string
+	class     string
-	cid    string
+	cid       string
-	file   string
+	file      string
-	retain string
+	retain    string
+	precursor string
 }
 
 var extractFlags extractOpts
@@ -81,6 +87,16 @@ var extractCmd = &cli.Command{
 			Value:       "accessed-cids",
 			Destination: &extractFlags.retain,
 		},
+		&cli.StringFlag{
+			Name: "precursor-select",
+			Usage: "precursors to apply; values: 'all', 'sender'; 'all' selects all preceding" +
+				"messages in the canonicalised tipset, 'sender' selects only preceding messages from the same" +
+				"sender. Usually, 'sender' is a good tradeoff and gives you sufficient accuracy. If the receipt sanity" +
+				"check fails due to gas reasons, switch to 'all', as previous messages in the tipset may have" +
+				"affected state in a disruptive way",
+			Value:       "sender",
+			Destination: &extractFlags.precursor,
+		},
 	},
 }
 
@@ -124,46 +140,38 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
 		return fmt.Errorf("failed while fetching circulating supply: %w", err)
 	}
 
-	circSupply := circSupplyDetail.FilCirculating.Int64()
+	circSupply := circSupplyDetail.FilCirculating
 
 	log.Printf("message was executed in tipset: %s", execTs.Key())
 	log.Printf("message was included in tipset: %s", incTs.Key())
 	log.Printf("circulating supply at inclusion tipset: %d", circSupply)
-	log.Printf("finding precursor messages")
+	log.Printf("finding precursor messages using mode: %s", opts.precursor)
 
-	// Iterate through blocks, finding the one that contains the message and its
+	// Fetch messages in canonical order from inclusion tipset.
-	// precursors, if any.
+	msgs, err := fapi.ChainGetParentMessages(ctx, execTs.Blocks()[0].Cid())
-	var allmsgs []*types.Message
+	if err != nil {
-	for _, b := range incTs.Blocks() {
+		return fmt.Errorf("failed to fetch messages in canonical order from inclusion tipset: %w", err)
-		messages, err := fapi.ChainGetBlockMessages(ctx, b.Cid())
-		if err != nil {
-			return err
-		}
-
-		related, found, err := findMsgAndPrecursors(messages, msg)
-		if err != nil {
-			return fmt.Errorf("invariant failed while scanning messages in block %s: %w", b.Cid(), err)
-		}
-
-		if found {
-			var mcids []cid.Cid
-			for _, m := range related {
-				mcids = append(mcids, m.Cid())
-			}
-			log.Printf("found message in block %s; precursors: %v", b.Cid(), mcids[:len(mcids)-1])
-			allmsgs = related
-			break
-		}
-
-		log.Printf("message not found in block %s; number of precursors found: %d; ignoring block", b.Cid(), len(related))
 	}
 
-	if allmsgs == nil {
+	related, found, err := findMsgAndPrecursors(opts.precursor, msg, msgs)
-		// Message was not found; abort.
+	if err != nil {
-		return fmt.Errorf("did not find a block containing the message")
+		return fmt.Errorf("failed while finding message and precursors: %w", err)
 	}
 
-	precursors := allmsgs[:len(allmsgs)-1]
+	if !found {
+		return fmt.Errorf("message not found; precursors found: %d", len(related))
+	}
+
+	var (
+		precursors     = related[:len(related)-1]
+		precursorsCids []cid.Cid
+	)
+
+	for _, p := range precursors {
+		precursorsCids = append(precursorsCids, p.Cid())
+	}
+
+	log.Println(color.GreenString("found message; precursors (count: %d): %v", len(precursors), precursorsCids))
 
 	var (
 		// create a read-through store that uses ChainGetObject to fetch unknown CIDs.
@@ -179,11 +187,20 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
 	root := incTs.ParentState()
 	log.Printf("base state tree root CID: %s", root)
 
+	basefee := incTs.Blocks()[0].ParentBaseFee
+	log.Printf("basefee: %s", basefee)
+
 	// on top of that state tree, we apply all precursors.
 	log.Printf("number of precursors to apply: %d", len(precursors))
 	for i, m := range precursors {
 		log.Printf("applying precursor %d, cid: %s", i, m.Cid())
-		_, root, err = driver.ExecuteMessage(pst.Blockstore, root, execTs.Height(), m, &circSupplyDetail.FilCirculating)
+		_, root, err = driver.ExecuteMessage(pst.Blockstore, conformance.ExecuteMessageParams{
+			Preroot:    root,
+			Epoch:      execTs.Height(),
+			Message:    m,
+			CircSupply: &circSupplyDetail.FilCirculating,
+			BaseFee:    &basefee,
+		})
 		if err != nil {
 			return fmt.Errorf("failed to execute precursor message: %w", err)
 		}
@@ -208,7 +225,13 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
 		tbs.StartTracing()
 
 		preroot = root
-		applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg, &circSupplyDetail.FilCirculating)
+		applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, conformance.ExecuteMessageParams{
+			Preroot:    preroot,
+			Epoch:      execTs.Height(),
+			Message:    msg,
+			CircSupply: &circSupplyDetail.FilCirculating,
+			BaseFee:    &basefee,
+		})
 		if err != nil {
 			return fmt.Errorf("failed to execute message: %w", err)
 		}
@@ -233,7 +256,13 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
 		if err != nil {
 			return err
 		}
-		applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg, &circSupplyDetail.FilCirculating)
+		applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, conformance.ExecuteMessageParams{
+			Preroot:    preroot,
+			Epoch:      execTs.Height(),
+			Message:    msg,
+			CircSupply: &circSupplyDetail.FilCirculating,
+			BaseFee:    &basefee,
+		})
 		if err != nil {
 			return fmt.Errorf("failed to execute message: %w", err)
 		}
@@ -248,21 +277,39 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
 	log.Printf("message applied; preroot: %s, postroot: %s", preroot, postroot)
 	log.Println("performing sanity check on receipt")
 
-	receipt := &schema.Receipt{
+	// TODO sometimes this returns a nil receipt and no error ¯\_(ツ)_/¯
-		ExitCode:    int64(applyret.ExitCode),
+	//  ex: https://filfox.info/en/message/bafy2bzacebpxw3yiaxzy2bako62akig46x3imji7fewszen6fryiz6nymu2b2
-		ReturnValue: applyret.Return,
+	//  This code is lenient and skips receipt comparison in case of a nil receipt.
-		GasUsed:     applyret.GasUsed,
+	rec, err := fapi.StateGetReceipt(ctx, mcid, execTs.Key())
+	if err != nil {
+		return fmt.Errorf("failed to find receipt on chain: %w", err)
 	}
+	log.Printf("found receipt: %+v", rec)
 
-	reporter := new(conformance.LogReporter)
+	// generate the schema receipt; if we got
-	conformance.AssertMsgResult(reporter, receipt, applyret, "as locally executed")
+	var receipt *schema.Receipt
-	if reporter.Failed() {
+	if rec != nil {
-		log.Println(color.RedString("receipt sanity check failed; aborting"))
+		receipt = &schema.Receipt{
-		return fmt.Errorf("vector generation aborted")
+			ExitCode:    int64(rec.ExitCode),
+			ReturnValue: rec.Return,
+			GasUsed:     rec.GasUsed,
+		}
+		reporter := new(conformance.LogReporter)
+		conformance.AssertMsgResult(reporter, receipt, applyret, "as locally executed")
+		if reporter.Failed() {
+			log.Println(color.RedString("receipt sanity check failed; aborting"))
+			return fmt.Errorf("vector generation aborted")
+		}
+		log.Println(color.GreenString("receipt sanity check succeeded"))
+	} else {
+		receipt = &schema.Receipt{
+			ExitCode:    int64(applyret.ExitCode),
+			ReturnValue: applyret.Return,
+			GasUsed:     applyret.GasUsed,
+		}
+		log.Println(color.YellowString("skipping receipts comparison; we got back a nil receipt from lotus"))
 	}
 
-	log.Println(color.GreenString("receipt sanity check succeeded"))
-
 	log.Println("generating vector")
 	msgBytes, err := msg.Serialize()
 	if err != nil {
@@ -312,7 +359,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
 		CAR: out.Bytes(),
 		Pre: &schema.Preconditions{
 			Epoch:      int64(execTs.Height()),
-			CircSupply: &circSupply,
+			CircSupply: circSupply.Int,
+			BaseFee:    basefee.Int,
 			StateTree: &schema.StateTree{
 				RootCID: preroot,
 			},
@@ -428,41 +476,22 @@ func fetchThisAndPrevTipset(ctx context.Context, api api.FullNode, target types.
 	return targetTs, prevTs, nil
 }
 
-// findMsgAndPrecursors scans the messages in a block to locate the supplied
+// findMsgAndPrecursors ranges through the canonical messages slice, locating
-// message, looking into the BLS or SECP section depending on the sender's
+// the target message and returning precursors in accordance to the supplied
-// address type.
+// mode.
-//
+func findMsgAndPrecursors(mode string, target *types.Message, msgs []api.Message) (related []*types.Message, found bool, err error) {
-// It returns any precursors (if they exist), and the found message (if found),
+	// Range through canonicalised messages, selecting only the precursors based
-// in a slice.
+	// on selection mode.
-//
+	for _, other := range msgs {
-// It also returns a boolean indicating whether the message was actually found.
+		switch {
-//
+		case mode == PrecursorSelectAll:
-// This function also asserts invariants, and if those fail, it returns an error.
+			fallthrough
-func findMsgAndPrecursors(messages *api.BlockMessages, target *types.Message) (related []*types.Message, found bool, err error) {
+		case mode == PrecursorSelectSender && other.Message.From == target.From:
-	// Decide which block of messages to process, depending on whether the
+			related = append(related, other.Message)
-	// sender is a BLS or a SECP account.
-	input := messages.BlsMessages
-	if senderKind := target.From.Protocol(); senderKind == address.SECP256K1 {
-		input = make([]*types.Message, 0, len(messages.SecpkMessages))
-		for _, sm := range messages.SecpkMessages {
-			input = append(input, &sm.Message)
-		}
-	}
-
-	for _, other := range input {
-		if other.From != target.From {
-			continue
-		}
-
-		// this message is from the same sender, so it's related.
-		related = append(related, other)
-
-		if other.Nonce > target.Nonce {
-			return nil, false, fmt.Errorf("a message with nonce higher than the target was found before the target; offending mcid: %s", other.Cid())
 		}
 
 		// this message is the target; we're done.
-		if other.Cid() == target.Cid() {
+		if other.Cid == target.Cid() {
 			return related, true, nil
 		}
 	}

cmd/tvx/extract_many.go

@@ -111,8 +111,12 @@ func runExtractMany(c *cli.Context) error {
 		log.Println(color.GreenString("csv sanity check succeeded; header contains fields: %v", header))
 	}
 
-	var generated []string
+	var (
-	merr := new(multierror.Error)
+		generated []string
+		merr      = new(multierror.Error)
+		retry     []extractOpts // to retry with 'canonical' precursor selection mode
+	)
+
 	// Read each row and extract the requested message.
 	for {
 		row, err := reader.Read()
@@ -164,19 +168,21 @@ func runExtractMany(c *cli.Context) error {
 		// Vector filename, using a base of outdir.
 		file := filepath.Join(outdir, actorcodename, methodname, exitcodename, id) + ".json"
 
-		log.Println(color.YellowString("processing message id: %s", id))
+		log.Println(color.YellowString("processing message cid with 'sender' precursor mode: %s", id))
 
 		opts := extractOpts{
-			id:     id,
+			id:        id,
-			block:  block,
+			block:     block,
-			class:  "message",
+			class:     "message",
-			cid:    cid,
+			cid:       cid,
-			file:   file,
+			file:      file,
-			retain: "accessed-cids",
+			retain:    "accessed-cids",
+			precursor: PrecursorSelectSender,
 		}
 
 		if err := doExtract(ctx, fapi, opts); err != nil {
-			merr = multierror.Append(err, fmt.Errorf("failed to extract vector for message %s: %w", cid, err))
+			log.Println(color.RedString("failed to extract vector for message %s: %s; queuing for 'canonical' precursor selection", cid, err))
+			retry = append(retry, opts)
 			continue
 		}
 
@@ -185,6 +191,21 @@ func runExtractMany(c *cli.Context) error {
 		generated = append(generated, file)
 	}
 
+	log.Printf("extractions to try with canonical precursor selection mode: %d", len(retry))
+
+	for _, r := range retry {
+		log.Printf("retrying %s: %s", r.cid, r.id)
+
+		r.precursor = PrecursorSelectAll
+		if err := doExtract(ctx, fapi, r); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("failed to extract vector for message %s: %w", r.cid, err))
+			continue
+		}
+
+		log.Println(color.MagentaString("generated file: %s", r.file))
+		generated = append(generated, r.file)
+	}
+
 	if len(generated) == 0 {
 		log.Println("no files generated")
 	} else {
@@ -195,7 +216,7 @@ func runExtractMany(c *cli.Context) error {
 	}
 
 	if merr.ErrorOrNil() != nil {
-		log.Println(color.YellowString("done processing with errors: %s"))
+		log.Println(color.YellowString("done processing with errors: %s", err))
 	} else {
 		log.Println(color.GreenString("done processing with no errors"))
 	}

conformance/driver.go

@@ -2,6 +2,7 @@ package conformance
 
 import (
 	"context"
+	"os"
 
 	"github.com/filecoin-project/lotus/chain/state"
 	"github.com/filecoin-project/lotus/chain/stmgr"
@@ -23,15 +24,14 @@ import (
 	ds "github.com/ipfs/go-datastore"
 )
 
-// DefaultCirculatingSupply is the fallback circulating supply returned by
-// the driver's CircSupplyCalculator function, used if the vector specifies
-// no circulating supply.
-var DefaultCirculatingSupply = types.TotalFilecoinInt
-
 var (
-	// BaseFee to use in the VM.
+	// DefaultCirculatingSupply is the fallback circulating supply returned by
-	// TODO make parametrisable through vector.
+	// the driver's CircSupplyCalculator function, used if the vector specifies
-	BaseFee = abi.NewTokenAmount(100)
+	// no circulating supply.
+	DefaultCirculatingSupply = types.TotalFilecoinInt
+
+	// DefaultBaseFee to use in the VM, if one is not supplied in the vector.
+	DefaultBaseFee = abi.NewTokenAmount(100)
 )
 
 type Driver struct {
@@ -139,24 +139,46 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot
 	return ret, nil
 }
 
+type ExecuteMessageParams struct {
+	Preroot    cid.Cid
+	Epoch      abi.ChainEpoch
+	Message    *types.Message
+	CircSupply *abi.TokenAmount
+	BaseFee    *abi.TokenAmount
+}
+
 // ExecuteMessage executes a conformance test vector message in a temporary VM.
-func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch abi.ChainEpoch, msg *types.Message, circSupply *abi.TokenAmount) (*vm.ApplyRet, cid.Cid, error) {
+func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, params ExecuteMessageParams) (*vm.ApplyRet, cid.Cid, error) {
+	if !d.vmFlush {
+		// do not flush the VM, just the state tree; this should be used with
+		// LOTUS_DISABLE_VM_BUF enabled, so writes will anyway be visible.
+		_ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea")
+	}
+
+	basefee := DefaultBaseFee
+	if params.BaseFee != nil {
+		basefee = *params.BaseFee
+	}
+
+	circSupply := DefaultCirculatingSupply
+	if params.CircSupply != nil {
+		circSupply = *params.CircSupply
+	}
+
 	// dummy state manager; only to reference the GetNetworkVersion method,
 	// which does not depend on state.
 	sm := new(stmgr.StateManager)
+
 	vmOpts := &vm.VMOpts{
-		StateBase: preroot,
+		StateBase: params.Preroot,
-		Epoch:     epoch,
+		Epoch:     params.Epoch,
 		Rand:      &testRand{}, // TODO always succeeds; need more flexibility.
 		Bstore:    bs,
 		Syscalls:  mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility.
 		CircSupplyCalc: func(_ context.Context, _ abi.ChainEpoch, _ *state.StateTree) (abi.TokenAmount, error) {
-			if circSupply != nil {
+			return circSupply, nil
-				return *circSupply, nil
-			}
-			return DefaultCirculatingSupply, nil
 		},
-		BaseFee:     BaseFee,
+		BaseFee:     basefee,
 		NtwkVersion: sm.GetNtwkVersion,
 	}
 
@@ -174,7 +196,7 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch
 
 	lvm.SetInvoker(invoker)
 
-	ret, err := lvm.ApplyMessage(d.ctx, toChainMsg(msg))
+	ret, err := lvm.ApplyMessage(d.ctx, toChainMsg(params.Message))
 	if err != nil {
 		return nil, cid.Undef, err
 	}
@@ -185,8 +207,6 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch
 		// recursive copoy from the temporary blcokstore to the real blockstore.
 		root, err = lvm.Flush(d.ctx)
 	} else {
-		// do not flush the VM, just the state tree; this should be used with
-		// LOTUS_DISABLE_VM_BUF enabled, so writes will anyway be visible.
 		root, err = lvm.StateTree().(*state.StateTree).Flush(d.ctx)
 	}
 

conformance/runner.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/fatih/color"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/exitcode"
 	"github.com/filecoin-project/test-vectors/schema"
 	"github.com/ipfs/go-blockservice"
@@ -43,14 +44,20 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) {
 	}
 
 	// Create a new Driver.
-	driver := NewDriver(ctx, vector.Selector, DriverOpts{})
+	driver := NewDriver(ctx, vector.Selector, DriverOpts{DisableVMFlush: true})
 
 	var circSupply *abi.TokenAmount
 	if cs := vector.Pre.CircSupply; cs != nil {
-		ta := abi.NewTokenAmount(*cs)
+		ta := big.NewFromGo(cs)
 		circSupply = &ta
 	}
 
+	var basefee *abi.TokenAmount
+	if bf := vector.Pre.BaseFee; bf != nil {
+		ta := big.NewFromGo(bf)
+		basefee = &ta
+	}
+
 	// Apply every message.
 	for i, m := range vector.ApplyMessages {
 		msg, err := types.DecodeMessage(m.Bytes)
@@ -65,7 +72,13 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) {
 
 		// Execute the message.
 		var ret *vm.ApplyRet
-		ret, root, err = driver.ExecuteMessage(bs, root, abi.ChainEpoch(epoch), msg, circSupply)
+		ret, root, err = driver.ExecuteMessage(bs, ExecuteMessageParams{
+			Preroot:    root,
+			Epoch:      abi.ChainEpoch(epoch),
+			Message:    msg,
+			CircSupply: circSupply,
+			BaseFee:    basefee,
+		})
 		if err != nil {
 			r.Fatalf("fatal failure when executing message: %s", err)
 		}

go.mod

@@ -38,7 +38,7 @@ require (
 	github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
 	github.com/filecoin-project/specs-actors v0.9.11
 	github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796
-	github.com/filecoin-project/test-vectors/schema v0.0.2
+	github.com/filecoin-project/test-vectors/schema v0.0.3
 	github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
 	github.com/go-kit/kit v0.10.0
 	github.com/go-ole/go-ole v1.2.4 // indirect

go.sum

@@ -258,8 +258,8 @@ github.com/filecoin-project/specs-actors v0.9.11 h1:TnpG7HAeiUrfj0mJM7UaPW0P2137
 github.com/filecoin-project/specs-actors v0.9.11/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys=
 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk=
 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g=
-github.com/filecoin-project/test-vectors/schema v0.0.2 h1:/Pp//88WBXe0h+ksntdL2HpEgAmbwXrftAfeVG39zdY=
+github.com/filecoin-project/test-vectors/schema v0.0.3 h1:1zuBo25B3016inbygYLgYFdpJ2m1BDTbAOCgABRleiU=
-github.com/filecoin-project/test-vectors/schema v0.0.2/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E=
+github.com/filecoin-project/test-vectors/schema v0.0.3/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as=
 github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ=