Expand extraction / replay to

* allow preceeding messages
* use `stateCall` rather than state replay
* work on state root CID rather than tipset
This commit is contained in:
Will Scott 2020-08-28 13:41:16 -07:00
parent 9cebb09a5f
commit ed12b83b4e
3 changed files with 90 additions and 27 deletions

View File

@ -6,12 +6,12 @@ import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/ipfs/go-cid"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/oni/tvx/lotus"
"github.com/filecoin-project/oni/tvx/schema"
@ -97,6 +97,7 @@ func runExtractMsg(c *cli.Context) error {
}
retain = append(retain, builtin.RewardActorAddr)
retain = append(retain, builtin.BurntFundsActorAddr)
fmt.Println("accessed actors:")
for _, k := range retain {
@ -116,10 +117,7 @@ func runExtractMsg(c *cli.Context) error {
return err
}
// Warn if the block contains more messages from this sender, preceding the
// extracted message.
//
// TODO https://github.com/filecoin-project/oni/issues/195
neededPrecursorMsgs := make([]*types.Message, 0)
for _, b := range includedTs.Blocks() {
messages, err := api.ChainGetBlockMessages(ctx, b.Cid())
if err != nil {
@ -130,7 +128,15 @@ func runExtractMsg(c *cli.Context) error {
break
}
if other.From == msg.From && other.Nonce < msg.Nonce {
log.Printf("WARN: tipset includes preceding message with lower nonce from same sender; this extraction won't work")
included := false
for _, o := range neededPrecursorMsgs {
if o.Cid() == other.Cid() {
included = true
}
}
if !included {
neededPrecursorMsgs = append(neededPrecursorMsgs, other)
}
}
}
for _, m := range messages.SecpkMessages {
@ -138,19 +144,36 @@ func runExtractMsg(c *cli.Context) error {
break
}
if m.Message.From == msg.From && m.Message.Nonce < msg.Nonce {
log.Printf("WARN: tipset includes preceding message with lower nonce from same sender; this extraction won't work")
included := false
for _, o := range neededPrecursorMsgs {
if o.Cid() == m.Message.Cid() {
included = true
}
}
if !included {
neededPrecursorMsgs = append(neededPrecursorMsgs, &m.Message)
}
}
}
}
fmt.Println("getting the _before_ filtered state tree")
preroot, err := g.GetMaskedStateTree(includedTs.Parents(), retain)
tree, err := g.GetStateTreeRootFromTipset(includedTs.Parents())
driver := lotus.NewDriver(ctx)
for _, pm := range neededPrecursorMsgs {
_, tree, err = driver.ExecuteMessage(pm, tree, pst.Blockstore, execTs.Height())
if err != nil {
return fmt.Errorf("Failed to execute preceding message: %w", err)
}
}
preroot, err := g.GetMaskedStateTree(tree, retain)
if err != nil {
return err
}
driver := lotus.NewDriver(ctx)
_, postroot, err := driver.ExecuteMessage(msg, preroot, pst.Blockstore, execTs.Height())
if err != nil {
return fmt.Errorf("failed to execute message: %w", err)

View File

@ -1182,6 +1182,7 @@ github.com/prometheus/procfs v0.1.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
github.com/raulk/clock v1.1.0 h1:dpb29+UKMbLqiU/jqIJptgLR1nn23HLgMY0sTCDza5Y=
github.com/raulk/clock v1.1.0/go.mod h1:3MpVxdZ/ODBQDxbN+kzshf5OSZwPjtMDx6BBXBmOeY0=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@ -1277,6 +1278,7 @@ github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFd
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g=
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
github.com/uber/jaeger-client-go v2.23.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
@ -1340,11 +1342,14 @@ github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:
github.com/whyrusleeping/yamux v1.1.5/go.mod h1:E8LnQQ8HKx5KD29HZFUwM1PxCOdPRzGwur1mcYhXcD8=
github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829/go.mod h1:h/1PEBwj7Ym/8kOuMWvO2ujZ6Lt+TMbySEXNhjjR87I=
github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245/go.mod h1:C+diUUz7pxhNY6KAoLgrTYARGWnt82zWTylZlxT92vk=
github.com/xorcare/golden v0.6.0/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ=
github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 h1:oWgZJmC1DorFZDpfMfWg7xk29yEOZiXmo/wZl+utTI8=
github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw=
go.dedis.ch/kyber/v3 v3.0.4/go.mod h1:OzvaEnPvKlyrWyp3kGXlFdp7ap1VC6RkZDTaPikqhsQ=
go.dedis.ch/kyber/v3 v3.0.9/go.mod h1:rhNjUUg6ahf8HEg5HUvVBYoWY4boAafX8tYxX+PS+qg=
@ -1421,6 +1426,8 @@ golang.org/x/crypto v0.0.0-20200427165652-729f1e841bcc/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM=
golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -1454,6 +1461,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -1496,6 +1504,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/
golang.org/x/net v0.0.0-20200519113804-d87ec0cfa476/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM=
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@ -1633,6 +1642,7 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK
golang.org/x/tools v0.0.0-20200318150045-ba25ddc85566/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4 h1:kDtqNkeBrZb8B+atrj50B5XLHpzXXqcCdZPP/ApQ5NY=
golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
golang.org/x/tools v0.0.0-20200711155855-7342f9734a7d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
@ -1746,6 +1756,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
@ -1763,6 +1774,11 @@ howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqp
howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=
launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54=
launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM=
modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

View File

@ -15,6 +15,7 @@ import (
"github.com/filecoin-project/specs-actors/actors/builtin"
init_ "github.com/filecoin-project/specs-actors/actors/builtin/init"
"github.com/filecoin-project/specs-actors/actors/util/adt"
"github.com/filecoin-project/lotus/chain/state"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-ipld-format"
@ -39,15 +40,24 @@ func NewSurgeon(ctx context.Context, api api.FullNode, stores *Stores) *Surgeon
}
}
func (sg *Surgeon) GetStateTreeRootFromTipset(tsk types.TipSetKey) (cid.Cid, error) {
ts, err := sg.api.ChainGetTipSet(sg.ctx, tsk)
if err != nil {
return cid.Undef, err
}
return ts.ParentState(), nil
}
// GetMaskedStateTree trims the state tree at the supplied tipset to contain
// only the state of the actors in the retain set. It also "dives" into some
// singleton system actors, like the init actor, to trim the state so as to
// compute a minimal state tree. In the future, thid method will dive into
// other system actors like the power actor and the market actor.
func (sg *Surgeon) GetMaskedStateTree(tsk types.TipSetKey, retain []address.Address) (cid.Cid, error) {
func (sg *Surgeon) GetMaskedStateTree(previousRoot cid.Cid, retain []address.Address) (cid.Cid, error) {
stateMap := adt.MakeEmptyMap(sg.stores.ADTStore)
initState, err := sg.loadInitActor(tsk)
initState, err := sg.loadInitActor(previousRoot)
if err != nil {
return cid.Undef, err
}
@ -67,7 +77,7 @@ func (sg *Surgeon) GetMaskedStateTree(tsk types.TipSetKey, retain []address.Addr
return cid.Undef, err
}
err = sg.pluckActorStates(tsk, resolved, stateMap)
err = sg.pluckActorStates(previousRoot, resolved, stateMap)
if err != nil {
return cid.Undef, err
}
@ -89,12 +99,17 @@ func (sg *Surgeon) GetAccessedActors(ctx context.Context, a api.FullNode, mid ci
return nil, err
}
msgObj, err := a.ChainGetMessage(ctx, mid)
if err != nil {
return nil, err
}
ts, err := a.ChainGetTipSet(ctx, msgInfo.TipSet)
if err != nil {
return nil, err
}
trace, err := a.StateReplay(ctx, ts.Parents(), mid)
trace, err := a.StateCall(ctx, msgObj, ts.Parents())
if err != nil {
return nil, fmt.Errorf("could not replay msg: %w", err)
}
@ -136,11 +151,17 @@ func (sg *Surgeon) WriteCAR(w io.Writer, roots ...cid.Cid) error {
// pluckActorStates plucks the state from the supplied actors at the given
// tipset, and places it into the supplied state map.
func (sg *Surgeon) pluckActorStates(tsk types.TipSetKey, pluck []address.Address, stateMap *adt.Map) error {
func (sg *Surgeon) pluckActorStates(stateRoot cid.Cid, pluck []address.Address, stateMap *adt.Map) error {
st, err := state.LoadStateTree(sg.stores.CBORStore, stateRoot)
if err != nil {
return err
}
for _, a := range pluck {
actor, err := sg.api.StateGetActor(sg.ctx, a, tsk)
actor, err := st.GetActor(a)
if err != nil {
return err
continue
//return fmt.Errorf("get actor %s failed: %w", a, err)
}
err = stateMap.Put(adt.AddrKey(a), actor)
@ -208,17 +229,14 @@ func (sg *Surgeon) retainInitEntries(oldState *init_.State, retain []address.Add
newAddrs := adt.MakeEmptyMap(sg.stores.ADTStore)
for _, r := range retain {
if r.Protocol() == address.ID {
// skip over ID addresses; they don't need a mapping in the init actor.
continue
}
var d cbg.Deferred
if _, err := oldAddrs.Get(adt.AddrKey(r), &d); err != nil {
return nil, err
}
if err := newAddrs.Put(adt.AddrKey(r), &d); err != nil {
return nil, err
if d.Raw != nil {
if err := newAddrs.Put(adt.AddrKey(r), &d); err != nil {
return nil, err
}
}
}
@ -261,10 +279,16 @@ func (sg *Surgeon) resolveAddresses(orig []address.Address, ist *init_.State) (r
}
// loadInitActor loads the init actor state from a given tipset.
func (sg *Surgeon) loadInitActor(tsk types.TipSetKey) (initState *init_.State, err error) {
log.Printf("loading the init actor for tipset: %s", tsk)
func (sg *Surgeon) loadInitActor(stateRoot cid.Cid) (initState *init_.State, err error) {
log.Printf("loading the init actor for root: %s", stateRoot)
actor, err := sg.api.StateGetActor(sg.ctx, builtin.InitActorAddr, tsk)
st, err := state.LoadStateTree(sg.stores.CBORStore, stateRoot)
if err != nil {
return initState, err
}
//sg.api.StateGetActor(sg.ctx, builtin.InitActorAddr, tsk)
actor, err := st.GetActor(builtin.InitActorAddr)
if err != nil {
return initState, err
}