From ed12b83b4e6120196d75bb4153ef2ca71d7ed54d Mon Sep 17 00:00:00 2001 From: Will Scott Date: Fri, 28 Aug 2020 13:41:16 -0700 Subject: [PATCH] Expand extraction / replay to * allow preceeding messages * use `stateCall` rather than state replay * work on state root CID rather than tipset --- tvx/extract_msg.go | 43 ++++++++++++++++++++++++-------- tvx/go.sum | 16 ++++++++++++ tvx/state/surgeon.go | 58 +++++++++++++++++++++++++++++++------------- 3 files changed, 90 insertions(+), 27 deletions(-) diff --git a/tvx/extract_msg.go b/tvx/extract_msg.go index 5648ad5a3..48a21d7ea 100644 --- a/tvx/extract_msg.go +++ b/tvx/extract_msg.go @@ -6,12 +6,12 @@ import ( "context" "encoding/json" "fmt" - "log" "os" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/ipfs/go-cid" "github.com/urfave/cli/v2" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/oni/tvx/lotus" "github.com/filecoin-project/oni/tvx/schema" @@ -97,6 +97,7 @@ func runExtractMsg(c *cli.Context) error { } retain = append(retain, builtin.RewardActorAddr) + retain = append(retain, builtin.BurntFundsActorAddr) fmt.Println("accessed actors:") for _, k := range retain { @@ -116,10 +117,7 @@ func runExtractMsg(c *cli.Context) error { return err } - // Warn if the block contains more messages from this sender, preceding the - // extracted message. - // - // TODO https://github.com/filecoin-project/oni/issues/195 + neededPrecursorMsgs := make([]*types.Message, 0) for _, b := range includedTs.Blocks() { messages, err := api.ChainGetBlockMessages(ctx, b.Cid()) if err != nil { @@ -130,7 +128,15 @@ func runExtractMsg(c *cli.Context) error { break } if other.From == msg.From && other.Nonce < msg.Nonce { - log.Printf("WARN: tipset includes preceding message with lower nonce from same sender; this extraction won't work") + included := false + for _, o := range neededPrecursorMsgs { + if o.Cid() == other.Cid() { + included = true + } + } + if !included { + neededPrecursorMsgs = append(neededPrecursorMsgs, other) + } } } for _, m := range messages.SecpkMessages { @@ -138,19 +144,36 @@ func runExtractMsg(c *cli.Context) error { break } if m.Message.From == msg.From && m.Message.Nonce < msg.Nonce { - log.Printf("WARN: tipset includes preceding message with lower nonce from same sender; this extraction won't work") + included := false + for _, o := range neededPrecursorMsgs { + if o.Cid() == m.Message.Cid() { + included = true + } + } + if !included { + neededPrecursorMsgs = append(neededPrecursorMsgs, &m.Message) + } } } } fmt.Println("getting the _before_ filtered state tree") - preroot, err := g.GetMaskedStateTree(includedTs.Parents(), retain) + tree, err := g.GetStateTreeRootFromTipset(includedTs.Parents()) + + driver := lotus.NewDriver(ctx) + + for _, pm := range neededPrecursorMsgs { + _, tree, err = driver.ExecuteMessage(pm, tree, pst.Blockstore, execTs.Height()) + if err != nil { + return fmt.Errorf("Failed to execute preceding message: %w", err) + } + } + + preroot, err := g.GetMaskedStateTree(tree, retain) if err != nil { return err } - driver := lotus.NewDriver(ctx) - _, postroot, err := driver.ExecuteMessage(msg, preroot, pst.Blockstore, execTs.Height()) if err != nil { return fmt.Errorf("failed to execute message: %w", err) diff --git a/tvx/go.sum b/tvx/go.sum index 1861d59e1..b80f8fcc5 100644 --- a/tvx/go.sum +++ b/tvx/go.sum @@ -1182,6 +1182,7 @@ github.com/prometheus/procfs v0.1.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/raulk/clock v1.1.0 h1:dpb29+UKMbLqiU/jqIJptgLR1nn23HLgMY0sTCDza5Y= github.com/raulk/clock v1.1.0/go.mod h1:3MpVxdZ/ODBQDxbN+kzshf5OSZwPjtMDx6BBXBmOeY0= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -1277,6 +1278,7 @@ github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFd github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g= +github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-client-go v2.23.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= @@ -1340,11 +1342,14 @@ github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1: github.com/whyrusleeping/yamux v1.1.5/go.mod h1:E8LnQQ8HKx5KD29HZFUwM1PxCOdPRzGwur1mcYhXcD8= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829/go.mod h1:h/1PEBwj7Ym/8kOuMWvO2ujZ6Lt+TMbySEXNhjjR87I= +github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245/go.mod h1:C+diUUz7pxhNY6KAoLgrTYARGWnt82zWTylZlxT92vk= github.com/xorcare/golden v0.6.0/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ= github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 h1:oWgZJmC1DorFZDpfMfWg7xk29yEOZiXmo/wZl+utTI8= github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw= go.dedis.ch/kyber/v3 v3.0.4/go.mod h1:OzvaEnPvKlyrWyp3kGXlFdp7ap1VC6RkZDTaPikqhsQ= go.dedis.ch/kyber/v3 v3.0.9/go.mod h1:rhNjUUg6ahf8HEg5HUvVBYoWY4boAafX8tYxX+PS+qg= @@ -1421,6 +1426,8 @@ golang.org/x/crypto v0.0.0-20200427165652-729f1e841bcc/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1454,6 +1461,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1496,6 +1504,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200519113804-d87ec0cfa476/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1633,6 +1642,7 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200318150045-ba25ddc85566/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4 h1:kDtqNkeBrZb8B+atrj50B5XLHpzXXqcCdZPP/ApQ5NY= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200711155855-7342f9734a7d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -1746,6 +1756,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= @@ -1763,6 +1774,11 @@ howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqp howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= +modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= +modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/tvx/state/surgeon.go b/tvx/state/surgeon.go index a549b29d4..59a90b820 100644 --- a/tvx/state/surgeon.go +++ b/tvx/state/surgeon.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/filecoin-project/lotus/chain/state" "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" @@ -39,15 +40,24 @@ func NewSurgeon(ctx context.Context, api api.FullNode, stores *Stores) *Surgeon } } +func (sg *Surgeon) GetStateTreeRootFromTipset(tsk types.TipSetKey) (cid.Cid, error) { + ts, err := sg.api.ChainGetTipSet(sg.ctx, tsk) + if err != nil { + return cid.Undef, err + } + + return ts.ParentState(), nil +} + // GetMaskedStateTree trims the state tree at the supplied tipset to contain // only the state of the actors in the retain set. It also "dives" into some // singleton system actors, like the init actor, to trim the state so as to // compute a minimal state tree. In the future, thid method will dive into // other system actors like the power actor and the market actor. -func (sg *Surgeon) GetMaskedStateTree(tsk types.TipSetKey, retain []address.Address) (cid.Cid, error) { +func (sg *Surgeon) GetMaskedStateTree(previousRoot cid.Cid, retain []address.Address) (cid.Cid, error) { stateMap := adt.MakeEmptyMap(sg.stores.ADTStore) - initState, err := sg.loadInitActor(tsk) + initState, err := sg.loadInitActor(previousRoot) if err != nil { return cid.Undef, err } @@ -67,7 +77,7 @@ func (sg *Surgeon) GetMaskedStateTree(tsk types.TipSetKey, retain []address.Addr return cid.Undef, err } - err = sg.pluckActorStates(tsk, resolved, stateMap) + err = sg.pluckActorStates(previousRoot, resolved, stateMap) if err != nil { return cid.Undef, err } @@ -89,12 +99,17 @@ func (sg *Surgeon) GetAccessedActors(ctx context.Context, a api.FullNode, mid ci return nil, err } + msgObj, err := a.ChainGetMessage(ctx, mid) + if err != nil { + return nil, err + } + ts, err := a.ChainGetTipSet(ctx, msgInfo.TipSet) if err != nil { return nil, err } - trace, err := a.StateReplay(ctx, ts.Parents(), mid) + trace, err := a.StateCall(ctx, msgObj, ts.Parents()) if err != nil { return nil, fmt.Errorf("could not replay msg: %w", err) } @@ -136,11 +151,17 @@ func (sg *Surgeon) WriteCAR(w io.Writer, roots ...cid.Cid) error { // pluckActorStates plucks the state from the supplied actors at the given // tipset, and places it into the supplied state map. -func (sg *Surgeon) pluckActorStates(tsk types.TipSetKey, pluck []address.Address, stateMap *adt.Map) error { +func (sg *Surgeon) pluckActorStates(stateRoot cid.Cid, pluck []address.Address, stateMap *adt.Map) error { + st, err := state.LoadStateTree(sg.stores.CBORStore, stateRoot) + if err != nil { + return err + } + for _, a := range pluck { - actor, err := sg.api.StateGetActor(sg.ctx, a, tsk) + actor, err := st.GetActor(a) if err != nil { - return err + continue + //return fmt.Errorf("get actor %s failed: %w", a, err) } err = stateMap.Put(adt.AddrKey(a), actor) @@ -208,17 +229,14 @@ func (sg *Surgeon) retainInitEntries(oldState *init_.State, retain []address.Add newAddrs := adt.MakeEmptyMap(sg.stores.ADTStore) for _, r := range retain { - if r.Protocol() == address.ID { - // skip over ID addresses; they don't need a mapping in the init actor. - continue - } - var d cbg.Deferred if _, err := oldAddrs.Get(adt.AddrKey(r), &d); err != nil { return nil, err } - if err := newAddrs.Put(adt.AddrKey(r), &d); err != nil { - return nil, err + if d.Raw != nil { + if err := newAddrs.Put(adt.AddrKey(r), &d); err != nil { + return nil, err + } } } @@ -261,10 +279,16 @@ func (sg *Surgeon) resolveAddresses(orig []address.Address, ist *init_.State) (r } // loadInitActor loads the init actor state from a given tipset. -func (sg *Surgeon) loadInitActor(tsk types.TipSetKey) (initState *init_.State, err error) { - log.Printf("loading the init actor for tipset: %s", tsk) +func (sg *Surgeon) loadInitActor(stateRoot cid.Cid) (initState *init_.State, err error) { + log.Printf("loading the init actor for root: %s", stateRoot) - actor, err := sg.api.StateGetActor(sg.ctx, builtin.InitActorAddr, tsk) + st, err := state.LoadStateTree(sg.stores.CBORStore, stateRoot) + if err != nil { + return initState, err + } + + //sg.api.StateGetActor(sg.ctx, builtin.InitActorAddr, tsk) + actor, err := st.GetActor(builtin.InitActorAddr) if err != nil { return initState, err }