From 163721651a0eb84a67d129bbe03a1dfd9c6fc35d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 5 Aug 2020 15:52:57 +0100 Subject: [PATCH] dedup state tree CARs from test vectors. (#187) --- corpus/schema.json | 54 ++++++++++++++++++++++---------------- corpus/single_message.json | 13 ++++++--- tvx/examine.go | 10 ++++--- tvx/exec_lotus.go | 9 ++++--- tvx/extract_msg.go | 33 +++++++---------------- tvx/schema.go | 16 +++++++---- tvx/state/serialize.go | 13 +++++---- tvx/state/surgeon.go | 4 +-- 8 files changed, 81 insertions(+), 71 deletions(-) diff --git a/corpus/schema.json b/corpus/schema.json index 8b54068dd..c59ce580d 100644 --- a/corpus/schema.json +++ b/corpus/schema.json @@ -49,46 +49,49 @@ } } }, + "state_tree": { + "additionalProperties": false, + "required": [ + "root_cid" + ], + "properties": { + "root_cid": { + "additionalProperties": false, + "required": [ + "/" + ], + "properties": { + "/": { + "type": "string" + } + } + } + } + }, "preconditions": { "title": "execution preconditions", "description": "preconditions that need to be applied and satisfied before this test vector can be executed", - "type": "object", "additionalProperties": false, "properties": { + "epoch": { + "type": "integer" + }, "state_tree": { "title": "state tree to seed", "description": "state tree to seed before applying this test vector; mapping of actor addresses => serialized state", - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/hex" - }, - "examples": [ - { - "t01": "0x0123456789abcdef", - "t02": "0x0123456789abcdef", - "t03": "0x0123456789abcdef" - } - ] + "$ref": "#/definitions/state_tree" } } }, "postconditions": { "title": "execution preconditions", "description": "postconditions that need to be satisfied after execution for this test vector to pass", - "type": "object", "additionalProperties": false, "properties": { "state_tree": { "title": "state tree postconditions", "description": "state tree postconditions that must be true for this test vector to pass", - "type": "object", - "properties": { - "car_bytes": { - "title": "the hex-encoded CAR containing the full state tree", - "description": "the hex-encoded CAR containing the full state tree, for debugging/diffing purposes", - "$ref": "#/definitions/hex" - } - } + "$ref": "#/definitions/state_tree" } } }, @@ -120,6 +123,11 @@ "_meta": { "$ref": "#/definitions/meta" }, + "car_bytes": { + "title": "car containing state trees", + "description": "the gzipped, hex-encoded CAR containing the pre- and post-condition state trees for this test vector", + "$ref": "#/definitions/hex" + }, "preconditions": { "$ref": "#/definitions/preconditions" }, @@ -137,7 +145,9 @@ } }, "then": { - "required": ["apply_message"], + "required": [ + "apply_message" + ], "properties": { "apply_message": { "$ref": "#/definitions/apply_message" diff --git a/corpus/single_message.json b/corpus/single_message.json index 69eeba821..892770167 100644 --- a/corpus/single_message.json +++ b/corpus/single_message.json @@ -11,11 +11,14 @@ } }, + "car_bytes": "0x890043009920583103b5b1fab6769fcb464146d192af785915d8e1dacff264b2be3bcb7bf4064a77cb3b13e27063ea4825dc48cd0a7df77e211916e748002740f0c493a2824200011a09d9c4dc0758c68219091c58c083e7c59e748e070b382a813f4ffe8ead2a090469aaa37f0664b82c092bc78b43fac300335cd2d6e4a8c71ce138003c0fa8ee1707b41aa87f35c811613d59ac53449aef8263aa51c876f2e67c185118ec9628f73442f58e093909ce952e431e770ac693cdbe99f2f9404ad3ca754196ab443c6c66afa41310e705e453496bdcedf0f271529dc7374021d7b80d343397f996d06bf11e753a49eaed7b3ce80a8e9bbaa48bf83223527bd8a2933f002949fdb8058103336a4a979375d5e963cdd7b1", + "preconditions": { + "height": 100, "state_tree": { - "t01": "0x0123456789abcdef", - "t02": "0x0123456789abcdef", - "t03": "0x0123456789abcdef" + "root_cid": { + "/": "bafy2bzacebbxsepazfgwepawspvzenb2x64pmqjyan3wgtfpxu5nxez33wzkc" + } } }, @@ -23,7 +26,9 @@ "postconditions": { "state_tree": { - "car_bytes": "0x890043009920583103b5b1fab6769fcb464146d192af785915d8e1dacff264b2be3bcb7bf4064a77cb3b13e27063ea4825dc48cd0a7df77e211916e748002740f0c493a2824200011a09d9c4dc0758c68219091c58c083e7c59e748e070b382a813f4ffe8ead2a090469aaa37f0664b82c092bc78b43fac300335cd2d6e4a8c71ce138003c0fa8ee1707b41aa87f35c811613d59ac53449aef8263aa51c876f2e67c185118ec9628f73442f58e093909ce952e431e770ac693cdbe99f2f9404ad3ca754196ab443c6c66afa41310e705e453496bdcedf0f271529dc7374021d7b80d343397f996d06bf11e753a49eaed7b3ce80a8e9bbaa48bf83223527bd8a2933f002949fdb8058103336a4a979375d5e963cdd7b1" + "root_cid": { + "/": "bafy2bzacebbxsepazfgwepawspvzenb2x64pmqjyan3wgtfpxu5nxez33wzkc" + } } } } \ No newline at end of file diff --git a/tvx/examine.go b/tvx/examine.go index 41c26b18c..40b3455ee 100644 --- a/tvx/examine.go +++ b/tvx/examine.go @@ -7,6 +7,7 @@ import ( "log" "os" + "github.com/ipfs/go-cid" "github.com/urfave/cli/v2" "github.com/filecoin-project/specs-actors/actors/builtin" @@ -66,8 +67,9 @@ func runExamineCmd(_ *cli.Context) error { return err } - examine := func(encoded []byte) error { - tree, err := state.RecoverStateTree(context.TODO(), encoded) + examine := func(root cid.Cid) error { + encoded := tv.CAR + tree, err := state.RecoverStateTree(context.TODO(), encoded, root) if err != nil { return err } @@ -96,14 +98,14 @@ func runExamineCmd(_ *cli.Context) error { if examineFlags.pre { log.Print("examining precondition tree") - if err := examine(tv.Pre.StateTree.CAR); err != nil { + if err := examine(tv.Pre.StateTree.RootCID); err != nil { return err } } if examineFlags.post { log.Print("examining postcondition tree") - if err := examine(tv.Post.StateTree.CAR); err != nil { + if err := examine(tv.Post.StateTree.RootCID); err != nil { return err } } diff --git a/tvx/exec_lotus.go b/tvx/exec_lotus.go index f8582fa7e..dca27dc74 100644 --- a/tvx/exec_lotus.go +++ b/tvx/exec_lotus.go @@ -57,13 +57,14 @@ func runExecLotus(_ *cli.Context) error { switch tv.Class { case "message": var ( - ctx = context.Background() - epoch = tv.Pre.Epoch + ctx = context.Background() + epoch = tv.Pre.Epoch + preroot = tv.Pre.StateTree.RootCID ) bs := blockstore.NewTemporary() - buf := bytes.NewReader(tv.Pre.StateTree.CAR) + buf := bytes.NewReader(tv.CAR) gr, err := gzip.NewReader(buf) if err != nil { return err @@ -86,7 +87,7 @@ func runExecLotus(_ *cli.Context) error { driver := lotus.NewDriver(ctx) fmt.Println("executing message") - spew.Dump(driver.ExecuteMessage(msg, header.Roots[0], bs, epoch)) + spew.Dump(driver.ExecuteMessage(msg, preroot, bs, epoch)) return nil diff --git a/tvx/extract_msg.go b/tvx/extract_msg.go index 81423307a..d6864b7ce 100644 --- a/tvx/extract_msg.go +++ b/tvx/extract_msg.go @@ -131,29 +131,15 @@ func runExtractMsg(c *cli.Context) error { return err } - getZippedCAR := func(root cid.Cid) ([]byte, error) { - out := new(bytes.Buffer) - gw := gzip.NewWriter(out) - if err := g.WriteCAR(gw, root); err != nil { - return nil, err - } - if err = gw.Flush(); err != nil { - return nil, err - } - if err = gw.Close(); err != nil { - return nil, err - } - - return out.Bytes(), nil - } - - pretree, err := getZippedCAR(preroot) - if err != nil { + out := new(bytes.Buffer) + gw := gzip.NewWriter(out) + if err := g.WriteCAR(gw, preroot, postroot); err != nil { return err } - - posttree, err := getZippedCAR(postroot) - if err != nil { + if err = gw.Flush(); err != nil { + return err + } + if err = gw.Close(); err != nil { return err } @@ -174,16 +160,17 @@ func runExtractMsg(c *cli.Context) error { Version: version.String(), }, }, + CAR: out.Bytes(), Pre: &Preconditions{ Epoch: ts.Height(), StateTree: &StateTree{ - CAR: pretree, + RootCID: preroot, }, }, ApplyMessage: msgBytes, Post: &Postconditions{ StateTree: &StateTree{ - CAR: posttree, + RootCID: postroot, }, }, } diff --git a/tvx/schema.go b/tvx/schema.go index 798433e84..b3710d22e 100644 --- a/tvx/schema.go +++ b/tvx/schema.go @@ -5,6 +5,7 @@ import ( "encoding/json" "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/ipfs/go-cid" ) // Class represents the type of test this instance is. @@ -39,8 +40,7 @@ type GenerationData struct { // StateTree represents a state tree within preconditions and postconditions. type StateTree struct { - // CAR is the car representation of a state tree - CAR HexEncodedBytes `json:"car_hex"` + RootCID cid.Cid `json:"root_cid"` } // HexEncodedBytes is a hex-encoded binary value. @@ -81,9 +81,15 @@ func (heb *HexEncodedBytes) UnmarshalJSON(v []byte) error { // TestVector is a single test case type TestVector struct { - Class `json:"class"` - Selector `json:"selector"` - Meta *Metadata `json:"_meta"` + Class `json:"class"` + Selector `json:"selector"` + Meta *Metadata `json:"_meta"` + + // CAR binary data to be loaded into the test environment, usually a CAR + // containing multiple state trees, addressed by root CID from the relevant + // objects. + CAR HexEncodedBytes `json:"car_hex"` + Pre *Preconditions `json:"preconditions"` ApplyMessage HexEncodedBytes `json:"apply_message"` Post *Postconditions `json:"postconditions"` diff --git a/tvx/state/serialize.go b/tvx/state/serialize.go index af1bc0c9c..f1244124f 100644 --- a/tvx/state/serialize.go +++ b/tvx/state/serialize.go @@ -8,6 +8,7 @@ import ( "github.com/filecoin-project/lotus/chain/state" bs "github.com/filecoin-project/lotus/lib/blockstore" + "github.com/ipfs/go-cid" "github.com/ipfs/go-hamt-ipld" cbor "github.com/ipfs/go-ipld-cbor" "github.com/ipfs/go-ipld-format" @@ -15,7 +16,7 @@ import ( ) // RecoverStateTree parses a car encoding of a state tree back to a structured format -func RecoverStateTree(ctx context.Context, raw []byte) (*state.StateTree, error) { +func RecoverStateTree(ctx context.Context, raw []byte, root cid.Cid) (*state.StateTree, error) { buf := bytes.NewBuffer(raw) store := bs.NewTemporary() gr, err := gzip.NewReader(buf) @@ -28,14 +29,12 @@ func RecoverStateTree(ctx context.Context, raw []byte) (*state.StateTree, error) if err != nil { return nil, err } - if len(ch.Roots) != 1 { - return nil, fmt.Errorf("car should have 1 root, has %d", len(ch.Roots)) - } + cborstore := cbor.NewCborStore(store) - fmt.Printf("root is %s\n", ch.Roots[0]) + fmt.Printf("roots are %v\n", ch.Roots) - nd, err := hamt.LoadNode(ctx, cborstore, ch.Roots[0], hamt.UseTreeBitWidth(5)) + nd, err := hamt.LoadNode(ctx, cborstore, root, hamt.UseTreeBitWidth(5)) if err != nil { return nil, err } @@ -51,5 +50,5 @@ func RecoverStateTree(ctx context.Context, raw []byte) (*state.StateTree, error) return nil, err } - return state.LoadStateTree(cborstore, ch.Roots[0]) + return state.LoadStateTree(cborstore, root) } diff --git a/tvx/state/surgeon.go b/tvx/state/surgeon.go index 7d0723b11..24ced62a0 100644 --- a/tvx/state/surgeon.go +++ b/tvx/state/surgeon.go @@ -121,7 +121,7 @@ func (sg *Surgeon) GetAccessedActors(ctx context.Context, a api.FullNode, mid ci // WriteCAR recursively writes the tree referenced by the root as a CAR into the // supplied io.Writer. -func (sg *Surgeon) WriteCAR(w io.Writer, root cid.Cid) error { +func (sg *Surgeon) WriteCAR(w io.Writer, roots ...cid.Cid) error { carWalkFn := func(nd format.Node) (out []*format.Link, err error) { for _, link := range nd.Links() { if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed { @@ -131,7 +131,7 @@ func (sg *Surgeon) WriteCAR(w io.Writer, root cid.Cid) error { } return out, nil } - return car.WriteCarWithWalker(sg.ctx, sg.stores.DAGService, []cid.Cid{root}, w, carWalkFn) + return car.WriteCarWithWalker(sg.ctx, sg.stores.DAGService, roots, w, carWalkFn) } // pluckActorStates plucks the state from the supplied actors at the given