dedup state tree CARs from test vectors. (#187)

corpus/schema.json

@@ -49,46 +49,49 @@
         }
       }
     },
+    "state_tree": {
+      "additionalProperties": false,
+      "required": [
+        "root_cid"
+      ],
+      "properties": {
+        "root_cid": {
+          "additionalProperties": false,
+          "required": [
+            "/"
+          ],
+          "properties": {
+            "/": {
+              "type": "string"
+            }
+          }
+        }
+      }
+    },
     "preconditions": {
       "title": "execution preconditions",
       "description": "preconditions that need to be applied and satisfied before this test vector can be executed",
-      "type": "object",
       "additionalProperties": false,
       "properties": {
+        "epoch": {
+          "type": "integer"
+        },
         "state_tree": {
           "title": "state tree to seed",
           "description": "state tree to seed before applying this test vector; mapping of actor addresses => serialized state",
-          "type": "object",
+          "$ref": "#/definitions/state_tree"
-          "additionalProperties": {
-            "$ref": "#/definitions/hex"
-          },
-          "examples": [
-            {
-              "t01": "0x0123456789abcdef",
-              "t02": "0x0123456789abcdef",
-              "t03": "0x0123456789abcdef"
-            }
-          ]
         }
       }
     },
     "postconditions": {
       "title": "execution preconditions",
       "description": "postconditions that need to be satisfied after execution for this test vector to pass",
-      "type": "object",
       "additionalProperties": false,
       "properties": {
         "state_tree": {
           "title": "state tree postconditions",
           "description": "state tree postconditions that must be true for this test vector to pass",
-          "type": "object",
+          "$ref": "#/definitions/state_tree"
-          "properties": {
-            "car_bytes": {
-              "title": "the hex-encoded CAR containing the full state tree",
-              "description": "the hex-encoded CAR containing the full state tree, for debugging/diffing purposes",
-              "$ref": "#/definitions/hex"
-            }
-          }
         }
       }
     },
@@ -120,6 +123,11 @@
     "_meta": {
       "$ref": "#/definitions/meta"
     },
+    "car_bytes": {
+      "title": "car containing state trees",
+      "description": "the gzipped, hex-encoded CAR containing the pre- and post-condition state trees for this test vector",
+      "$ref": "#/definitions/hex"
+    },
     "preconditions": {
       "$ref": "#/definitions/preconditions"
     },
@@ -137,7 +145,9 @@
         }
       },
       "then": {
-        "required": ["apply_message"],
+        "required": [
+          "apply_message"
+        ],
         "properties": {
           "apply_message": {
             "$ref": "#/definitions/apply_message"

corpus/single_message.json

@@ -11,11 +11,14 @@
     }
   },
 
+  "car_bytes": "0x890043009920583103b5b1fab6769fcb464146d192af785915d8e1dacff264b2be3bcb7bf4064a77cb3b13e27063ea4825dc48cd0a7df77e211916e748002740f0c493a2824200011a09d9c4dc0758c68219091c58c083e7c59e748e070b382a813f4ffe8ead2a090469aaa37f0664b82c092bc78b43fac300335cd2d6e4a8c71ce138003c0fa8ee1707b41aa87f35c811613d59ac53449aef8263aa51c876f2e67c185118ec9628f73442f58e093909ce952e431e770ac693cdbe99f2f9404ad3ca754196ab443c6c66afa41310e705e453496bdcedf0f271529dc7374021d7b80d343397f996d06bf11e753a49eaed7b3ce80a8e9bbaa48bf83223527bd8a2933f002949fdb8058103336a4a979375d5e963cdd7b1",
+
   "preconditions": {
+    "height": 100,
     "state_tree": {
-      "t01": "0x0123456789abcdef",
+      "root_cid": {
-      "t02": "0x0123456789abcdef",
+        "/": "bafy2bzacebbxsepazfgwepawspvzenb2x64pmqjyan3wgtfpxu5nxez33wzkc"
-      "t03": "0x0123456789abcdef"
+      }
     }
   },
 
@@ -23,7 +26,9 @@
 
   "postconditions": {
     "state_tree": {
-      "car_bytes": "0x890043009920583103b5b1fab6769fcb464146d192af785915d8e1dacff264b2be3bcb7bf4064a77cb3b13e27063ea4825dc48cd0a7df77e211916e748002740f0c493a2824200011a09d9c4dc0758c68219091c58c083e7c59e748e070b382a813f4ffe8ead2a090469aaa37f0664b82c092bc78b43fac300335cd2d6e4a8c71ce138003c0fa8ee1707b41aa87f35c811613d59ac53449aef8263aa51c876f2e67c185118ec9628f73442f58e093909ce952e431e770ac693cdbe99f2f9404ad3ca754196ab443c6c66afa41310e705e453496bdcedf0f271529dc7374021d7b80d343397f996d06bf11e753a49eaed7b3ce80a8e9bbaa48bf83223527bd8a2933f002949fdb8058103336a4a979375d5e963cdd7b1"
+      "root_cid": {
+        "/": "bafy2bzacebbxsepazfgwepawspvzenb2x64pmqjyan3wgtfpxu5nxez33wzkc"
+      }
     }
   }
 }

tvx/examine.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"os"
 
+	"github.com/ipfs/go-cid"
 	"github.com/urfave/cli/v2"
 
 	"github.com/filecoin-project/specs-actors/actors/builtin"
@@ -66,8 +67,9 @@ func runExamineCmd(_ *cli.Context) error {
 		return err
 	}
 
-	examine := func(encoded []byte) error {
+	examine := func(root cid.Cid) error {
-		tree, err := state.RecoverStateTree(context.TODO(), encoded)
+		encoded := tv.CAR
+		tree, err := state.RecoverStateTree(context.TODO(), encoded, root)
 		if err != nil {
 			return err
 		}
@@ -96,14 +98,14 @@ func runExamineCmd(_ *cli.Context) error {
 
 	if examineFlags.pre {
 		log.Print("examining precondition tree")
-		if err := examine(tv.Pre.StateTree.CAR); err != nil {
+		if err := examine(tv.Pre.StateTree.RootCID); err != nil {
 			return err
 		}
 	}
 
 	if examineFlags.post {
 		log.Print("examining postcondition tree")
-		if err := examine(tv.Post.StateTree.CAR); err != nil {
+		if err := examine(tv.Post.StateTree.RootCID); err != nil {
 			return err
 		}
 	}

tvx/exec_lotus.go

@@ -57,13 +57,14 @@ func runExecLotus(_ *cli.Context) error {
 	switch tv.Class {
 	case "message":
 		var (
-			ctx   = context.Background()
+			ctx     = context.Background()
-			epoch = tv.Pre.Epoch
+			epoch   = tv.Pre.Epoch
+			preroot = tv.Pre.StateTree.RootCID
 		)
 
 		bs := blockstore.NewTemporary()
 
-		buf := bytes.NewReader(tv.Pre.StateTree.CAR)
+		buf := bytes.NewReader(tv.CAR)
 		gr, err := gzip.NewReader(buf)
 		if err != nil {
 			return err
@@ -86,7 +87,7 @@ func runExecLotus(_ *cli.Context) error {
 		driver := lotus.NewDriver(ctx)
 
 		fmt.Println("executing message")
-		spew.Dump(driver.ExecuteMessage(msg, header.Roots[0], bs, epoch))
+		spew.Dump(driver.ExecuteMessage(msg, preroot, bs, epoch))
 
 		return nil
 

tvx/extract_msg.go

@@ -131,29 +131,15 @@ func runExtractMsg(c *cli.Context) error {
 		return err
 	}
 
-	getZippedCAR := func(root cid.Cid) ([]byte, error) {
+	out := new(bytes.Buffer)
-		out := new(bytes.Buffer)
+	gw := gzip.NewWriter(out)
-		gw := gzip.NewWriter(out)
+	if err := g.WriteCAR(gw, preroot, postroot); err != nil {
-		if err := g.WriteCAR(gw, root); err != nil {
-			return nil, err
-		}
-		if err = gw.Flush(); err != nil {
-			return nil, err
-		}
-		if err = gw.Close(); err != nil {
-			return nil, err
-		}
-
-		return out.Bytes(), nil
-	}
-
-	pretree, err := getZippedCAR(preroot)
-	if err != nil {
 		return err
 	}
-
+	if err = gw.Flush(); err != nil {
-	posttree, err := getZippedCAR(postroot)
+		return err
-	if err != nil {
+	}
+	if err = gw.Close(); err != nil {
 		return err
 	}
 
@@ -174,16 +160,17 @@ func runExtractMsg(c *cli.Context) error {
 				Version: version.String(),
 			},
 		},
+		CAR: out.Bytes(),
 		Pre: &Preconditions{
 			Epoch: ts.Height(),
 			StateTree: &StateTree{
-				CAR: pretree,
+				RootCID: preroot,
 			},
 		},
 		ApplyMessage: msgBytes,
 		Post: &Postconditions{
 			StateTree: &StateTree{
-				CAR: posttree,
+				RootCID: postroot,
 			},
 		},
 	}

tvx/schema.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 
 	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/ipfs/go-cid"
 )
 
 // Class represents the type of test this instance is.
@@ -39,8 +40,7 @@ type GenerationData struct {
 
 // StateTree represents a state tree within preconditions and postconditions.
 type StateTree struct {
-	// CAR is the car representation of a state tree
+	RootCID cid.Cid `json:"root_cid"`
-	CAR HexEncodedBytes `json:"car_hex"`
 }
 
 // HexEncodedBytes is a hex-encoded binary value.
@@ -81,9 +81,15 @@ func (heb *HexEncodedBytes) UnmarshalJSON(v []byte) error {
 
 // TestVector is a single test case
 type TestVector struct {
-	Class        `json:"class"`
+	Class    `json:"class"`
-	Selector     `json:"selector"`
+	Selector `json:"selector"`
-	Meta         *Metadata       `json:"_meta"`
+	Meta     *Metadata `json:"_meta"`
+
+	// CAR binary data to be loaded into the test environment, usually a CAR
+	// containing multiple state trees, addressed by root CID from the relevant
+	// objects.
+	CAR HexEncodedBytes `json:"car_hex"`
+
 	Pre          *Preconditions  `json:"preconditions"`
 	ApplyMessage HexEncodedBytes `json:"apply_message"`
 	Post         *Postconditions `json:"postconditions"`

tvx/state/serialize.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/filecoin-project/lotus/chain/state"
 	bs "github.com/filecoin-project/lotus/lib/blockstore"
+	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-hamt-ipld"
 	cbor "github.com/ipfs/go-ipld-cbor"
 	"github.com/ipfs/go-ipld-format"
@@ -15,7 +16,7 @@ import (
 )
 
 // RecoverStateTree parses a car encoding of a state tree back to a structured format
-func RecoverStateTree(ctx context.Context, raw []byte) (*state.StateTree, error) {
+func RecoverStateTree(ctx context.Context, raw []byte, root cid.Cid) (*state.StateTree, error) {
 	buf := bytes.NewBuffer(raw)
 	store := bs.NewTemporary()
 	gr, err := gzip.NewReader(buf)
@@ -28,14 +29,12 @@ func RecoverStateTree(ctx context.Context, raw []byte) (*state.StateTree, error)
 	if err != nil {
 		return nil, err
 	}
-	if len(ch.Roots) != 1 {
+
-		return nil, fmt.Errorf("car should have 1 root, has %d", len(ch.Roots))
-	}
 	cborstore := cbor.NewCborStore(store)
 
-	fmt.Printf("root is %s\n", ch.Roots[0])
+	fmt.Printf("roots are %v\n", ch.Roots)
 
-	nd, err := hamt.LoadNode(ctx, cborstore, ch.Roots[0], hamt.UseTreeBitWidth(5))
+	nd, err := hamt.LoadNode(ctx, cborstore, root, hamt.UseTreeBitWidth(5))
 	if err != nil {
 		return nil, err
 	}
@@ -51,5 +50,5 @@ func RecoverStateTree(ctx context.Context, raw []byte) (*state.StateTree, error)
 		return nil, err
 	}
 
-	return state.LoadStateTree(cborstore, ch.Roots[0])
+	return state.LoadStateTree(cborstore, root)
 }

tvx/state/surgeon.go

@@ -121,7 +121,7 @@ func (sg *Surgeon) GetAccessedActors(ctx context.Context, a api.FullNode, mid ci
 
 // WriteCAR recursively writes the tree referenced by the root as a CAR into the
 // supplied io.Writer.
-func (sg *Surgeon) WriteCAR(w io.Writer, root cid.Cid) error {
+func (sg *Surgeon) WriteCAR(w io.Writer, roots ...cid.Cid) error {
 	carWalkFn := func(nd format.Node) (out []*format.Link, err error) {
 		for _, link := range nd.Links() {
 			if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
@@ -131,7 +131,7 @@ func (sg *Surgeon) WriteCAR(w io.Writer, root cid.Cid) error {
 		}
 		return out, nil
 	}
-	return car.WriteCarWithWalker(sg.ctx, sg.stores.DAGService, []cid.Cid{root}, w, carWalkFn)
+	return car.WriteCarWithWalker(sg.ctx, sg.stores.DAGService, roots, w, carWalkFn)
 }
 
 // pluckActorStates plucks the state from the supplied actors at the given