feat: shed: Add v13 migration to migrate-state (#11601)

* shed: Add v13 migration to migrate-state

* shed: some ADL tools, update GST

* shed: diff hamt address mode

* shed migration debug tooling

* shed migration: market diff on error

* shed: Fix cached migration diff

* shed: Diff deal states on bad migration

* shed: Use std json

* shed: Drill in the migration diff more

* shed: Show proposals in migration market diff

* shed: Show added provider sectors diff

* shed: hamts are hard to use

* update  gst

* update gst

* update gst

* update GST with fixed invartiant

* go mod tidy

cmd/lotus-shed/adl.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/ipfs/go-cid"
+	cbor "github.com/ipfs/go-ipld-cbor"
+	"github.com/ipld/go-car"
+	"github.com/urfave/cli/v2"
+	cbg "github.com/whyrusleeping/cbor-gen"
+	"golang.org/x/xerrors"
+
+	adt13 "github.com/filecoin-project/go-state-types/builtin/v13/util/adt"
+
+	"github.com/filecoin-project/lotus/blockstore"
+	"github.com/filecoin-project/lotus/chain/actors/adt"
+)
+
+var adlCmd = &cli.Command{
+	Name:  "adl",
+	Usage: "adl manipulation commands",
+	Subcommands: []*cli.Command{
+		adlAmtCmd,
+	},
+}
+
+var adlAmtCmd = &cli.Command{
+	Name:  "amt",
+	Usage: "AMT manipulation commands",
+	Subcommands: []*cli.Command{
+		adlAmtGetCmd,
+	},
+}
+
+var adlAmtGetCmd = &cli.Command{
+	Name:  "get",
+	Usage: "Get an element from an AMT",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "car-file",
+			Usage: "write a car file with two hamts (use lotus-shed export-car)",
+		},
+		&cli.IntFlag{
+			Name:  "bitwidth",
+			Usage: "bitwidth of the HAMT",
+			Value: 5,
+		},
+		&cli.StringFlag{
+			Name:  "root",
+			Usage: "root cid of the HAMT",
+		},
+		&cli.Int64Flag{
+			Name:  "key",
+			Usage: "key to get",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		bs := blockstore.NewMemorySync()
+
+		f, err := os.Open(cctx.String("car-file"))
+		if err != nil {
+			return err
+		}
+		defer func(f *os.File) {
+			_ = f.Close()
+		}(f)
+
+		cr, err := car.NewCarReader(f)
+		if err != nil {
+			return err
+		}
+
+		for {
+			blk, err := cr.Next()
+			if err != nil {
+				if err == io.EOF {
+					break
+				}
+				return err
+			}
+
+			if err := bs.Put(cctx.Context, blk); err != nil {
+				return err
+			}
+		}
+
+		root, err := cid.Parse(cctx.String("root"))
+		if err != nil {
+			return err
+		}
+
+		m, err := adt13.AsArray(adt.WrapStore(cctx.Context, cbor.NewCborStore(bs)), root, cctx.Int("bitwidth"))
+		if err != nil {
+			return err
+		}
+
+		var out cbg.Deferred
+		ok, err := m.Get(cctx.Uint64("key"), &out)
+		if err != nil {
+			return err
+		}
+		if !ok {
+			return xerrors.Errorf("no such element")
+		}
+
+		fmt.Printf("RAW: %x\n", out.Raw)
+		fmt.Println("----")
+
+		var i interface{}
+		if err := cbor.DecodeInto(out.Raw, &i); err == nil {
+			ij, err := json.MarshalIndent(i, "", "  ")
+			if err != nil {
+				return err
+			}
+
+			fmt.Println(string(ij))
+		}
+
+		return nil
+	},
+}

cmd/lotus-shed/diff.go

@@ -1,20 +1,31 @@
 package main
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
+	"os"
 
+	"github.com/fatih/color"
 	"github.com/ipfs/go-cid"
+	cbor "github.com/ipfs/go-ipld-cbor"
+	"github.com/ipld/go-car"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-amt-ipld/v4"
+	"github.com/filecoin-project/go-hamt-ipld/v3"
 	"github.com/filecoin-project/go-state-types/abi"
 	miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner"
 
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/lib/must"
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
@@ -24,6 +35,8 @@ var diffCmd = &cli.Command{
 	Subcommands: []*cli.Command{
 		diffStateTrees,
 		diffMinerStates,
+		diffHAMTs,
+		diffAMTs,
 	},
 }
 
@@ -64,7 +77,9 @@ var diffMinerStates = &cli.Command{
 			return err
 		}
 
-		defer lkrepo.Close() //nolint:errcheck
+		defer func(lkrepo repo.LockedRepo) {
+			_ = lkrepo.Close()
+		}(lkrepo)
 
 		bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore)
 		if err != nil {
@@ -258,3 +273,247 @@ var diffStateTrees = &cli.Command{
 		return nil
 	},
 }
+
+var diffHAMTs = &cli.Command{
+	Name:      "hamts",
+	Usage:     "diff two HAMTs",
+	ArgsUsage: "<hamt-a> <hamt-b>",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "car-file",
+			Usage: "write a car file with two hamts (use lotus-shed export-car)",
+		},
+		&cli.IntFlag{
+			Name:  "bitwidth",
+			Usage: "bitwidth of the HAMT",
+			Value: 5,
+		},
+		&cli.StringFlag{
+			Name:  "key-type",
+			Usage: "type of the key",
+			Value: "uint",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		var bs blockstore.Blockstore = blockstore.NewMemorySync()
+
+		if cctx.IsSet("car-file") {
+			f, err := os.Open(cctx.String("car-file"))
+			if err != nil {
+				return err
+			}
+			defer func(f *os.File) {
+				_ = f.Close()
+			}(f)
+
+			cr, err := car.NewCarReader(f)
+			if err != nil {
+				return err
+			}
+
+			for {
+				blk, err := cr.Next()
+				if err != nil {
+					if err == io.EOF {
+						break
+					}
+					return err
+				}
+
+				if err := bs.Put(cctx.Context, blk); err != nil {
+					return err
+				}
+			}
+		} else {
+			// use running node
+			api, closer, err := lcli.GetFullNodeAPI(cctx)
+			if err != nil {
+				return xerrors.Errorf("connect to full node: %w", err)
+			}
+			defer closer()
+
+			bs = blockstore.NewAPIBlockstore(api)
+		}
+
+		cidA, err := cid.Parse(cctx.Args().Get(0))
+		if err != nil {
+			return err
+		}
+
+		cidB, err := cid.Parse(cctx.Args().Get(1))
+		if err != nil {
+			return err
+		}
+
+		cst := cbor.NewCborStore(bs)
+
+		var keyParser func(k string) (interface{}, error)
+		switch cctx.String("key-type") {
+		case "uint":
+			keyParser = func(k string) (interface{}, error) {
+				return abi.ParseUIntKey(k)
+			}
+		case "actor":
+			keyParser = func(k string) (interface{}, error) {
+				return address.NewFromBytes([]byte(k))
+			}
+		default:
+			return fmt.Errorf("unknown key type: %s", cctx.String("key-type"))
+		}
+
+		diffs, err := hamt.Diff(cctx.Context, cst, cst, cidA, cidB, hamt.UseTreeBitWidth(cctx.Int("bitwidth")))
+		if err != nil {
+			return err
+		}
+
+		for _, d := range diffs {
+			switch d.Type {
+			case hamt.Add:
+				color.Green("+ Add %v", must.One(keyParser(d.Key)))
+			case hamt.Remove:
+				color.Red("- Remove %v", must.One(keyParser(d.Key)))
+			case hamt.Modify:
+				color.Yellow("~ Modify %v", must.One(keyParser(d.Key)))
+			}
+		}
+
+		return nil
+	},
+}
+
+var diffAMTs = &cli.Command{
+	Name:      "amts",
+	Usage:     "diff two AMTs",
+	ArgsUsage: "<amt-a> <amt-b>",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "car-file",
+			Usage: "write a car file with two amts (use lotus-shed export-car)",
+		},
+		&cli.UintFlag{
+			Name:  "bitwidth",
+			Usage: "bitwidth of the AMT",
+			Value: 5,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		var bs blockstore.Blockstore = blockstore.NewMemorySync()
+
+		if cctx.IsSet("car-file") {
+			f, err := os.Open(cctx.String("car-file"))
+			if err != nil {
+				return err
+			}
+			defer func(f *os.File) {
+				_ = f.Close()
+			}(f)
+
+			cr, err := car.NewCarReader(f)
+			if err != nil {
+				return err
+			}
+
+			for {
+				blk, err := cr.Next()
+				if err != nil {
+					if err == io.EOF {
+						break
+					}
+					return err
+				}
+
+				if err := bs.Put(cctx.Context, blk); err != nil {
+					return err
+				}
+			}
+		} else {
+			// use running node
+			api, closer, err := lcli.GetFullNodeAPI(cctx)
+			if err != nil {
+				return xerrors.Errorf("connect to full node: %w", err)
+			}
+			defer closer()
+
+			bs = blockstore.NewAPIBlockstore(api)
+		}
+
+		cidA, err := cid.Parse(cctx.Args().Get(0))
+		if err != nil {
+			return err
+		}
+
+		cidB, err := cid.Parse(cctx.Args().Get(1))
+		if err != nil {
+			return err
+		}
+
+		cst := cbor.NewCborStore(bs)
+
+		diffs, err := amt.Diff(cctx.Context, cst, cst, cidA, cidB, amt.UseTreeBitWidth(cctx.Uint("bitwidth")))
+		if err != nil {
+			return err
+		}
+
+		for _, d := range diffs {
+			switch d.Type {
+			case amt.Add:
+				color.Green("+ Add %v", d.Key)
+			case amt.Remove:
+				color.Red("- Remove %v", d.Key)
+			case amt.Modify:
+				color.Yellow("~ Modify %v", d.Key)
+
+				var vb, va interface{}
+				err := cbor.DecodeInto(d.Before.Raw, &vb)
+				if err != nil {
+					return err
+				}
+				err = cbor.DecodeInto(d.After.Raw, &va)
+				if err != nil {
+					return err
+				}
+
+				vjsonb, err := json.MarshalIndent(vb, " ", "  ")
+				if err != nil {
+					return err
+				}
+				vjsona, err := json.MarshalIndent(va, " ", "  ")
+				if err != nil {
+					return err
+				}
+
+				linesb := bytes.Split(vjsonb, []byte("\n")) // -
+				linesa := bytes.Split(vjsona, []byte("\n")) // +
+
+				maxLen := len(linesb)
+				if len(linesa) > maxLen {
+					maxLen = len(linesa)
+				}
+
+				for i := 0; i < maxLen; i++ {
+					// Check if 'linesb' has run out of lines but 'linesa' hasn't
+					if i >= len(linesb) && i < len(linesa) {
+						color.Green("+ %s\n", linesa[i])
+						continue
+					}
+					// Check if 'linesa' has run out of lines but 'linesb' hasn't
+					if i >= len(linesa) && i < len(linesb) {
+						color.Red("- %s\n", linesb[i])
+						continue
+					}
+					// Compare lines if both slices have lines at index i
+					if !bytes.Equal(linesb[i], linesa[i]) {
+						color.Red("- %s\n", linesb[i])
+						color.Green("+ %s\n", linesa[i])
+					} else {
+						// Print the line if it is the same in both slices
+						fmt.Printf("  %s\n", linesb[i])
+					}
+				}
+
+			}
+		}
+
+		return nil
+	},
+}

cmd/lotus-shed/main.go

@@ -91,6 +91,7 @@ func main() {
 		FevmAnalyticsCmd,
 		mismatchesCmd,
 		blockCmd,
+		adlCmd,
 	}
 
 	app := &cli.App{

cmd/lotus-shed/migrations.go

@@ -1,27 +1,41 @@
 package main
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
 	"strconv"
 	"time"
 
+	"github.com/fatih/color"
+	"github.com/ipfs/boxo/blockservice"
+	"github.com/ipfs/boxo/exchange/offline"
+	"github.com/ipfs/boxo/ipld/merkledag"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
+	cbornode "github.com/ipfs/go-ipld-cbor"
+	"github.com/ipld/go-car"
 	"github.com/urfave/cli/v2"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"golang.org/x/xerrors"
 
 	ffi "github.com/filecoin-project/filecoin-ffi"
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-amt-ipld/v4"
+	"github.com/filecoin-project/go-hamt-ipld/v3"
 	"github.com/filecoin-project/go-state-types/abi"
 	actorstypes "github.com/filecoin-project/go-state-types/actors"
+	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/builtin"
 	v10 "github.com/filecoin-project/go-state-types/builtin/v10"
 	v11 "github.com/filecoin-project/go-state-types/builtin/v11"
 	v12 "github.com/filecoin-project/go-state-types/builtin/v12"
+	v13 "github.com/filecoin-project/go-state-types/builtin/v13"
+	market13 "github.com/filecoin-project/go-state-types/builtin/v13/market"
+	adt13 "github.com/filecoin-project/go-state-types/builtin/v13/util/adt"
 	market8 "github.com/filecoin-project/go-state-types/builtin/v8/market"
 	adt8 "github.com/filecoin-project/go-state-types/builtin/v8/util/adt"
 	v9 "github.com/filecoin-project/go-state-types/builtin/v9"
@@ -53,6 +67,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
 	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/lib/must"
 	"github.com/filecoin-project/lotus/node/repo"
 	"github.com/filecoin-project/lotus/storage/sealer/ffiwrapper"
 )
@@ -72,6 +87,9 @@ var migrationsCmd = &cli.Command{
 		&cli.BoolFlag{
 			Name: "check-invariants",
 		},
+		&cli.StringFlag{
+			Name: "export-bad-migration",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		fmt.Println("REMINDER: If you are running this, you likely want to ALSO run the continuity testing tool!")
@@ -215,6 +233,31 @@ var migrationsCmd = &cli.Command{
 			cachedMigrationTime := time.Since(startTime)
 
 			if newCid1 != newCid2 {
+				{
+					if err := printStateDiff(ctx, network.Version(nv), newCid2, newCid1, bs); err != nil {
+						fmt.Println("failed to print state diff: ", err)
+					}
+				}
+
+				if cctx.IsSet("export-bad-migration") {
+					fi, err := os.Create(cctx.String("export-bad-migration"))
+					if err != nil {
+						return xerrors.Errorf("opening the output file: %w", err)
+					}
+
+					defer fi.Close() //nolint:errcheck
+
+					roots := []cid.Cid{newCid1, newCid2}
+
+					dag := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs)))
+					err = car.WriteCarWithWalker(ctx, dag, roots, fi, carWalkFunc)
+					if err != nil {
+						return err
+					}
+
+					fmt.Println("exported bad migration to ", cctx.String("export-bad-migration"))
+				}
+
 				return xerrors.Errorf("got different results with and without the cache: %s, %s", newCid1,
 					newCid2)
 			}
@@ -246,6 +289,8 @@ func getMigrationFuncsForNetwork(nv network.Version) (UpgradeActorsFunc, PreUpgr
 		return filcns.UpgradeActorsV11, filcns.PreUpgradeActorsV11, checkNv19Invariants, nil
 	case network.Version21:
 		return filcns.UpgradeActorsV12, filcns.PreUpgradeActorsV12, checkNv21Invariants, nil
+	case network.Version22:
+		return filcns.UpgradeActorsV13, filcns.PreUpgradeActorsV13, checkNv22Invariants, nil
 	default:
 		return nil, nil, nil, xerrors.Errorf("migration not implemented for nv%d", nv)
 	}
@@ -255,6 +300,357 @@ type UpgradeActorsFunc = func(context.Context, *stmgr.StateManager, stmgr.Migrat
 type PreUpgradeActorsFunc = func(context.Context, *stmgr.StateManager, stmgr.MigrationCache, cid.Cid, abi.ChainEpoch, *types.TipSet) error
 type CheckInvariantsFunc = func(context.Context, cid.Cid, cid.Cid, blockstore.Blockstore, abi.ChainEpoch) error
 
+func printStateDiff(ctx context.Context, nv network.Version, newCid1, newCid2 cid.Cid, bs blockstore.Blockstore) error {
+	// migration diff
+	var sra, srb types.StateRoot
+	cst := cbornode.NewCborStore(bs)
+
+	if err := cst.Get(ctx, newCid1, &sra); err != nil {
+		return err
+	}
+	if err := cst.Get(ctx, newCid2, &srb); err != nil {
+		return err
+	}
+
+	if sra.Version != srb.Version {
+		fmt.Println("state root versions do not match: ", sra.Version, srb.Version)
+	}
+	if sra.Info != srb.Info {
+		fmt.Println("state root infos do not match: ", sra.Info, srb.Info)
+	}
+	if sra.Actors != srb.Actors {
+		fmt.Println("state root actors do not match: ", sra.Actors, srb.Actors)
+		if err := printActorsDiff(ctx, cst, nv, sra.Actors, srb.Actors); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func printActorsDiff(ctx context.Context, cst *cbornode.BasicIpldStore, nv network.Version, a, b cid.Cid) error {
+	// actor diff, a b are a hamt
+
+	diffs, err := hamt.Diff(ctx, cst, cst, a, b, hamt.UseTreeBitWidth(builtin.DefaultHamtBitwidth))
+	if err != nil {
+		return err
+	}
+
+	keyParser := func(k string) (interface{}, error) {
+		return address.NewFromBytes([]byte(k))
+	}
+
+	for _, d := range diffs {
+		switch d.Type {
+		case hamt.Add:
+			color.Green("+ Add %v", must.One(keyParser(d.Key)))
+		case hamt.Remove:
+			color.Red("- Remove %v", must.One(keyParser(d.Key)))
+		case hamt.Modify:
+			addr := must.One(keyParser(d.Key)).(address.Address)
+			color.Yellow("~ Modify %v", addr)
+			var aa, bb types.ActorV5
+
+			if err := aa.UnmarshalCBOR(bytes.NewReader(d.Before.Raw)); err != nil {
+				return err
+			}
+			if err := bb.UnmarshalCBOR(bytes.NewReader(d.After.Raw)); err != nil {
+				return err
+			}
+
+			if err := printActorDiff(ctx, cst, nv, addr, aa, bb); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func printActorDiff(ctx context.Context, cst *cbornode.BasicIpldStore, nv network.Version, addr address.Address, a, b types.ActorV5) error {
+	if a.Code != b.Code {
+		fmt.Println("  Code: ", a.Code, b.Code)
+	}
+	if a.Head != b.Head {
+		fmt.Println("  Head: ", a.Head, b.Head)
+	}
+	if a.Nonce != b.Nonce {
+		fmt.Println("  Nonce: ", a.Nonce, b.Nonce)
+	}
+	if big.Cmp(a.Balance, b.Balance) == 0 {
+		fmt.Println("  Balance: ", a.Balance, b.Balance)
+	}
+
+	switch addr.String() {
+	case "f05":
+		if err := printMarketActorDiff(ctx, cst, nv, a.Head, b.Head); err != nil {
+			return err
+		}
+	default:
+		fmt.Println("no logic to diff actor state for ", addr)
+	}
+
+	return nil
+}
+
+func printMarketActorDiff(ctx context.Context, cst *cbornode.BasicIpldStore, nv network.Version, a, b cid.Cid) error {
+	if nv != network.Version22 {
+		return xerrors.Errorf("market actor diff not implemented for nv%d", nv)
+	}
+
+	var ma, mb market13.State
+	if err := cst.Get(ctx, a, &ma); err != nil {
+		return err
+	}
+	if err := cst.Get(ctx, b, &mb); err != nil {
+		return err
+	}
+
+	if ma.Proposals != mb.Proposals {
+		fmt.Println("  Proposals: ", ma.Proposals, mb.Proposals)
+	}
+	if ma.States != mb.States {
+		fmt.Println("  States: ", ma.States, mb.States)
+
+		// diff the AMTs
+		amtDiff, err := amt.Diff(ctx, cst, cst, ma.States, mb.States, amt.UseTreeBitWidth(market13.StatesAmtBitwidth))
+		if err != nil {
+			return err
+		}
+
+		proposalsArrA, err := adt13.AsArray(adt8.WrapStore(ctx, cst), ma.Proposals, market13.ProposalsAmtBitwidth)
+		if err != nil {
+			return err
+		}
+		proposalsArrB, err := adt13.AsArray(adt8.WrapStore(ctx, cst), mb.Proposals, market13.ProposalsAmtBitwidth)
+		if err != nil {
+			return err
+		}
+
+		for _, d := range amtDiff {
+			switch d.Type {
+			case amt.Add:
+				color.Green("  state + Add %v", d.Key)
+			case amt.Remove:
+				color.Red("  state - Remove %v", d.Key)
+			case amt.Modify:
+				color.Yellow("  state ~ Modify %v", d.Key)
+
+				var a, b market13.DealState
+				if err := a.UnmarshalCBOR(bytes.NewReader(d.Before.Raw)); err != nil {
+					return err
+				}
+				if err := b.UnmarshalCBOR(bytes.NewReader(d.After.Raw)); err != nil {
+					return err
+				}
+
+				ja, err := json.Marshal(a)
+				if err != nil {
+					return err
+				}
+				jb, err := json.Marshal(b)
+				if err != nil {
+					return err
+				}
+
+				fmt.Println("   A: ", string(ja))
+				fmt.Println("   B: ", string(jb))
+
+				var propA, propB market13.DealProposal
+
+				if _, err := proposalsArrA.Get(d.Key, &propA); err != nil {
+					return err
+				}
+				if _, err := proposalsArrB.Get(d.Key, &propB); err != nil {
+					return err
+				}
+
+				pab, err := json.Marshal(propA)
+				if err != nil {
+					return err
+				}
+				pbb, err := json.Marshal(propB)
+				if err != nil {
+					return err
+				}
+				if string(pab) != string(pbb) {
+					fmt.Println("   PropA: ", string(pab))
+					fmt.Println("   PropB: ", string(pbb))
+				} else {
+					fmt.Println("   Prop: ", string(pab))
+				}
+
+			}
+		}
+	}
+	if ma.PendingProposals != mb.PendingProposals {
+		fmt.Println("  PendingProposals: ", ma.PendingProposals, mb.PendingProposals)
+	}
+	if ma.EscrowTable != mb.EscrowTable {
+		fmt.Println("  EscrowTable: ", ma.EscrowTable, mb.EscrowTable)
+	}
+	if ma.LockedTable != mb.LockedTable {
+		fmt.Println("  LockedTable: ", ma.LockedTable, mb.LockedTable)
+	}
+	if ma.NextID != mb.NextID {
+		fmt.Println("  NextID: ", ma.NextID, mb.NextID)
+	}
+	if ma.DealOpsByEpoch != mb.DealOpsByEpoch {
+		fmt.Println("  DealOpsByEpoch: ", ma.DealOpsByEpoch, mb.DealOpsByEpoch)
+	}
+	if ma.LastCron != mb.LastCron {
+		fmt.Println("  LastCron: ", ma.LastCron, mb.LastCron)
+	}
+	if ma.TotalClientLockedCollateral != mb.TotalClientLockedCollateral {
+		fmt.Println("  TotalClientLockedCollateral: ", ma.TotalClientLockedCollateral, mb.TotalClientLockedCollateral)
+	}
+	if ma.TotalProviderLockedCollateral != mb.TotalProviderLockedCollateral {
+		fmt.Println("  TotalProviderLockedCollateral: ", ma.TotalProviderLockedCollateral, mb.TotalProviderLockedCollateral)
+	}
+	if ma.TotalClientStorageFee != mb.TotalClientStorageFee {
+		fmt.Println("  TotalClientStorageFee: ", ma.TotalClientStorageFee, mb.TotalClientStorageFee)
+	}
+	if ma.PendingDealAllocationIds != mb.PendingDealAllocationIds {
+		fmt.Println("  PendingDealAllocationIds: ", ma.PendingDealAllocationIds, mb.PendingDealAllocationIds)
+	}
+	if ma.ProviderSectors != mb.ProviderSectors {
+		fmt.Println("  ProviderSectors: ", ma.ProviderSectors, mb.ProviderSectors)
+
+		// diff the HAMTs
+		hamtDiff, err := hamt.Diff(ctx, cst, cst, ma.ProviderSectors, mb.ProviderSectors, hamt.UseTreeBitWidth(market13.ProviderSectorsHamtBitwidth))
+		if err != nil {
+			return err
+		}
+
+		for _, d := range hamtDiff {
+			spIDk := must.One(abi.ParseUIntKey(d.Key))
+
+			switch d.Type {
+			case hamt.Add:
+				color.Green("  ProviderSectors + Add f0%v", spIDk)
+
+				var b cbg.CborCid
+				if err := b.UnmarshalCBOR(bytes.NewReader(d.After.Raw)); err != nil {
+					return err
+				}
+
+				fmt.Println("  |-B: ", cid.Cid(b).String())
+
+				inner, err := adt13.AsMap(adt8.WrapStore(ctx, cst), cid.Cid(b), market13.ProviderSectorsHamtBitwidth)
+				if err != nil {
+					return err
+				}
+
+				var ids market13.SectorDealIDs
+				err = inner.ForEach(&ids, func(k string) error {
+					sectorNumber := must.One(abi.ParseUIntKey(k))
+
+					color.Green("  |-- ProviderSectors + Add %v", sectorNumber)
+					fmt.Printf("  |+: %v\n", ids)
+
+					return nil
+				})
+				if err != nil {
+					return err
+				}
+			case hamt.Remove:
+				color.Red("  ProviderSectors - Remove f0%v", spIDk)
+			case hamt.Modify:
+				color.Yellow("  ProviderSectors ~ Modify f0%v", spIDk)
+
+				var a, b cbg.CborCid
+				if err := a.UnmarshalCBOR(bytes.NewReader(d.Before.Raw)); err != nil {
+					return err
+				}
+				if err := b.UnmarshalCBOR(bytes.NewReader(d.After.Raw)); err != nil {
+					return err
+				}
+
+				fmt.Println("  |-A: ", cid.Cid(b).String())
+				fmt.Println("  |-B: ", cid.Cid(a).String())
+
+				// diff the inner HAMTs
+				innerHamtDiff, err := hamt.Diff(ctx, cst, cst, cid.Cid(a), cid.Cid(b), hamt.UseTreeBitWidth(market13.ProviderSectorsHamtBitwidth))
+				if err != nil {
+					return err
+				}
+
+				for _, d := range innerHamtDiff {
+					sectorNumber := must.One(abi.ParseUIntKey(d.Key))
+
+					switch d.Type {
+					case hamt.Add:
+						var b market13.SectorDealIDs
+
+						if err := b.UnmarshalCBOR(bytes.NewReader(d.After.Raw)); err != nil {
+							return err
+						}
+
+						color.Green("  |-- ProviderSectors + Add %v", sectorNumber)
+						fmt.Printf("  |B: %v\n", b)
+					case hamt.Remove:
+						var a market13.SectorDealIDs
+
+						if err := a.UnmarshalCBOR(bytes.NewReader(d.Before.Raw)); err != nil {
+							return err
+						}
+
+						color.Red("  |-- ProviderSectors - Remove %v", sectorNumber)
+						fmt.Printf("  |A: %v\n", a)
+					case hamt.Modify:
+						var a, b market13.SectorDealIDs
+
+						if err := a.UnmarshalCBOR(bytes.NewReader(d.Before.Raw)); err != nil {
+							return err
+						}
+						if err := b.UnmarshalCBOR(bytes.NewReader(d.After.Raw)); err != nil {
+							return err
+						}
+
+						color.Yellow("  |-- ProviderSectors ~ Modify %v", sectorNumber)
+						fmt.Printf("  |A: %v\n", a)
+						fmt.Printf("  |B: %v\n", b)
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func checkNv22Invariants(ctx context.Context, oldStateRootCid cid.Cid, newStateRootCid cid.Cid, bs blockstore.Blockstore, epoch abi.ChainEpoch) error {
+
+	actorStore := store.ActorStore(ctx, bs)
+	startTime := time.Now()
+
+	// Load the new state root.
+	var newStateRoot types.StateRoot
+	if err := actorStore.Get(ctx, newStateRootCid, &newStateRoot); err != nil {
+		return xerrors.Errorf("failed to decode state root: %w", err)
+	}
+
+	actorCodeCids, err := actors.GetActorCodeIDs(actorstypes.Version13)
+	if err != nil {
+		return err
+	}
+	newActorTree, err := builtin.LoadTree(actorStore, newStateRoot.Actors)
+	if err != nil {
+		return err
+	}
+	messages, err := v13.CheckStateInvariants(newActorTree, epoch, actorCodeCids)
+	if err != nil {
+		return xerrors.Errorf("checking state invariants: %w", err)
+	}
+
+	for _, message := range messages.Messages() {
+		fmt.Println("got the following error: ", message)
+	}
+
+	fmt.Println("completed invariant checks, took ", time.Since(startTime))
+
+	return nil
+}
 func checkNv21Invariants(ctx context.Context, oldStateRootCid cid.Cid, newStateRootCid cid.Cid, bs blockstore.Blockstore, epoch abi.ChainEpoch) error {
 
 	actorStore := store.ActorStore(ctx, bs)

go.mod

@@ -43,6 +43,7 @@ require (
 	github.com/filecoin-project/go-fil-commcid v0.1.0
 	github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
 	github.com/filecoin-project/go-fil-markets v1.28.3
+	github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0
 	github.com/filecoin-project/go-jsonrpc v0.3.1
 	github.com/filecoin-project/go-padreader v0.0.1
 	github.com/filecoin-project/go-paramfetch v0.0.4
@@ -207,7 +208,6 @@ require (
 	github.com/filecoin-project/go-ds-versioning v0.1.2 // indirect
 	github.com/filecoin-project/go-hamt-ipld v0.1.5 // indirect
 	github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 // indirect
-	github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0 // indirect
 	github.com/flynn/noise v1.1.0 // indirect
 	github.com/francoispqt/gojay v1.2.13 // indirect
 	github.com/gdamore/encoding v1.0.0 // indirect