feat: chain: light-weight patch to fix calibrationnet (#11363)

* refactor: make GetEmbeddedBuiltinActorsBundle take network bundle name

* update calibnet actor bundle to v12.0.0-rc.2, but include v12.0.0-rc.1 as calibrationnet-buggy.car

* wip: calibnet unbork migration

* calibnet: add buggy miner actor CID to actorMeta

* fix incorrect buggy calibnet manifest

* make UpgradeWatermelonFixHeight a build param

* calibnet patch: check whether network is calibration from init actor state

* add sanity checks to the v12 calibnet patch upgrade

* address review

build/builtin_actors.go

@@ -42,6 +42,12 @@ func init() {
 	if err := loadManifests(NetworkBundle); err != nil {
 		panic(err)
 	}
+
+	// The following code cid existed temporarily on the calibnet testnet, as a "buggy" storage miner actor implementation.
+	// We include it in our builtin bundle, but intentionally omit from metadata.
+	if NetworkBundle == "calibrationnet" {
+		actors.AddActorMeta("storageminer", cid.MustParse("bafk2bzacecnh2ouohmonvebq7uughh4h3ppmg4cjsk74dzxlbbtlcij4xbzxq"), actorstypes.Version12)
+	}
 }
 
 // UseNetworkBundle switches to a different network bundle, by name.
@@ -183,6 +189,12 @@ func readEmbeddedBuiltinActorsMetadata(bundle string) ([]*BuiltinActorsMetadata,
 		if err != nil {
 			return nil, xerrors.Errorf("error loading builtin actors bundle: %w", err)
 		}
+
+		// The following manifest cid existed temporarily on the calibnet testnet
+		// We include it in our builtin bundle, but intentionally omit from metadata
+		if root == cid.MustParse("bafy2bzacedrunxfqta5skb7q7x32lnp4efz2oq7fn226ffm7fu5iqs62jkmvs") {
+			continue
+		}
 		bundles = append(bundles, &BuiltinActorsMetadata{
 			Network:     name,
 			Version:     actorstypes.Version(version),
@@ -232,7 +244,7 @@ func readBundleManifest(r io.Reader) (cid.Cid, map[string]cid.Cid, error) {
 }
 
 // GetEmbeddedBuiltinActorsBundle returns the builtin-actors bundle for the given actors version.
-func GetEmbeddedBuiltinActorsBundle(version actorstypes.Version) ([]byte, bool) {
+func GetEmbeddedBuiltinActorsBundle(version actorstypes.Version, networkBundleName string) ([]byte, bool) {
 	fi, err := embeddedBuiltinActorReleases.Open(fmt.Sprintf("actors/v%d.tar.zst", version))
 	if err != nil {
 		return nil, false
@@ -243,7 +255,7 @@ func GetEmbeddedBuiltinActorsBundle(version actorstypes.Version) ([]byte, bool)
 	defer uncompressed.Close() //nolint
 
 	tarReader := tar.NewReader(uncompressed)
-	targetFileName := fmt.Sprintf("builtin-actors-%s.car", NetworkBundle)
+	targetFileName := fmt.Sprintf("builtin-actors-%s.car", networkBundleName)
 	for {
 		header, err := tarReader.Next()
 		switch err {

build/builtin_actors_gen.go

@@ -203,8 +203,8 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 }, {
 	Network:      "calibrationnet",
 	Version:      12,
-	BundleGitTag: "v12.0.0-rc.1",
+	BundleGitTag: "v12.0.0-rc.2",
-	ManifestCid:  MustParseCid("bafy2bzacedrunxfqta5skb7q7x32lnp4efz2oq7fn226ffm7fu5iqs62jkmvs"),
+	ManifestCid:  MustParseCid("bafy2bzacebl4w5ptfvuw6746w7ev562idkbf5ppq72e6zub22435ws2rukzru"),
 	Actors: map[string]cid.Cid{
 		"account":          MustParseCid("bafk2bzacechwwxdqvggkdylm37zldjsra2ivkdzwp7fee56bzxbzs544wv6u6"),
 		"cron":             MustParseCid("bafk2bzacec4gdxxkqwxqqodsv6ug5dmdbqdfqwyqfek3yhxc2wweh5psxaeq6"),
@@ -218,7 +218,7 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 		"placeholder":      MustParseCid("bafk2bzacedfvut2myeleyq67fljcrw4kkmn5pb5dpyozovj7jpoez5irnc3ro"),
 		"reward":           MustParseCid("bafk2bzacedra77pcglf7vdca2itcaa4vd6xrxynxmgfgdjdxqxfwqyhtoxehy"),
 		"storagemarket":    MustParseCid("bafk2bzacea7g46y7xxu2zjq2h75x6mmx3utz2uxnlvnwi6tzpsvulna3bmiva"),
-		"storageminer":     MustParseCid("bafk2bzacecnh2ouohmonvebq7uughh4h3ppmg4cjsk74dzxlbbtlcij4xbzxq"),
+		"storageminer":     MustParseCid("bafk2bzaced7emkbbnrewv5uvrokxpf5tlm4jslu2jsv77ofw2yqdglg657uie"),
 		"storagepower":     MustParseCid("bafk2bzacedd3ka44k7d46ckbinjhv3diyuu2epgbyvhqqyjkc64qlrg3wlgzi"),
 		"system":           MustParseCid("bafk2bzacecioupndtcnyw6iq2hbrxag3aufvczlv5nobnfbkbywqzcyfaa376"),
 		"verifiedregistry": MustParseCid("bafk2bzaceavldupmf7bimeeacs67z5xdfdlfca6p7sn6bev3mt5ggepfqvhqo"),

build/params_2k.go

@@ -67,6 +67,9 @@ var UpgradeThunderHeight = abi.ChainEpoch(-23)
 
 var UpgradeWatermelonHeight = abi.ChainEpoch(200)
 
+// This fix upgrade only ran on calibrationnet
+const UpgradeWatermelonFixHeight = -100
+
 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 	0: DrandMainnet,
 }

build/params_butterfly.go

@@ -57,6 +57,9 @@ const UpgradeThunderHeight = -23
 
 const UpgradeWatermelonHeight = 400
 
+// This fix upgrade only ran on calibrationnet
+const UpgradeWatermelonFixHeight = -100
+
 var SupportedProofTypes = []abi.RegisteredSealProof{
 	abi.RegisteredSealProof_StackedDrg512MiBV1,
 	abi.RegisteredSealProof_StackedDrg32GiBV1,

build/params_calibnet.go

@@ -79,9 +79,11 @@ const UpgradeLightningHeight = 489094
 // 2023-04-21T16:00:00Z
 const UpgradeThunderHeight = UpgradeLightningHeight + 3120
 
-// 2023-10-19T13:00:00Z
+// 2023-11-07T13:00:00Z
 const UpgradeWatermelonHeight = 1013134
 
+const UpgradeWatermelonFixHeight = 1070494
+
 var SupportedProofTypes = []abi.RegisteredSealProof{
 	abi.RegisteredSealProof_StackedDrg32GiBV1,
 	abi.RegisteredSealProof_StackedDrg64GiBV1,

build/params_interop.go

@@ -56,6 +56,9 @@ var UpgradeThunderHeight = abi.ChainEpoch(-23)
 
 const UpgradeWatermelonHeight = 50
 
+// This fix upgrade only ran on calibrationnet
+const UpgradeWatermelonFixHeight = -1
+
 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 	0: DrandMainnet,
 }

build/params_mainnet.go

@@ -98,6 +98,9 @@ const UpgradeThunderHeight = UpgradeLightningHeight + 2880*21
 // ???????
 var UpgradeWatermelonHeight = abi.ChainEpoch(9999999999)
 
+// This fix upgrade only ran on calibrationnet
+const UpgradeWatermelonFixHeight = -1
+
 var SupportedProofTypes = []abi.RegisteredSealProof{
 	abi.RegisteredSealProof_StackedDrg32GiBV1,
 	abi.RegisteredSealProof_StackedDrg64GiBV1,

build/params_testground.go

@@ -110,6 +110,7 @@ var (
 	UpgradeLightningHeight     abi.ChainEpoch = -21
 	UpgradeThunderHeight       abi.ChainEpoch = -22
 	UpgradeWatermelonHeight    abi.ChainEpoch = -23
+	UpgradeWatermelonFixHeight abi.ChainEpoch = -24
 
 	DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 		0: DrandMainnet,

chain/actors/builtin/builtin.go

@@ -23,6 +23,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors"
 )
 
+var InitActorAddr = builtin.InitActorAddr
 var SystemActorAddr = builtin.SystemActorAddr
 var BurntFundsActorAddr = builtin.BurntFundsActorAddr
 var CronActorAddr = builtin.CronActorAddr

chain/actors/builtin/builtin.go.template

@@ -23,6 +23,7 @@ import (
 	smoothingtypes "github.com/filecoin-project/go-state-types/builtin/v8/util/smoothing"
 )
 
+var InitActorAddr = builtin.InitActorAddr
 var SystemActorAddr = builtin.SystemActorAddr
 var BurntFundsActorAddr = builtin.BurntFundsActorAddr
 var CronActorAddr = builtin.CronActorAddr

chain/actors/manifest.go

@@ -51,6 +51,12 @@ func RegisterManifest(av actorstypes.Version, manifestCid cid.Cid, entries map[s
 	}
 }
 
+func AddActorMeta(name string, codeId cid.Cid, av actorstypes.Version) {
+	manifestMx.Lock()
+	defer manifestMx.Unlock()
+	actorMeta[codeId] = actorEntry{name: name, version: av}
+}
+
 // GetManifest gets a loaded manifest.
 func GetManifest(av actorstypes.Version) (cid.Cid, bool) {
 	manifestMx.RLock()

chain/consensus/filcns/upgrades.go

@@ -1,6 +1,7 @@
 package filcns
 
 import (
+	"bytes"
 	"context"
 	_ "embed"
 	"fmt"
@@ -19,7 +20,9 @@ import (
 	actorstypes "github.com/filecoin-project/go-state-types/actors"
 	"github.com/filecoin-project/go-state-types/big"
 	nv18 "github.com/filecoin-project/go-state-types/builtin/v10/migration"
+	init11 "github.com/filecoin-project/go-state-types/builtin/v11/init"
 	nv19 "github.com/filecoin-project/go-state-types/builtin/v11/migration"
+	system11 "github.com/filecoin-project/go-state-types/builtin/v11/system"
 	nv21 "github.com/filecoin-project/go-state-types/builtin/v12/migration"
 	nv17 "github.com/filecoin-project/go-state-types/builtin/v9/migration"
 	"github.com/filecoin-project/go-state-types/manifest"
@@ -273,6 +276,10 @@ func DefaultUpgradeSchedule() stmgr.UpgradeSchedule {
 			StopWithin:      10,
 		}},
 		Expensive: true,
+	}, {
+		Height:    build.UpgradeWatermelonFixHeight,
+		Network:   network.Version21,
+		Migration: upgradeActorsV12Fix,
 	},
 	}
 
@@ -1869,6 +1876,8 @@ func UpgradeActorsV12(ctx context.Context, sm *stmgr.StateManager, cache stmgr.M
 	return newRoot, nil
 }
 
+var calibnetv12BuggyBundle = cid.MustParse("bafy2bzacedrunxfqta5skb7q7x32lnp4efz2oq7fn226ffm7fu5iqs62jkmvs")
+
 func upgradeActorsV12Common(
 	ctx context.Context, sm *stmgr.StateManager, cache stmgr.MigrationCache,
 	root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet,
@@ -1894,13 +1903,50 @@ func upgradeActorsV12Common(
 		)
 	}
 
-	manifest, ok := actors.GetManifest(actorstypes.Version12)
+	// check whether or not this is a calibnet upgrade
+	// we do this because calibnet upgraded to a "wrong" actors bundle, which was then corrected
+	// we thus upgrade to calibrationnet-buggy in this upgrade
+	actorsIn, err := state.LoadStateTree(adtStore, root)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("loading state tree: %w", err)
+	}
+
+	initActor, err := actorsIn.GetActor(builtin.InitActorAddr)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("failed to get system actor: %w", err)
+	}
+
+	var initState init11.State
+	if err := adtStore.Get(ctx, initActor.Head, &initState); err != nil {
+		return cid.Undef, xerrors.Errorf("failed to get system actor state: %w", err)
+	}
+
+	var manifestCid cid.Cid
+	if initState.NetworkName == "calibrationnet" {
+		embedded, ok := build.GetEmbeddedBuiltinActorsBundle(actorstypes.Version12, "calibrationnet-buggy")
+		if !ok {
+			return cid.Undef, xerrors.Errorf("didn't find buggy calibrationnet bundle")
+		}
+
+		var err error
+		manifestCid, err = bundle.LoadBundle(ctx, writeStore, bytes.NewReader(embedded))
+		if err != nil {
+			return cid.Undef, xerrors.Errorf("failed to load buggy calibnet bundle: %w", err)
+		}
+
+		if manifestCid != calibnetv12BuggyBundle {
+			return cid.Undef, xerrors.Errorf("didn't find expected buggy calibnet bundle manifest: %s != %s", manifestCid, calibnetv12BuggyBundle)
+		}
+	} else {
+		ok := false
+		manifestCid, ok = actors.GetManifest(actorstypes.Version12)
 		if !ok {
 			return cid.Undef, xerrors.Errorf("no manifest CID for v12 upgrade")
 		}
+	}
 
 	// Perform the migration
-	newHamtRoot, err := nv21.MigrateStateTree(ctx, adtStore, manifest, stateRoot.Actors, epoch, config,
+	newHamtRoot, err := nv21.MigrateStateTree(ctx, adtStore, manifestCid, stateRoot.Actors, epoch, config,
 		migrationLogger{}, cache)
 	if err != nil {
 		return cid.Undef, xerrors.Errorf("upgrading to actors v12: %w", err)
@@ -1928,6 +1974,139 @@ func upgradeActorsV12Common(
 	return newRoot, nil
 }
 
+//////////////////////
+
+var calibnetv12BuggyMinerCID = cid.MustParse("bafk2bzacecnh2ouohmonvebq7uughh4h3ppmg4cjsk74dzxlbbtlcij4xbzxq")
+
+func upgradeActorsV12Fix(ctx context.Context, sm *stmgr.StateManager, cache stmgr.MigrationCache, cb stmgr.ExecMonitor,
+	root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
+	stateStore := sm.ChainStore().StateBlockstore()
+	adtStore := store.ActorStore(ctx, stateStore)
+
+	// ensure that the manifest is loaded in the blockstore
+	if err := bundle.LoadBundles(ctx, stateStore, actorstypes.Version12); err != nil {
+		return cid.Undef, xerrors.Errorf("failed to load manifest bundle: %w", err)
+	}
+
+	// Load input state tree
+	actorsIn, err := state.LoadStateTree(adtStore, root)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("loading state tree: %w", err)
+	}
+
+	// load old manifest data
+	systemActor, err := actorsIn.GetActor(builtin.SystemActorAddr)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("failed to get system actor: %w", err)
+	}
+
+	var systemState system11.State
+	if err := adtStore.Get(ctx, systemActor.Head, &systemState); err != nil {
+		return cid.Undef, xerrors.Errorf("failed to get system actor state: %w", err)
+	}
+
+	var oldManifestData manifest.ManifestData
+	if err := adtStore.Get(ctx, systemState.BuiltinActors, &oldManifestData); err != nil {
+		return cid.Undef, xerrors.Errorf("failed to get old manifest data: %w", err)
+	}
+
+	newManifestCID, ok := actors.GetManifest(actorstypes.Version12)
+	if !ok {
+		return cid.Undef, xerrors.Errorf("no manifest CID for v12 upgrade")
+	}
+
+	// load new manifest
+	var newManifest manifest.Manifest
+	if err := adtStore.Get(ctx, newManifestCID, &newManifest); err != nil {
+		return cid.Undef, xerrors.Errorf("error reading actor manifest: %w", err)
+	}
+
+	if err := newManifest.Load(ctx, adtStore); err != nil {
+		return cid.Undef, xerrors.Errorf("error loading actor manifest: %w", err)
+	}
+
+	// build the CID mapping
+	codeMapping := make(map[cid.Cid]cid.Cid, len(oldManifestData.Entries))
+	for _, oldEntry := range oldManifestData.Entries {
+		newCID, ok := newManifest.Get(oldEntry.Name)
+		if !ok {
+			return cid.Undef, xerrors.Errorf("missing manifest entry for %s", oldEntry.Name)
+		}
+
+		// Note: we expect newCID to be the same as oldEntry.Code for all actors except the miner actor
+		codeMapping[oldEntry.Code] = newCID
+	}
+
+	// Create empty actorsOut
+
+	actorsOut, err := state.NewStateTree(adtStore, actorsIn.Version())
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("failed to create new tree: %w", err)
+	}
+
+	// Perform the migration
+	err = actorsIn.ForEach(func(a address.Address, actor *types.Actor) error {
+		newCid, ok := codeMapping[actor.Code]
+		if !ok {
+			return xerrors.Errorf("didn't find mapping for %s", actor.Code)
+		}
+
+		return actorsOut.SetActor(a, &types.ActorV5{
+			Code:    newCid,
+			Head:    actor.Head,
+			Nonce:   actor.Nonce,
+			Balance: actor.Balance,
+			Address: actor.Address,
+		})
+	})
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("failed to perform migration: %w", err)
+	}
+
+	err = actorsIn.ForEach(func(a address.Address, inActor *types.Actor) error {
+		outActor, err := actorsOut.GetActor(a)
+		if err != nil {
+			return xerrors.Errorf("failed to get actor in outTree: %w", err)
+		}
+
+		if inActor.Nonce != outActor.Nonce {
+			return xerrors.Errorf("mismatched nonce for actor %s", a)
+		}
+
+		if !inActor.Balance.Equals(outActor.Balance) {
+			return xerrors.Errorf("mismatched balance for actor %s: %d != %d", a, inActor.Balance, outActor.Balance)
+		}
+
+		if inActor.Address != outActor.Address && inActor.Address.String() != outActor.Address.String() {
+			return xerrors.Errorf("mismatched address for actor %s: %s != %s", a, inActor.Address, outActor.Address)
+		}
+
+		if inActor.Head != outActor.Head {
+			return xerrors.Errorf("mismatched head for actor %s", a)
+		}
+
+		// This is the hard-coded "buggy" miner actor Code ID
+		if inActor.Code != calibnetv12BuggyMinerCID && inActor.Code != outActor.Code {
+			return xerrors.Errorf("unexpected change in code for actor %s", a)
+		}
+
+		return nil
+	})
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("failed to sanity check migration: %w", err)
+	}
+
+	// Persist the result.
+	newRoot, err := actorsOut.Flush(ctx)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("failed to persist new state root: %w", err)
+	}
+
+	return newRoot, nil
+}
+
+////////////////////
+
 // Example upgrade function if upgrade requires only code changes
 //func UpgradeActorsV9(ctx context.Context, sm *stmgr.StateManager, _ stmgr.MigrationCache, _ stmgr.ExecMonitor, root cid.Cid, _ abi.ChainEpoch, _ *types.TipSet) (cid.Cid, error) {
 //	buf := blockstore.NewTieredBstore(sm.ChainStore().StateBlockstore(), blockstore.NewMemorySync())

node/bundle/bundle.go

@@ -69,7 +69,7 @@ func LoadBundles(ctx context.Context, bs blockstore.Blockstore, versions ...acto
 		)
 		if path, ok := build.BundleOverrides[av]; ok {
 			root, err = LoadBundleFromFile(ctx, bs, path)
-		} else if embedded, ok := build.GetEmbeddedBuiltinActorsBundle(av); ok {
+		} else if embedded, ok := build.GetEmbeddedBuiltinActorsBundle(av, build.NetworkBundle); ok {
 			root, err = LoadBundle(ctx, bs, bytes.NewReader(embedded))
 		} else {
 			err = xerrors.Errorf("bundle for actors version v%d not found", av)