lotus/cmd/lotus-shed/migrations.go

735 lines
21 KiB
Go

package main
import (
"context"
"fmt"
"io"
"strconv"
"time"
"github.com/ipfs/go-cid"
"github.com/urfave/cli/v2"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
actorstypes "github.com/filecoin-project/go-state-types/actors"
"github.com/filecoin-project/go-state-types/builtin"
market8 "github.com/filecoin-project/go-state-types/builtin/v8/market"
adt8 "github.com/filecoin-project/go-state-types/builtin/v8/util/adt"
v9 "github.com/filecoin-project/go-state-types/builtin/v9"
market9 "github.com/filecoin-project/go-state-types/builtin/v9/market"
miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner"
adt9 "github.com/filecoin-project/go-state-types/builtin/v9/util/adt"
verifreg9 "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
"github.com/filecoin-project/go-state-types/manifest"
mutil "github.com/filecoin-project/go-state-types/migration"
"github.com/filecoin-project/go-state-types/network"
"github.com/filecoin-project/specs-actors/v7/actors/migration/nv15"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/adt"
lbuiltin "github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/actors/builtin/datacap"
"github.com/filecoin-project/lotus/chain/actors/builtin/market"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/verifreg"
"github.com/filecoin-project/lotus/chain/consensus"
"github.com/filecoin-project/lotus/chain/consensus/filcns"
"github.com/filecoin-project/lotus/chain/state"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/lotus/storage/sealer/ffiwrapper"
)
var migrationsCmd = &cli.Command{
Name: "migrate-state",
Description: "Run a network upgrade migration",
ArgsUsage: "[new network version, block to look back from]",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "repo",
Value: "~/.lotus",
EnvVars: []string{"LOTUS_PATH"},
},
&cli.BoolFlag{
Name: "skip-pre-migration",
},
&cli.BoolFlag{
Name: "check-invariants",
},
},
Action: func(cctx *cli.Context) error {
ctx := context.TODO()
if cctx.NArg() != 2 {
return lcli.IncorrectNumArgs(cctx)
}
nv, err := strconv.ParseUint(cctx.Args().Get(0), 10, 32)
if err != nil {
return fmt.Errorf("failed to parse network version: %w", err)
}
upgradeActorsFunc, preUpgradeActorsFunc, checkInvariantsFunc, err := getMigrationFuncsForNetwork(network.Version(nv))
if err != nil {
return err
}
blkCid, err := cid.Decode(cctx.Args().Get(1))
if err != nil {
return fmt.Errorf("failed to parse input: %w", err)
}
fsrepo, err := repo.NewFS(cctx.String("repo"))
if err != nil {
return err
}
lkrepo, err := fsrepo.Lock(repo.FullNode)
if err != nil {
return err
}
defer lkrepo.Close() //nolint:errcheck
bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore)
if err != nil {
return fmt.Errorf("failed to open blockstore: %w", err)
}
defer func() {
if c, ok := bs.(io.Closer); ok {
if err := c.Close(); err != nil {
log.Warnf("failed to close blockstore: %s", err)
}
}
}()
mds, err := lkrepo.Datastore(context.Background(), "/metadata")
if err != nil {
return err
}
cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil)
defer cs.Close() //nolint:errcheck
sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil)
if err != nil {
return err
}
blk, err := cs.GetBlock(ctx, blkCid)
if err != nil {
return err
}
migrationTs, err := cs.LoadTipSet(ctx, types.NewTipSetKey(blk.Parents...))
if err != nil {
return err
}
startTime := time.Now()
newCid2, err := upgradeActorsFunc(ctx, sm, nv15.NewMemMigrationCache(), nil, blk.ParentStateRoot, blk.Height-1, migrationTs)
if err != nil {
return err
}
uncachedMigrationTime := time.Since(startTime)
fmt.Println("migration height ", blk.Height-1)
fmt.Println("old cid ", blk.ParentStateRoot)
fmt.Println("new cid ", newCid2)
fmt.Println("completed round actual (without cache), took ", uncachedMigrationTime)
if !cctx.IsSet("skip-pre-migration") {
cache := mutil.NewMemMigrationCache()
ts1, err := cs.GetTipsetByHeight(ctx, blk.Height-240, migrationTs, false)
if err != nil {
return err
}
startTime = time.Now()
err = preUpgradeActorsFunc(ctx, sm, cache, ts1.ParentState(), ts1.Height()-1, ts1)
if err != nil {
return err
}
preMigration1Time := time.Since(startTime)
ts2, err := cs.GetTipsetByHeight(ctx, blk.Height-15, migrationTs, false)
if err != nil {
return err
}
startTime = time.Now()
err = preUpgradeActorsFunc(ctx, sm, cache, ts2.ParentState(), ts2.Height()-1, ts2)
if err != nil {
return err
}
preMigration2Time := time.Since(startTime)
startTime = time.Now()
newCid1, err := upgradeActorsFunc(ctx, sm, cache, nil, blk.ParentStateRoot, blk.Height-1, migrationTs)
if err != nil {
return err
}
cachedMigrationTime := time.Since(startTime)
if newCid1 != newCid2 {
return xerrors.Errorf("got different results with and without the cache: %s, %s", newCid1,
newCid2)
}
fmt.Println("completed premigration 1, took ", preMigration1Time)
fmt.Println("completed premigration 2, took ", preMigration2Time)
fmt.Println("completed round actual (with cache), took ", cachedMigrationTime)
}
if cctx.Bool("check-invariants") {
if checkInvariantsFunc == nil {
return xerrors.Errorf("check invariants not implemented for nv%d", nv)
}
err = checkInvariantsFunc(ctx, blk.ParentStateRoot, newCid2, bs, blk.Height-1)
if err != nil {
return err
}
}
return nil
},
}
func getMigrationFuncsForNetwork(nv network.Version) (UpgradeActorsFunc, PreUpgradeActorsFunc, CheckInvariantsFunc, error) {
switch nv {
case network.Version17:
return filcns.UpgradeActorsV9, filcns.PreUpgradeActorsV9, checkNv17Invariants, nil
case network.Version18:
return filcns.UpgradeActorsV10, filcns.PreUpgradeActorsV10, nil, nil
default:
return nil, nil, nil, xerrors.Errorf("migration not implemented for nv%d", nv)
}
}
type UpgradeActorsFunc = func(context.Context, *stmgr.StateManager, stmgr.MigrationCache, stmgr.ExecMonitor, cid.Cid, abi.ChainEpoch, *types.TipSet) (cid.Cid, error)
type PreUpgradeActorsFunc = func(context.Context, *stmgr.StateManager, stmgr.MigrationCache, cid.Cid, abi.ChainEpoch, *types.TipSet) error
type CheckInvariantsFunc = func(context.Context, cid.Cid, cid.Cid, blockstore.Blockstore, abi.ChainEpoch) error
func checkNv17Invariants(ctx context.Context, v8StateRootCid cid.Cid, v9StateRootCid cid.Cid, bs blockstore.Blockstore, epoch abi.ChainEpoch) error {
actorStore := store.ActorStore(ctx, bs)
startTime := time.Now()
stateTreeV8, err := state.LoadStateTree(actorStore, v8StateRootCid)
if err != nil {
return err
}
stateTreeV9, err := state.LoadStateTree(actorStore, v9StateRootCid)
if err != nil {
return err
}
err = checkDatacaps(stateTreeV8, stateTreeV9, actorStore)
if err != nil {
return err
}
err = checkPendingVerifiedDeals(stateTreeV8, stateTreeV9, actorStore)
if err != nil {
return err
}
err = checkAllMinersUnsealedCID(stateTreeV9, actorStore)
if err != nil {
return err
}
// Load the state root.
var v9stateRoot types.StateRoot
if err := actorStore.Get(ctx, v9StateRootCid, &v9stateRoot); err != nil {
return xerrors.Errorf("failed to decode state root: %w", err)
}
actorCodeCids, err := actors.GetActorCodeIDs(actorstypes.Version9)
if err != nil {
return err
}
v9actorTree, err := builtin.LoadTree(actorStore, v9stateRoot.Actors)
if err != nil {
return err
}
messages, err := v9.CheckStateInvariants(v9actorTree, epoch, actorCodeCids)
if err != nil {
return xerrors.Errorf("checking state invariants: %w", err)
}
for _, message := range messages.Messages() {
fmt.Println("got the following error: ", message)
}
fmt.Println("completed invariant checks, took ", time.Since(startTime))
return nil
}
func checkDatacaps(stateTreeV8 *state.StateTree, stateTreeV9 *state.StateTree, actorStore adt.Store) error {
verifregDatacaps, err := getVerifreg8Datacaps(stateTreeV8, actorStore)
if err != nil {
return err
}
newDatacaps, err := getDatacap9Datacaps(stateTreeV9, actorStore)
if err != nil {
return err
}
// Should have all the v8 datacaps, plus the verifreg actor itself
if len(verifregDatacaps)+1 != len(newDatacaps) {
return xerrors.Errorf("size of datacap maps do not match. verifreg: %d, datacap: %d", len(verifregDatacaps), len(newDatacaps))
}
for addr, oldDcap := range verifregDatacaps {
dcap, ok := newDatacaps[addr]
if !ok {
return xerrors.Errorf("datacap for address: %s not found in datacap state", addr)
}
if !dcap.Equals(oldDcap) {
return xerrors.Errorf("datacap for address: %s do not match. verifreg: %d, datacap: %d", addr, oldDcap, dcap)
}
}
return nil
}
func getVerifreg8Datacaps(stateTreeV8 *state.StateTree, actorStore adt.Store) (map[address.Address]abi.StoragePower, error) {
verifregStateV8, err := getVerifregActorV8(stateTreeV8, actorStore)
if err != nil {
return nil, xerrors.Errorf("failed to get verifreg actor state: %w", err)
}
var verifregDatacaps = make(map[address.Address]abi.StoragePower)
err = verifregStateV8.ForEachClient(func(addr address.Address, dcap abi.StoragePower) error {
verifregDatacaps[addr] = dcap
return nil
})
if err != nil {
return nil, err
}
return verifregDatacaps, nil
}
func getDatacap9Datacaps(stateTreeV9 *state.StateTree, actorStore adt.Store) (map[address.Address]abi.StoragePower, error) {
datacapStateV9, err := getDatacapActorV9(stateTreeV9, actorStore)
if err != nil {
return nil, xerrors.Errorf("failed to get datacap actor state: %w", err)
}
var datacaps = make(map[address.Address]abi.StoragePower)
err = datacapStateV9.ForEachClient(func(addr address.Address, dcap abi.StoragePower) error {
datacaps[addr] = dcap
return nil
})
if err != nil {
return nil, err
}
return datacaps, nil
}
func checkPendingVerifiedDeals(stateTreeV8 *state.StateTree, stateTreeV9 *state.StateTree, actorStore adt.Store) error {
marketActorV9, err := getMarketActorV9(stateTreeV9, actorStore)
if err != nil {
return err
}
verifregActorV9, err := getVerifregActorV9(stateTreeV9, actorStore)
if err != nil {
return err
}
verifregStateV9, err := getVerifregStateV9(stateTreeV9, actorStore)
if err != nil {
return err
}
marketStateV8, err := getMarketStateV8(stateTreeV8, actorStore)
if err != nil {
return err
}
marketStateV9, err := getMarketStateV9(stateTreeV9, actorStore)
if err != nil {
return err
}
pendingProposalsV8, err := adt8.AsSet(actorStore, marketStateV8.PendingProposals, builtin.DefaultHamtBitwidth)
if err != nil {
return xerrors.Errorf("failed to load pending proposals: %w", err)
}
dealProposalsV8, err := market8.AsDealProposalArray(actorStore, marketStateV8.Proposals)
if err != nil {
return xerrors.Errorf("failed to get proposals: %w", err)
}
// We only want those pending deals that haven't been activated -- an activated deal has an entry in dealStates8
dealStates8, err := adt9.AsArray(actorStore, marketStateV8.States, market8.StatesAmtBitwidth)
if err != nil {
return xerrors.Errorf("failed to load v8 states array: %w", err)
}
var numPendingVerifiedDeals = 0
var proposal market8.DealProposal
err = dealProposalsV8.ForEach(&proposal, func(dealID int64) error {
// If not verified, do nothing
if !proposal.VerifiedDeal {
return nil
}
pcid, err := proposal.Cid()
if err != nil {
return err
}
isPending, err := pendingProposalsV8.Has(abi.CidKey(pcid))
if err != nil {
return xerrors.Errorf("failed to check pending: %w", err)
}
// Nothing to do for not-pending deals
if !isPending {
return nil
}
var _dealState8 market8.DealState
found, err := dealStates8.Get(uint64(dealID), &_dealState8)
if err != nil {
return xerrors.Errorf("failed to lookup deal state: %w", err)
}
// the deal has an entry in deal states, which means it's already been allocated, nothing to do
if found {
return nil
}
numPendingVerifiedDeals++
// Checks if allocation ID is in market map
allocationId, err := marketActorV9.GetAllocationIdForPendingDeal(abi.DealID(dealID))
if err != nil {
return err
}
// Checks if allocation is in verifreg
allocation, found, err := verifregActorV9.GetAllocation(proposal.Client, allocationId)
if !found {
return xerrors.Errorf("allocation %d not found for address %s", allocationId, proposal.Client)
}
if err != nil {
return err
}
err = compareProposalToAllocation(proposal, *allocation)
if err != nil {
return err
}
return nil
})
if err != nil {
return err
}
fmt.Printf("Pending Verified deals in market v8: %d\n", numPendingVerifiedDeals)
numAllocationIds, err := countAllocationIds(actorStore, marketStateV9)
if err != nil {
return err
}
fmt.Printf("Allocation IDs in market v9: %d\n", numAllocationIds)
if numAllocationIds != numPendingVerifiedDeals {
return xerrors.Errorf("number of allocation IDsf: %d did not match the number of pending verified deals: %d", numAllocationIds, numPendingVerifiedDeals)
}
numAllocations, err := countAllocations(verifregStateV9, actorStore)
if err != nil {
return err
}
fmt.Printf("Allocations in verifreg v9: %d\n", numAllocations)
if numAllocations != numPendingVerifiedDeals {
return xerrors.Errorf("number of allocations: %d did not match the number of pending verified deals: %d", numAllocations, numPendingVerifiedDeals)
}
nextAllocationId := int(verifregStateV9.NextAllocationId)
fmt.Printf("Next Allocation ID: %d\n", nextAllocationId)
if numAllocations+1 != nextAllocationId {
return xerrors.Errorf("number of allocations + 1: %d did not match the next allocation ID: %d", numAllocations+1, nextAllocationId)
}
return nil
}
func compareProposalToAllocation(prop market8.DealProposal, alloc verifreg9.Allocation) error {
if prop.PieceCID != alloc.Data {
return xerrors.Errorf("piece cid mismatch between proposal and allocation: %s, %s", prop.PieceCID, alloc.Data)
}
proposalClientID, err := address.IDFromAddress(prop.Client)
if err != nil {
return xerrors.Errorf("couldnt get ID from address")
}
if proposalClientID != uint64(alloc.Client) {
return xerrors.Errorf("client id mismatch between proposal and allocation: %v, %v", proposalClientID, alloc.Client)
}
proposalProviderID, err := address.IDFromAddress(prop.Provider)
if err != nil {
return xerrors.Errorf("couldnt get ID from address")
}
if proposalProviderID != uint64(alloc.Provider) {
return xerrors.Errorf("provider id mismatch between proposal and allocation: %v, %v", proposalProviderID, alloc.Provider)
}
if prop.PieceSize != alloc.Size {
return xerrors.Errorf("piece size mismatch between proposal and allocation: %v, %v", prop.PieceSize, alloc.Size)
}
if alloc.TermMax != 540*builtin.EpochsInDay {
return xerrors.Errorf("allocation term should be 540 days. Got %d epochs", alloc.TermMax)
}
if prop.EndEpoch-prop.StartEpoch != alloc.TermMin {
return xerrors.Errorf("allocation term mismatch between proposal and allocation: %d, %d", prop.EndEpoch-prop.StartEpoch, alloc.TermMin)
}
return nil
}
func getMarketStateV8(stateTreeV8 *state.StateTree, actorStore adt.Store) (market8.State, error) {
marketV8, err := stateTreeV8.GetActor(market.Address)
if err != nil {
return market8.State{}, err
}
var marketStateV8 market8.State
if err = actorStore.Get(actorStore.Context(), marketV8.Head, &marketStateV8); err != nil {
return market8.State{}, xerrors.Errorf("failed to get market actor state: %w", err)
}
return marketStateV8, nil
}
func getMarketStateV9(stateTreeV9 *state.StateTree, actorStore adt.Store) (market9.State, error) {
marketV9, err := stateTreeV9.GetActor(market.Address)
if err != nil {
return market9.State{}, err
}
var marketStateV9 market9.State
if err = actorStore.Get(actorStore.Context(), marketV9.Head, &marketStateV9); err != nil {
return market9.State{}, xerrors.Errorf("failed to get market actor state: %w", err)
}
return marketStateV9, nil
}
func getMarketActorV9(stateTreeV9 *state.StateTree, actorStore adt.Store) (market.State, error) {
marketV9, err := stateTreeV9.GetActor(market.Address)
if err != nil {
return nil, err
}
return market.Load(actorStore, marketV9)
}
func getVerifregActorV8(stateTreeV8 *state.StateTree, actorStore adt.Store) (verifreg.State, error) {
verifregV8, err := stateTreeV8.GetActor(verifreg.Address)
if err != nil {
return nil, err
}
return verifreg.Load(actorStore, verifregV8)
}
func getVerifregActorV9(stateTreeV9 *state.StateTree, actorStore adt.Store) (verifreg.State, error) {
verifregV9, err := stateTreeV9.GetActor(verifreg.Address)
if err != nil {
return nil, err
}
return verifreg.Load(actorStore, verifregV9)
}
func getVerifregStateV9(stateTreeV9 *state.StateTree, actorStore adt.Store) (verifreg9.State, error) {
verifregV9, err := stateTreeV9.GetActor(verifreg.Address)
if err != nil {
return verifreg9.State{}, err
}
var verifregStateV9 verifreg9.State
if err = actorStore.Get(actorStore.Context(), verifregV9.Head, &verifregStateV9); err != nil {
return verifreg9.State{}, xerrors.Errorf("failed to get verifreg actor state: %w", err)
}
return verifregStateV9, nil
}
func getDatacapActorV9(stateTreeV9 *state.StateTree, actorStore adt.Store) (datacap.State, error) {
datacapV9, err := stateTreeV9.GetActor(datacap.Address)
if err != nil {
return nil, err
}
return datacap.Load(actorStore, datacapV9)
}
func checkAllMinersUnsealedCID(stateTreeV9 *state.StateTree, store adt.Store) error {
return stateTreeV9.ForEach(func(addr address.Address, actor *types.Actor) error {
if !lbuiltin.IsStorageMinerActor(actor.Code) {
return nil // no need to check
}
err := checkMinerUnsealedCID(actor, stateTreeV9, store)
if err != nil {
fmt.Println("failure for miner ", addr)
return err
}
return nil
})
}
func checkMinerUnsealedCID(act *types.Actor, stateTreeV9 *state.StateTree, store adt.Store) error {
minerCodeCid, found := actors.GetActorCodeID(actorstypes.Version9, manifest.MinerKey)
if !found {
return xerrors.Errorf("could not find code cid for miner actor")
}
if minerCodeCid != act.Code {
return nil // no need to check
}
marketActorV9, err := getMarketActorV9(stateTreeV9, store)
if err != nil {
return err
}
dealProposals, err := marketActorV9.Proposals()
if err != nil {
return err
}
m, err := miner.Load(store, act)
if err != nil {
return err
}
err = m.ForEachPrecommittedSector(func(info miner9.SectorPreCommitOnChainInfo) error {
dealIDs := info.Info.DealIDs
if len(dealIDs) == 0 {
return nil // Nothing to check here
}
pieceCids := make([]abi.PieceInfo, len(dealIDs))
for i, dealId := range dealIDs {
dealProposal, found, err := dealProposals.Get(dealId)
if err != nil {
return err
}
if !found {
return nil
}
pieceCids[i] = abi.PieceInfo{
Size: dealProposal.PieceSize,
PieceCID: dealProposal.PieceCID,
}
}
if len(pieceCids) == 0 {
return nil
}
if info.Info.UnsealedCid == nil {
return xerrors.Errorf("nil unsealed CID for sector with deals")
}
pieceCID, err := ffi.GenerateUnsealedCID(abi.RegisteredSealProof_StackedDrg64GiBV1_1, pieceCids)
if err != nil {
return err
}
if pieceCID != *info.Info.UnsealedCid {
return xerrors.Errorf("calculated piece CID %s did not match unsealed CID in precommitted sector info: %s", pieceCID, *info.Info.UnsealedCid)
}
return nil
})
if err != nil {
return err
}
return nil
}
func countAllocations(verifregState verifreg9.State, store adt.Store) (int, error) {
var count = 0
actorToHamtMap, err := adt9.AsMap(store, verifregState.Allocations, builtin.DefaultHamtBitwidth)
if err != nil {
return 0, xerrors.Errorf("couldn't get outer map: %x", err)
}
var innerHamtCid cbg.CborCid
err = actorToHamtMap.ForEach(&innerHamtCid, func(key string) error {
innerMap, err := adt9.AsMap(store, cid.Cid(innerHamtCid), builtin.DefaultHamtBitwidth)
if err != nil {
return xerrors.Errorf("couldn't get outer map: %x", err)
}
var allocation verifreg9.Allocation
err = innerMap.ForEach(&allocation, func(key string) error {
count++
return nil
})
if err != nil {
return xerrors.Errorf("couldn't iterate over inner map: %x", err)
}
return nil
})
if err != nil {
return 0, xerrors.Errorf("couldn't iterate over outer map: %x", err)
}
return count, nil
}
func countAllocationIds(store adt.Store, marketState market9.State) (int, error) {
allocationIds, err := adt9.AsMap(store, marketState.PendingDealAllocationIds, builtin.DefaultHamtBitwidth)
if err != nil {
return 0, err
}
var numAllocationIds int
_ = allocationIds.ForEach(nil, func(key string) error {
numAllocationIds++
return nil
})
return numAllocationIds, nil
}