cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #11149 from filecoin-project/feat/nv21-migrations-optimized-tree

Browse Source 
Feat/nv21 migrations optimized tree diff algorithm
This commit is contained in:
Aayush Rajasekaran 2023-08-14 17:05:27 -04:00 committed by GitHub
commit fa0e512510
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 111 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
cmd/lotus-shed/main.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"os"
"os/signal"
"runtime/pprof"
logging "github.com/ipfs/go-log/v2"
"github.com/urfave/cli/v2"
@ -114,10 +115,31 @@ func main() {
Name: "log-level",
Value: "info",
},
&cli.StringFlag{
Name: "pprof",
Usage: "specify name of file for writing cpu profile to",
},
},
Before: func(cctx *cli.Context) error {
if prof := cctx.String("pprof"); prof != "" {
profile, err := os.Create(prof)
if err != nil {
return err
}
if err := pprof.StartCPUProfile(profile); err != nil {
return err
}
}
return logging.SetLogLevel("lotus-shed", cctx.String("log-level"))
},
After: func(cctx *cli.Context) error {
if prof := cctx.String("pprof"); prof != "" {
pprof.StopCPUProfile()
}
return nil
},
}
// terminate early on ctrl+c

35
cmd/lotus-shed/migrations.go
View File

@ -21,6 +21,7 @@ import (
"github.com/filecoin-project/go-state-types/builtin"
v10 "github.com/filecoin-project/go-state-types/builtin/v10"
v11 "github.com/filecoin-project/go-state-types/builtin/v11"
v12 "github.com/filecoin-project/go-state-types/builtin/v12"
market8 "github.com/filecoin-project/go-state-types/builtin/v8/market"
adt8 "github.com/filecoin-project/go-state-types/builtin/v8/util/adt"
v9 "github.com/filecoin-project/go-state-types/builtin/v9"
@ -243,6 +244,8 @@ func getMigrationFuncsForNetwork(nv network.Version) (UpgradeActorsFunc, PreUpgr
return filcns.UpgradeActorsV10, filcns.PreUpgradeActorsV10, checkNv18Invariants, nil
case network.Version19:
return filcns.UpgradeActorsV11, filcns.PreUpgradeActorsV11, checkNv19Invariants, nil
case network.Version21:
return filcns.UpgradeActorsV12, filcns.PreUpgradeActorsV12, checkNv21Invariants, nil
default:
return nil, nil, nil, xerrors.Errorf("migration not implemented for nv%d", nv)
}
@ -252,6 +255,38 @@ type UpgradeActorsFunc = func(context.Context, *stmgr.StateManager, stmgr.Migrat
type PreUpgradeActorsFunc = func(context.Context, *stmgr.StateManager, stmgr.MigrationCache, cid.Cid, abi.ChainEpoch, *types.TipSet) error
type CheckInvariantsFunc = func(context.Context, cid.Cid, cid.Cid, blockstore.Blockstore, abi.ChainEpoch) error
func checkNv21Invariants(ctx context.Context, oldStateRootCid cid.Cid, newStateRootCid cid.Cid, bs blockstore.Blockstore, epoch abi.ChainEpoch) error {
actorStore := store.ActorStore(ctx, bs)
startTime := time.Now()
// Load the new state root.
var newStateRoot types.StateRoot
if err := actorStore.Get(ctx, newStateRootCid, &newStateRoot); err != nil {
return xerrors.Errorf("failed to decode state root: %w", err)
}
actorCodeCids, err := actors.GetActorCodeIDs(actorstypes.Version12)
if err != nil {
return err
}
newActorTree, err := builtin.LoadTree(actorStore, newStateRoot.Actors)
if err != nil {
return err
}
messages, err := v12.CheckStateInvariants(newActorTree, epoch, actorCodeCids)
if err != nil {
return xerrors.Errorf("checking state invariants: %w", err)
}
for _, message := range messages.Messages() {
fmt.Println("got the following error: ", message)
}
fmt.Println("completed invariant checks, took ", time.Since(startTime))
return nil
}
func checkNv19Invariants(ctx context.Context, oldStateRootCid cid.Cid, newStateRootCid cid.Cid, bs blockstore.Blockstore, epoch abi.ChainEpoch) error {
actorStore := store.ActorStore(ctx, bs)

4
go.mod
View File

@ -33,7 +33,7 @@ require (
github.com/filecoin-project/dagstore v0.5.2
github.com/filecoin-project/filecoin-ffi v0.30.4-0.20220519234331-bfd1f5f9fe38
github.com/filecoin-project/go-address v1.1.0
github.com/filecoin-project/go-amt-ipld/v4 v4.0.0
github.com/filecoin-project/go-amt-ipld/v4 v4.2.0
github.com/filecoin-project/go-bitfield v0.2.4
github.com/filecoin-project/go-cbor-util v0.0.1
github.com/filecoin-project/go-commp-utils v0.1.3
@ -45,7 +45,7 @@ require (
github.com/filecoin-project/go-jsonrpc v0.3.1
github.com/filecoin-project/go-padreader v0.0.1
github.com/filecoin-project/go-paramfetch v0.0.4
github.com/filecoin-project/go-state-types v0.11.2-0.20230713122442-314a21a94fd4
github.com/filecoin-project/go-state-types v0.12.1-0.20230804024420-40b13e0840c1
github.com/filecoin-project/go-statemachine v1.0.3
github.com/filecoin-project/go-statestore v0.2.0
github.com/filecoin-project/go-storedcounter v0.1.0

9
go.sum
View File

@ -288,8 +288,9 @@ github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoC
github.com/filecoin-project/go-amt-ipld/v3 v3.0.0/go.mod h1:Qa95YNAbtoVCTSVtX38aAC1ptBnJfPma1R/zZsKmx4o=
github.com/filecoin-project/go-amt-ipld/v3 v3.1.0 h1:ZNJ9tEG5bE72vBWYiuh5bkxJVM3ViHNOmQ7qew9n6RE=
github.com/filecoin-project/go-amt-ipld/v3 v3.1.0/go.mod h1:UjM2QhDFrrjD5s1CdnkJkat4ga+LqZBZgTMniypABRo=
github.com/filecoin-project/go-amt-ipld/v4 v4.0.0 h1:XM81BJ4/6h3FV0WfFjh74cIDIgqMbJsMBLM0fIuLUUk=
github.com/filecoin-project/go-amt-ipld/v4 v4.0.0/go.mod h1:gF053YQ4BIpzTNDoEwHZas7U3oAwncDVGvOHyY8oDpE=
github.com/filecoin-project/go-amt-ipld/v4 v4.2.0 h1:DQTXQwMXxaetd+lhZGODjt5qC1WYT7tMAlYrWqI/fwI=
github.com/filecoin-project/go-amt-ipld/v4 v4.2.0/go.mod h1:0eDVF7pROvxrsxvLJx+SJZXqRaXXcEPUcgb/rG0zGU4=
github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
github.com/filecoin-project/go-bitfield v0.2.3/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
github.com/filecoin-project/go-bitfield v0.2.4 h1:uZ7MeE+XfM5lqrHJZ93OnhQKc/rveW8p9au0C68JPgk=
@ -337,8 +338,8 @@ github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psS
github.com/filecoin-project/go-state-types v0.1.6/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q=
github.com/filecoin-project/go-state-types v0.1.10/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q=
github.com/filecoin-project/go-state-types v0.11.2-0.20230712101859-8f37624fa540/go.mod h1:SyNPwTsU7I22gL2r0OAPcImvLoTVfgRwdK/Y5rR1zz8=
github.com/filecoin-project/go-state-types v0.11.2-0.20230713122442-314a21a94fd4 h1:yrq+HTk51G1n9QhdGZULWMgeYGjHNXjdJhEhbFyMlZA=
github.com/filecoin-project/go-state-types v0.11.2-0.20230713122442-314a21a94fd4/go.mod h1:SyNPwTsU7I22gL2r0OAPcImvLoTVfgRwdK/Y5rR1zz8=
github.com/filecoin-project/go-state-types v0.12.1-0.20230804024420-40b13e0840c1 h1:FBuJ7aJ6uKT67eedEpaTWTz6lDRukAuhg9fEgCzzl1w=
github.com/filecoin-project/go-state-types v0.12.1-0.20230804024420-40b13e0840c1/go.mod h1:KOBGyvCalT8uHBS7KSKOVbjsilD90bBZHgLAqrzz6gU=
github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
github.com/filecoin-project/go-statemachine v1.0.3 h1:N07o6alys+V1tNoSTi4WuuoeNC4erS/6jE74+NsgQuk=
github.com/filecoin-project/go-statemachine v1.0.3/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54=
@ -934,6 +935,7 @@ github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQs
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/koalacxr/quantile v0.0.1 h1:wAW+SQ286Erny9wOjVww96t8ws+x5Zj6AKHDULUK+o0=
@ -2044,6 +2046,7 @@ golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

46
itests/migration_test.go
View File

@ -762,3 +762,49 @@ waitForProof20:
require.Equal(t, v1proof, minerInfo.WindowPoStProofType)
}
func TestMigrationNV21(t *testing.T) {
kit.QuietMiningLogs()
nv21epoch := abi.ChainEpoch(100)
testClient, _, ens := kit.EnsembleMinimal(t, kit.MockProofs(),
kit.UpgradeSchedule(stmgr.Upgrade{
Network: network.Version20,
Height: -1,
}, stmgr.Upgrade{
Network: network.Version21,
Height: nv21epoch,
Migration: filcns.UpgradeActorsV12,
},
))
ens.InterconnectAll().BeginMining(10 * time.Millisecond)
clientApi := testClient.FullNode.(*impl.FullNodeAPI)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
testClient.WaitTillChain(ctx, kit.HeightAtLeast(nv21epoch+5))
// Now that we have upgraded, we need to verify:
// - Sector info changes executed successfully
// - Direct data onboarding correct
bs := blockstore.NewAPIBlockstore(testClient)
ctxStore := gstStore.WrapBlockStore(ctx, bs)
currTs, err := clientApi.ChainHead(ctx)
require.NoError(t, err)
newStateTree, err := state.LoadStateTree(ctxStore, currTs.Blocks()[0].ParentStateRoot)
require.NoError(t, err)
require.Equal(t, types.StateTreeVersion5, newStateTree.Version())
// start post migration checks
//todo @aayush sector info changes
//todo @zen Direct data onboarding tests
}