From 65fa75fd01fc3f781db72eed3069b0c0e51df441 Mon Sep 17 00:00:00 2001 From: zenground0 Date: Mon, 10 Apr 2023 11:39:56 -0600 Subject: [PATCH 01/13] - Update go state types - Keep current expiration defaults on creation, extension some tests - Update ffi --- chain/actors/policy/policy.go | 6 +++++ chain/actors/policy/policy.go.template | 6 +++++ chain/gen/genesis/miners.go | 2 +- cmd/lotus-miner/sectors.go | 2 +- extern/filecoin-ffi | 2 +- go.mod | 12 +++++----- go.sum | 29 +++++++++++++---------- node/config/def.go | 2 +- storage/pipeline/precommit_policy.go | 4 ++-- storage/pipeline/precommit_policy_test.go | 2 +- 10 files changed, 41 insertions(+), 26 deletions(-) diff --git a/chain/actors/policy/policy.go b/chain/actors/policy/policy.go index 4b90c46a0..ed437c5d1 100644 --- a/chain/actors/policy/policy.go +++ b/chain/actors/policy/policy.go @@ -527,6 +527,12 @@ func GetMaxSectorExpirationExtension() abi.ChainEpoch { return miner11.MaxSectorExpirationExtension } +func GetDefaultSectorExpirationExtension() abi.ChainEpoch { + // Keep default at the previous max of 1.5 years rather than the current 3.5 years. + // This way extension and pledge defaults do not subvert existing expectations. + return miner10.MaxSectorExpirationExtension +} + func GetMinSectorExpiration() abi.ChainEpoch { return miner11.MinSectorExpiration } diff --git a/chain/actors/policy/policy.go.template b/chain/actors/policy/policy.go.template index f5178500a..43a6455d0 100644 --- a/chain/actors/policy/policy.go.template +++ b/chain/actors/policy/policy.go.template @@ -223,6 +223,12 @@ func GetMaxSectorExpirationExtension() abi.ChainEpoch { return miner{{.latestVersion}}.MaxSectorExpirationExtension } +func GetDefaultSectorExpirationExtension() abi.ChainEpoch { + // Keep default at the previous max of 1.5 years rather than the current 3.5 years. + // This way extension and pledge defaults do not subvert existing expectations. + return miner10.MaxSectorExpirationExtension +} + func GetMinSectorExpiration() abi.ChainEpoch { return miner{{.latestVersion}}.MinSectorExpiration } diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index 97f5de10d..7658e05d8 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -124,7 +124,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sys vm.Syscal sectorWeight []abi.StoragePower }, len(miners)) - maxPeriods := policy.GetMaxSectorExpirationExtension() / minertypes.WPoStProvingPeriod + maxPeriods := policy.GetDefaultSectorExpirationExtension() / minertypes.WPoStProvingPeriod rawPow, qaPow := big.NewInt(0), big.NewInt(0) for i, m := range miners { // Create miner through power actor diff --git a/cmd/lotus-miner/sectors.go b/cmd/lotus-miner/sectors.go index fc5fdcef6..b9134093c 100644 --- a/cmd/lotus-miner/sectors.go +++ b/cmd/lotus-miner/sectors.go @@ -1693,7 +1693,7 @@ var sectorsCapacityCollateralCmd = &cli.Command{ return err } - pci.Expiration = policy.GetMaxSectorExpirationExtension() + h.Height() + pci.Expiration = policy.GetDefaultSectorExpirationExtension() + h.Height() } pc, err := nApi.StateMinerInitialPledgeCollateral(ctx, maddr, pci, types.EmptyTSK) diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index cec06f7f9..027c9111a 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit cec06f7f9d4c8ffd38caf4f7ac97381b453575ef +Subproject commit 027c9111a943c52d790240e72455e2c889afeed3 diff --git a/go.mod b/go.mod index 4b81f1f57..9fa893c6f 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/filecoin-project/go-legs v0.4.4 github.com/filecoin-project/go-padreader v0.0.1 github.com/filecoin-project/go-paramfetch v0.0.4 - github.com/filecoin-project/go-state-types v0.11.0-alpha-3.0.20230410150147-71c769ab29ce + github.com/filecoin-project/go-state-types v0.11.0-rc1 github.com/filecoin-project/go-statemachine v1.0.2 github.com/filecoin-project/go-statestore v0.2.0 github.com/filecoin-project/go-storedcounter v0.1.0 @@ -158,11 +158,11 @@ require ( go.uber.org/fx v1.15.0 go.uber.org/multierr v1.8.0 go.uber.org/zap v1.23.0 - golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 + golang.org/x/crypto v0.1.0 golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b - golang.org/x/net v0.0.0-20220920183852-bf014ff85ad5 + golang.org/x/net v0.1.0 golang.org/x/sync v0.0.0-20220907140024-f12130a52804 - golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 + golang.org/x/sys v0.1.0 golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 golang.org/x/tools v0.1.12 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 @@ -340,8 +340,8 @@ require ( go.uber.org/dig v1.12.0 // indirect go4.org v0.0.0-20200411211856-f5505b9728dd // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/term v0.1.0 // indirect + golang.org/x/text v0.4.0 // indirect google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4 // indirect google.golang.org/grpc v1.45.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/go.sum b/go.sum index c7a3ba9eb..0fb03491e 100644 --- a/go.sum +++ b/go.sum @@ -354,9 +354,8 @@ github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.6/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= github.com/filecoin-project/go-state-types v0.1.10/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= -github.com/filecoin-project/go-state-types v0.11.0-alpha-2.0.20230331142914-3c737db98e58/go.mod h1:aLIas+W8BWAfpLWEPUOGMPBdhcVwoCG4pIQSQk26024= -github.com/filecoin-project/go-state-types v0.11.0-alpha-3.0.20230410150147-71c769ab29ce h1:QWTON/7X/fINO9bov7LExE0+N0kmQhooFbv0ajZrDt0= -github.com/filecoin-project/go-state-types v0.11.0-alpha-3.0.20230410150147-71c769ab29ce/go.mod h1:SyNPwTsU7I22gL2r0OAPcImvLoTVfgRwdK/Y5rR1zz8= +github.com/filecoin-project/go-state-types v0.11.0-rc1 h1:UvifFHZ+S/jbshCjRHgAUuZ1qAEyeItsNywh2tsSpY4= +github.com/filecoin-project/go-state-types v0.11.0-rc1/go.mod h1:SyNPwTsU7I22gL2r0OAPcImvLoTVfgRwdK/Y5rR1zz8= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statemachine v1.0.2 h1:421SSWBk8GIoCoWYYTE/d+qCWccgmRH0uXotXRDjUbc= github.com/filecoin-project/go-statemachine v1.0.2/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54= @@ -1922,6 +1921,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zondax/hid v0.9.0/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= github.com/zondax/hid v0.9.1 h1:gQe66rtmyZ8VeGFcOpbuH3r7erYtNEAezCAYu8LdkJo= github.com/zondax/hid v0.9.1/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= @@ -2055,11 +2055,10 @@ golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM= -golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -2170,8 +2169,9 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220418201149-a630d4f3e7a2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220920183852-bf014ff85ad5 h1:KafLifaRFIuSJ5C+7CyFJOF9haxKNC1CEIDk8GX6X0k= -golang.org/x/net v0.0.0-20220920183852-bf014ff85ad5/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2297,19 +2297,21 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211209171907-798191bca915/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc= -golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2317,8 +2319,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/node/config/def.go b/node/config/def.go index 9a24449ba..cf498e875 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -137,7 +137,7 @@ func DefaultStorageMiner() *StorageMiner { // XXX snap deals wait deals slack if first PreCommitBatchSlack: Duration(3 * time.Hour), // time buffer for forceful batch submission before sectors/deals in batch would start expiring, higher value will lower the chances for message fail due to expiration - CommittedCapacitySectorLifetime: Duration(builtin.EpochDurationSeconds * uint64(policy.GetMaxSectorExpirationExtension()) * uint64(time.Second)), + CommittedCapacitySectorLifetime: Duration(builtin.EpochDurationSeconds * uint64(policy.GetDefaultSectorExpirationExtension()) * uint64(time.Second)), AggregateCommits: true, MinCommitBatch: miner5.MinAggregatedSectors, // per FIP13, we must have at least four proofs to aggregate, where 4 is the cross over point where aggregation wins out on single provecommit gas costs diff --git a/storage/pipeline/precommit_policy.go b/storage/pipeline/precommit_policy.go index 04f69a93a..a4a17ece9 100644 --- a/storage/pipeline/precommit_policy.go +++ b/storage/pipeline/precommit_policy.go @@ -112,9 +112,9 @@ func (p *BasicPreCommitPolicy) getCCSectorLifetime() (abi.ChainEpoch, error) { } var ccLifetimeEpochs = abi.ChainEpoch(uint64(c.CommittedCapacitySectorLifetime.Seconds()) / builtin.EpochDurationSeconds) - // if zero value in config, assume maximum sector extension + // if zero value in config, assume default sector extension if ccLifetimeEpochs == 0 { - ccLifetimeEpochs = policy.GetMaxSectorExpirationExtension() + ccLifetimeEpochs = policy.GetDefaultSectorExpirationExtension() } if minExpiration := abi.ChainEpoch(miner.MinSectorExpiration); ccLifetimeEpochs < minExpiration { diff --git a/storage/pipeline/precommit_policy_test.go b/storage/pipeline/precommit_policy_test.go index 9f23e58d6..76a421461 100644 --- a/storage/pipeline/precommit_policy_test.go +++ b/storage/pipeline/precommit_policy_test.go @@ -68,7 +68,7 @@ func TestBasicPolicyEmptySector(t *testing.T) { require.NoError(t, err) // as set when there are no deal pieces - expected := h + policy.GetMaxSectorExpirationExtension() - pBuffer + expected := h + policy.GetDefaultSectorExpirationExtension() - pBuffer assert.Equal(t, int(expected), int(exp)) } From 7e99c62f6e825037c9993c2a031e62daaa3368d3 Mon Sep 17 00:00:00 2001 From: zenground0 Date: Mon, 10 Apr 2023 18:18:56 -0600 Subject: [PATCH 02/13] ffi experiment --- extern/filecoin-ffi | 2 +- go.sum | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 027c9111a..cec06f7f9 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 027c9111a943c52d790240e72455e2c889afeed3 +Subproject commit cec06f7f9d4c8ffd38caf4f7ac97381b453575ef diff --git a/go.sum b/go.sum index 0fb03491e..e48010660 100644 --- a/go.sum +++ b/go.sum @@ -354,6 +354,7 @@ github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.6/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= github.com/filecoin-project/go-state-types v0.1.10/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= +github.com/filecoin-project/go-state-types v0.11.0-alpha-2.0.20230331142914-3c737db98e58/go.mod h1:aLIas+W8BWAfpLWEPUOGMPBdhcVwoCG4pIQSQk26024= github.com/filecoin-project/go-state-types v0.11.0-rc1 h1:UvifFHZ+S/jbshCjRHgAUuZ1qAEyeItsNywh2tsSpY4= github.com/filecoin-project/go-state-types v0.11.0-rc1/go.mod h1:SyNPwTsU7I22gL2r0OAPcImvLoTVfgRwdK/Y5rR1zz8= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= @@ -1921,7 +1922,6 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zondax/hid v0.9.0/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= github.com/zondax/hid v0.9.1 h1:gQe66rtmyZ8VeGFcOpbuH3r7erYtNEAezCAYu8LdkJo= github.com/zondax/hid v0.9.1/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= @@ -2055,6 +2055,7 @@ golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= @@ -2169,7 +2170,6 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220418201149-a630d4f3e7a2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2297,12 +2297,11 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211209171907-798191bca915/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From 4d7e6732fa1c86dc6eae8f8cfad50fbc67b5f99b Mon Sep 17 00:00:00 2001 From: zenground0 Date: Thu, 30 Mar 2023 11:28:33 -0600 Subject: [PATCH 03/13] Integration nv19 migration - Open splitstore in migration shed tool - Update state root version --- chain/consensus/filcns/upgrades.go | 150 ++++++++++++++++++++++++----- cmd/lotus-shed/migrations.go | 61 +++++++++--- go.mod | 2 +- go.sum | 10 -- 4 files changed, 177 insertions(+), 46 deletions(-) diff --git a/chain/consensus/filcns/upgrades.go b/chain/consensus/filcns/upgrades.go index 4c5f3bd30..5cf7c4746 100644 --- a/chain/consensus/filcns/upgrades.go +++ b/chain/consensus/filcns/upgrades.go @@ -1597,29 +1597,6 @@ func upgradeActorsV9Common( return newRoot, nil } -func UpgradeActorsV10(ctx context.Context, sm *stmgr.StateManager, cache stmgr.MigrationCache, cb stmgr.ExecMonitor, - root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { - // Use all the CPUs except 3. - workerCount := MigrationMaxWorkerCount - 3 - if workerCount <= 0 { - workerCount = 1 - } - - config := migration.Config{ - MaxWorkers: uint(workerCount), - JobQueueSize: 1000, - ResultQueueSize: 100, - ProgressLogPeriod: 10 * time.Second, - } - - newRoot, err := upgradeActorsV10Common(ctx, sm, cache, root, epoch, ts, config) - if err != nil { - return cid.Undef, xerrors.Errorf("migrating actors v10 state: %w", err) - } - - return newRoot, nil -} - func PreUpgradeActorsV10(ctx context.Context, sm *stmgr.StateManager, cache stmgr.MigrationCache, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) error { // Use half the CPUs for pre-migration, but leave at least 3. workerCount := MigrationMaxWorkerCount @@ -1643,6 +1620,29 @@ func PreUpgradeActorsV10(ctx context.Context, sm *stmgr.StateManager, cache stmg return err } +func UpgradeActorsV10(ctx context.Context, sm *stmgr.StateManager, cache stmgr.MigrationCache, cb stmgr.ExecMonitor, + root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { + // Use all the CPUs except 3. + workerCount := MigrationMaxWorkerCount - 3 + if workerCount <= 0 { + workerCount = 1 + } + + config := migration.Config{ + MaxWorkers: uint(workerCount), + JobQueueSize: 1000, + ResultQueueSize: 100, + ProgressLogPeriod: 10 * time.Second, + } + + newRoot, err := upgradeActorsV10Common(ctx, sm, cache, root, epoch, ts, config) + if err != nil { + return cid.Undef, xerrors.Errorf("migrating actors v10 state: %w", err) + } + + return newRoot, nil +} + func upgradeActorsV10Common( ctx context.Context, sm *stmgr.StateManager, cache stmgr.MigrationCache, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet, @@ -1664,7 +1664,7 @@ func upgradeActorsV10Common( if stateRoot.Version != types.StateTreeVersion4 { return cid.Undef, xerrors.Errorf( - "expected state root version 4 for actors v10 upgrade, got %d", + "expected state root version 4 for actors v9 upgrade, got %d", stateRoot.Version, ) } @@ -1748,6 +1748,108 @@ func UpgradeActorsV11(ctx context.Context, sm *stmgr.StateManager, cache stmgr.M return newRoot, nil } +func upgradeActorsV11Common( + ctx context.Context, sm *stmgr.StateManager, cache stmgr.MigrationCache, + root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet, + config migration.Config, +) (cid.Cid, error) { + writeStore := blockstore.NewAutobatch(ctx, sm.ChainStore().StateBlockstore(), units.GiB/4) + store := store.ActorStore(ctx, writeStore) + // ensure that the manifest is loaded in the blockstore + if err := bundle.LoadBundles(ctx, sm.ChainStore().StateBlockstore(), actorstypes.Version11); err != nil { + return cid.Undef, xerrors.Errorf("failed to load manifest bundle: %w", err) + } + + // Load the state root. + var stateRoot types.StateRoot + if err := store.Get(ctx, root, &stateRoot); err != nil { + return cid.Undef, xerrors.Errorf("failed to decode state root: %w", err) + } + + if stateRoot.Version != types.StateTreeVersion5 { + return cid.Undef, xerrors.Errorf( + "expected state root version 4 for actors v10 upgrade, got %d", + stateRoot.Version, + ) + } + + manifest, ok := actors.GetManifest(actorstypes.Version10) + if !ok { + return cid.Undef, xerrors.Errorf("no manifest CID for v11 upgrade") + } + + // Perform the migration + newHamtRoot, err := nv19.MigrateStateTree(ctx, store, manifest, stateRoot.Actors, epoch, config, + migrationLogger{}, cache) + if err != nil { + return cid.Undef, xerrors.Errorf("upgrading to actors v10: %w", err) + } + + // Persist the result. + newRoot, err := store.Put(ctx, &types.StateRoot{ + Version: types.StateTreeVersion5, + Actors: newHamtRoot, + Info: stateRoot.Info, + }) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to persist new state root: %w", err) + } + + // Persists the new tree and shuts down the flush worker + if err := writeStore.Flush(ctx); err != nil { + return cid.Undef, xerrors.Errorf("writeStore flush failed: %w", err) + } + + if err := writeStore.Shutdown(ctx); err != nil { + return cid.Undef, xerrors.Errorf("writeStore shutdown failed: %w", err) + } + + return newRoot, nil +} + +func PreUpgradeActorsV11(ctx context.Context, sm *stmgr.StateManager, cache stmgr.MigrationCache, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) error { + // Use half the CPUs for pre-migration, but leave at least 3. + workerCount := MigrationMaxWorkerCount + if workerCount <= 4 { + workerCount = 1 + } else { + workerCount /= 2 + } + + lbts, lbRoot, err := stmgr.GetLookbackTipSetForRound(ctx, sm, ts, epoch) + if err != nil { + return xerrors.Errorf("error getting lookback ts for premigration: %w", err) + } + + config := migration.Config{ + MaxWorkers: uint(workerCount), + ProgressLogPeriod: time.Minute * 5, + } + + _, err = upgradeActorsV11Common(ctx, sm, cache, lbRoot, epoch, lbts, config) + return err +} + +func UpgradeActorsV11(ctx context.Context, sm *stmgr.StateManager, cache stmgr.MigrationCache, cb stmgr.ExecMonitor, + root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { + // Use all the CPUs except 2. + workerCount := MigrationMaxWorkerCount - 3 + if workerCount <= 0 { + workerCount = 1 + } + config := migration.Config{ + MaxWorkers: uint(workerCount), + JobQueueSize: 1000, + ResultQueueSize: 100, + ProgressLogPeriod: 10 * time.Second, + } + newRoot, err := upgradeActorsV11Common(ctx, sm, cache, root, epoch, ts, config) + if err != nil { + return cid.Undef, xerrors.Errorf("migrating actors v11 state: %w", err) + } + return newRoot, nil +} + func upgradeActorsV11Common( ctx context.Context, sm *stmgr.StateManager, cache stmgr.MigrationCache, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet, diff --git a/cmd/lotus-shed/migrations.go b/cmd/lotus-shed/migrations.go index 8a1007882..f804404cb 100644 --- a/cmd/lotus-shed/migrations.go +++ b/cmd/lotus-shed/migrations.go @@ -3,7 +3,8 @@ package main import ( "context" "fmt" - "io" + "os" + "path/filepath" "strconv" "time" @@ -28,9 +29,11 @@ import ( "github.com/filecoin-project/go-state-types/manifest" mutil "github.com/filecoin-project/go-state-types/migration" "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/blockstore/splitstore" "github.com/filecoin-project/specs-actors/v7/actors/migration/nv15" "github.com/filecoin-project/lotus/blockstore" + badgerbs "github.com/filecoin-project/lotus/blockstore/badger" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/adt" lbuiltin "github.com/filecoin-project/lotus/chain/actors/builtin" @@ -99,24 +102,52 @@ var migrationsCmd = &cli.Command{ defer lkrepo.Close() //nolint:errcheck - bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) + cold, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { - return fmt.Errorf("failed to open blockstore: %w", err) + return fmt.Errorf("failed to open universal blockstore %w", err) } - defer func() { - if c, ok := bs.(io.Closer); ok { - if err := c.Close(); err != nil { - log.Warnf("failed to close blockstore: %s", err) - } - } - }() + path, err := lkrepo.SplitstorePath() + if err != nil { + return err + } + + path = filepath.Join(path, "hot.badger") + if err := os.MkdirAll(path, 0755); err != nil { + return err + } + + opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, lkrepo.Readonly()) + if err != nil { + return err + } + + hot, err := badgerbs.Open(opts) + if err != nil { + return err + } mds, err := lkrepo.Datastore(context.Background(), "/metadata") if err != nil { return err } + cfg := &splitstore.Config{ + MarkSetType: "map", + DiscardColdBlocks: true, + } + ss, err := splitstore.Open(path, mds, hot, cold, cfg) + if err != nil { + return err + } + defer func() { + if err := ss.Close(); err != nil { + log.Warnf("failed to close blockstore: %s", err) + + } + }() + bs := ss + cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil) defer cs.Close() //nolint:errcheck @@ -165,6 +196,7 @@ var migrationsCmd = &cli.Command{ } preMigrationTime := time.Since(startTime) + fmt.Println("completed premigration, took ", preMigrationTime) startTime = time.Now() @@ -179,7 +211,7 @@ var migrationsCmd = &cli.Command{ return xerrors.Errorf("got different results with and without the cache: %s, %s", newCid1, newCid2) } - fmt.Println("completed premigration, took ", preMigrationTime) + fmt.Println("completed round actual (with cache), took ", cachedMigrationTime) } @@ -203,6 +235,8 @@ func getMigrationFuncsForNetwork(nv network.Version) (UpgradeActorsFunc, PreUpgr return filcns.UpgradeActorsV9, filcns.PreUpgradeActorsV9, checkNv17Invariants, nil case network.Version18: return filcns.UpgradeActorsV10, filcns.PreUpgradeActorsV10, checkNv18Invariants, nil + case network.Version19: + return filcns.UpgradeActorsV11, filcns.PreUpgradeActorsV11, checkNv19Invariants, nil default: return nil, nil, nil, xerrors.Errorf("migration not implemented for nv%d", nv) } @@ -212,6 +246,11 @@ type UpgradeActorsFunc = func(context.Context, *stmgr.StateManager, stmgr.Migrat type PreUpgradeActorsFunc = func(context.Context, *stmgr.StateManager, stmgr.MigrationCache, cid.Cid, abi.ChainEpoch, *types.TipSet) error type CheckInvariantsFunc = func(context.Context, cid.Cid, cid.Cid, blockstore.Blockstore, abi.ChainEpoch) error +func checkNv19Invariants(context.Context, cid.Cid, cid.Cid, blockstore.Blockstore, abi.ChainEpoch) error { + fmt.Printf("noop -- nv19 shed invariant check not wired up yet") + return nil +} + func checkNv18Invariants(ctx context.Context, oldStateRootCid cid.Cid, newStateRootCid cid.Cid, bs blockstore.Blockstore, epoch abi.ChainEpoch) error { actorStore := store.ActorStore(ctx, bs) startTime := time.Now() diff --git a/go.mod b/go.mod index 9fa893c6f..b07e58029 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/filecoin-project/go-padreader v0.0.1 github.com/filecoin-project/go-paramfetch v0.0.4 github.com/filecoin-project/go-state-types v0.11.0-rc1 - github.com/filecoin-project/go-statemachine v1.0.2 + github.com/filecoin-project/go-statemachine v1.0.3 github.com/filecoin-project/go-statestore v0.2.0 github.com/filecoin-project/go-storedcounter v0.1.0 github.com/filecoin-project/index-provider v0.9.1 diff --git a/go.sum b/go.sum index e48010660..509c39417 100644 --- a/go.sum +++ b/go.sum @@ -354,7 +354,6 @@ github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.6/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= github.com/filecoin-project/go-state-types v0.1.10/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= -github.com/filecoin-project/go-state-types v0.11.0-alpha-2.0.20230331142914-3c737db98e58/go.mod h1:aLIas+W8BWAfpLWEPUOGMPBdhcVwoCG4pIQSQk26024= github.com/filecoin-project/go-state-types v0.11.0-rc1 h1:UvifFHZ+S/jbshCjRHgAUuZ1qAEyeItsNywh2tsSpY4= github.com/filecoin-project/go-state-types v0.11.0-rc1/go.mod h1:SyNPwTsU7I22gL2r0OAPcImvLoTVfgRwdK/Y5rR1zz8= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= @@ -401,7 +400,6 @@ github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVB github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= -github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -544,7 +542,6 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= @@ -709,8 +706,6 @@ github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqg github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro= -github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= -github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= github.com/ipfs/go-cidutil v0.1.0 h1:RW5hO7Vcf16dplUU60Hs0AKDkQAVPVplr7lk97CFL+Q= github.com/ipfs/go-cidutil v0.1.0/go.mod h1:e7OEVBMIv9JaOxt9zaGEmAoSlXW9jdFZ5lP/0PwcfpA= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= @@ -900,8 +895,6 @@ github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvB github.com/ipld/go-ipld-prime v0.10.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= -github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= -github.com/ipld/go-ipld-prime v0.17.0/go.mod h1:aYcKm5TIvGfY8P3QBKz/2gKcLxzJ1zDaD+o0bOowhgs= github.com/ipld/go-ipld-prime v0.19.0/go.mod h1:Q9j3BaVXwaA3o5JUDNvptDDr/x8+F7FG6XJ8WI3ILg4= github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g= github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= @@ -1524,9 +1517,6 @@ github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyD github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= -github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= -github.com/multiformats/go-multicodec v0.4.1/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= -github.com/multiformats/go-multicodec v0.5.0/go.mod h1:DiY2HFaEp5EhEXb/iYzVAunmyX/aSFMxq2KMKfWEues= github.com/multiformats/go-multicodec v0.6.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= github.com/multiformats/go-multicodec v0.8.0 h1:evBmgkbSQux+Ds2IgfhkO38Dl2GDtRW8/Rp6YiSHX/Q= github.com/multiformats/go-multicodec v0.8.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= From 86c545325434100c3b027d96e40dab9bafb7fcaa Mon Sep 17 00:00:00 2001 From: zenground0 Date: Mon, 10 Apr 2023 22:18:56 -0600 Subject: [PATCH 04/13] Post rebase fixup --- go.sum | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 509c39417..6bc69c5b3 100644 --- a/go.sum +++ b/go.sum @@ -354,11 +354,12 @@ github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.6/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= github.com/filecoin-project/go-state-types v0.1.10/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= +github.com/filecoin-project/go-state-types v0.11.0-alpha-2.0.20230331142914-3c737db98e58/go.mod h1:aLIas+W8BWAfpLWEPUOGMPBdhcVwoCG4pIQSQk26024= github.com/filecoin-project/go-state-types v0.11.0-rc1 h1:UvifFHZ+S/jbshCjRHgAUuZ1qAEyeItsNywh2tsSpY4= github.com/filecoin-project/go-state-types v0.11.0-rc1/go.mod h1:SyNPwTsU7I22gL2r0OAPcImvLoTVfgRwdK/Y5rR1zz8= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= -github.com/filecoin-project/go-statemachine v1.0.2 h1:421SSWBk8GIoCoWYYTE/d+qCWccgmRH0uXotXRDjUbc= -github.com/filecoin-project/go-statemachine v1.0.2/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54= +github.com/filecoin-project/go-statemachine v1.0.3 h1:N07o6alys+V1tNoSTi4WuuoeNC4erS/6jE74+NsgQuk= +github.com/filecoin-project/go-statemachine v1.0.3/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54= github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-statestore v0.2.0 h1:cRRO0aPLrxKQCZ2UOQbzFGn4WDNdofHZoGPjfNaAo5Q= github.com/filecoin-project/go-statestore v0.2.0/go.mod h1:8sjBYbS35HwPzct7iT4lIXjLlYyPor80aU7t7a/Kspo= @@ -400,6 +401,7 @@ github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVB github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= +github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -542,6 +544,7 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= @@ -706,6 +709,8 @@ github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqg github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro= +github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= +github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= github.com/ipfs/go-cidutil v0.1.0 h1:RW5hO7Vcf16dplUU60Hs0AKDkQAVPVplr7lk97CFL+Q= github.com/ipfs/go-cidutil v0.1.0/go.mod h1:e7OEVBMIv9JaOxt9zaGEmAoSlXW9jdFZ5lP/0PwcfpA= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= @@ -895,6 +900,8 @@ github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvB github.com/ipld/go-ipld-prime v0.10.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= +github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= +github.com/ipld/go-ipld-prime v0.17.0/go.mod h1:aYcKm5TIvGfY8P3QBKz/2gKcLxzJ1zDaD+o0bOowhgs= github.com/ipld/go-ipld-prime v0.19.0/go.mod h1:Q9j3BaVXwaA3o5JUDNvptDDr/x8+F7FG6XJ8WI3ILg4= github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g= github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= @@ -1517,6 +1524,9 @@ github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyD github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= +github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= +github.com/multiformats/go-multicodec v0.4.1/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= +github.com/multiformats/go-multicodec v0.5.0/go.mod h1:DiY2HFaEp5EhEXb/iYzVAunmyX/aSFMxq2KMKfWEues= github.com/multiformats/go-multicodec v0.6.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= github.com/multiformats/go-multicodec v0.8.0 h1:evBmgkbSQux+Ds2IgfhkO38Dl2GDtRW8/Rp6YiSHX/Q= github.com/multiformats/go-multicodec v0.8.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= From 9d0f586ae7c805b845475b96b47d1e067dbc970c Mon Sep 17 00:00:00 2001 From: zenground0 Date: Mon, 10 Apr 2023 22:25:25 -0600 Subject: [PATCH 05/13] Fix --- chain/consensus/filcns/upgrades.go | 105 ----------------------------- 1 file changed, 105 deletions(-) diff --git a/chain/consensus/filcns/upgrades.go b/chain/consensus/filcns/upgrades.go index 5cf7c4746..260bac8dd 100644 --- a/chain/consensus/filcns/upgrades.go +++ b/chain/consensus/filcns/upgrades.go @@ -1807,111 +1807,6 @@ func upgradeActorsV11Common( return newRoot, nil } -func PreUpgradeActorsV11(ctx context.Context, sm *stmgr.StateManager, cache stmgr.MigrationCache, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) error { - // Use half the CPUs for pre-migration, but leave at least 3. - workerCount := MigrationMaxWorkerCount - if workerCount <= 4 { - workerCount = 1 - } else { - workerCount /= 2 - } - - lbts, lbRoot, err := stmgr.GetLookbackTipSetForRound(ctx, sm, ts, epoch) - if err != nil { - return xerrors.Errorf("error getting lookback ts for premigration: %w", err) - } - - config := migration.Config{ - MaxWorkers: uint(workerCount), - ProgressLogPeriod: time.Minute * 5, - } - - _, err = upgradeActorsV11Common(ctx, sm, cache, lbRoot, epoch, lbts, config) - return err -} - -func UpgradeActorsV11(ctx context.Context, sm *stmgr.StateManager, cache stmgr.MigrationCache, cb stmgr.ExecMonitor, - root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { - // Use all the CPUs except 2. - workerCount := MigrationMaxWorkerCount - 3 - if workerCount <= 0 { - workerCount = 1 - } - config := migration.Config{ - MaxWorkers: uint(workerCount), - JobQueueSize: 1000, - ResultQueueSize: 100, - ProgressLogPeriod: 10 * time.Second, - } - newRoot, err := upgradeActorsV11Common(ctx, sm, cache, root, epoch, ts, config) - if err != nil { - return cid.Undef, xerrors.Errorf("migrating actors v11 state: %w", err) - } - return newRoot, nil -} - -func upgradeActorsV11Common( - ctx context.Context, sm *stmgr.StateManager, cache stmgr.MigrationCache, - root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet, - config migration.Config, -) (cid.Cid, error) { - buf := blockstore.NewTieredBstore(sm.ChainStore().StateBlockstore(), blockstore.NewMemorySync()) - store := store.ActorStore(ctx, buf) - - // ensure that the manifest is loaded in the blockstore - if err := bundle.LoadBundles(ctx, sm.ChainStore().StateBlockstore(), actorstypes.Version11); err != nil { - return cid.Undef, xerrors.Errorf("failed to load manifest bundle: %w", err) - } - - // Load the state root. - var stateRoot types.StateRoot - if err := store.Get(ctx, root, &stateRoot); err != nil { - return cid.Undef, xerrors.Errorf("failed to decode state root: %w", err) - } - - if stateRoot.Version != types.StateTreeVersion5 { - return cid.Undef, xerrors.Errorf( - "expected state root version 5 for actors v11 upgrade, got %d", - stateRoot.Version, - ) - } - - manifest, ok := actors.GetManifest(actorstypes.Version11) - if !ok { - return cid.Undef, xerrors.Errorf("no manifest CID for v11 upgrade") - } - - // Perform the migration - newHamtRoot, err := nv19.MigrateStateTree(ctx, store, manifest, stateRoot.Actors, epoch, config, - migrationLogger{}, cache) - if err != nil { - return cid.Undef, xerrors.Errorf("upgrading to actors v11: %w", err) - } - - // Persist the result. - newRoot, err := store.Put(ctx, &types.StateRoot{ - Version: types.StateTreeVersion5, - Actors: newHamtRoot, - Info: stateRoot.Info, - }) - if err != nil { - return cid.Undef, xerrors.Errorf("failed to persist new state root: %w", err) - } - - // Persist the new tree. - - { - from := buf - to := buf.Read() - - if err := vm.Copy(ctx, from, to, newRoot); err != nil { - return cid.Undef, xerrors.Errorf("copying migrated tree: %w", err) - } - } - - return newRoot, nil -} - // Example upgrade function if upgrade requires only code changes //func UpgradeActorsV9(ctx context.Context, sm *stmgr.StateManager, _ stmgr.MigrationCache, _ stmgr.ExecMonitor, root cid.Cid, _ abi.ChainEpoch, _ *types.TipSet) (cid.Cid, error) { // buf := blockstore.NewTieredBstore(sm.ChainStore().StateBlockstore(), blockstore.NewMemorySync()) From 1b8d627af8550f94e6614f1ff88eec8545b8be4e Mon Sep 17 00:00:00 2001 From: zenground0 Date: Mon, 10 Apr 2023 22:40:06 -0600 Subject: [PATCH 06/13] gen --- cmd/lotus-shed/migrations.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-shed/migrations.go b/cmd/lotus-shed/migrations.go index f804404cb..d26818ac4 100644 --- a/cmd/lotus-shed/migrations.go +++ b/cmd/lotus-shed/migrations.go @@ -29,11 +29,11 @@ import ( "github.com/filecoin-project/go-state-types/manifest" mutil "github.com/filecoin-project/go-state-types/migration" "github.com/filecoin-project/go-state-types/network" - "github.com/filecoin-project/lotus/blockstore/splitstore" "github.com/filecoin-project/specs-actors/v7/actors/migration/nv15" "github.com/filecoin-project/lotus/blockstore" badgerbs "github.com/filecoin-project/lotus/blockstore/badger" + "github.com/filecoin-project/lotus/blockstore/splitstore" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/adt" lbuiltin "github.com/filecoin-project/lotus/chain/actors/builtin" From 63560c820b9ceebd3cdae624f3a3bea93716b765 Mon Sep 17 00:00:00 2001 From: zenground0 Date: Mon, 10 Apr 2023 23:01:18 -0600 Subject: [PATCH 07/13] nv19 invariant checking --- cmd/lotus-shed/migrations.go | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-shed/migrations.go b/cmd/lotus-shed/migrations.go index d26818ac4..9f0d92101 100644 --- a/cmd/lotus-shed/migrations.go +++ b/cmd/lotus-shed/migrations.go @@ -19,6 +19,7 @@ import ( actorstypes "github.com/filecoin-project/go-state-types/actors" "github.com/filecoin-project/go-state-types/builtin" v10 "github.com/filecoin-project/go-state-types/builtin/v10" + v11 "github.com/filecoin-project/go-state-types/builtin/v11" market8 "github.com/filecoin-project/go-state-types/builtin/v8/market" adt8 "github.com/filecoin-project/go-state-types/builtin/v8/util/adt" v9 "github.com/filecoin-project/go-state-types/builtin/v9" @@ -246,8 +247,36 @@ type UpgradeActorsFunc = func(context.Context, *stmgr.StateManager, stmgr.Migrat type PreUpgradeActorsFunc = func(context.Context, *stmgr.StateManager, stmgr.MigrationCache, cid.Cid, abi.ChainEpoch, *types.TipSet) error type CheckInvariantsFunc = func(context.Context, cid.Cid, cid.Cid, blockstore.Blockstore, abi.ChainEpoch) error -func checkNv19Invariants(context.Context, cid.Cid, cid.Cid, blockstore.Blockstore, abi.ChainEpoch) error { - fmt.Printf("noop -- nv19 shed invariant check not wired up yet") +func checkNv19Invariants(ctx context.Context, oldStateRootCid cid.Cid, newStateRootCid cid.Cid, bs blockstore.Blockstore, epoch abi.ChainEpoch) error { + + actorStore := store.ActorStore(ctx, bs) + startTime := time.Now() + + // Load the new state root. + var newStateRoot types.StateRoot + if err := actorStore.Get(ctx, newStateRootCid, &newStateRoot); err != nil { + return xerrors.Errorf("failed to decode state root: %w", err) + } + + actorCodeCids, err := actors.GetActorCodeIDs(actorstypes.Version10) + if err != nil { + return err + } + newActorTree, err := builtin.LoadTree(actorStore, newStateRoot.Actors) + if err != nil { + return err + } + messages, err := v11.CheckStateInvariants(newActorTree, epoch, actorCodeCids) + if err != nil { + return xerrors.Errorf("checking state invariants: %w", err) + } + + for _, message := range messages.Messages() { + fmt.Println("got the following error: ", message) + } + + fmt.Println("completed invariant checks, took ", time.Since(startTime)) + return nil } From c00e0c9f48701548e6a38834a8ef16d6a59c709f Mon Sep 17 00:00:00 2001 From: zenground0 Date: Mon, 10 Apr 2023 23:01:36 -0600 Subject: [PATCH 08/13] Try fixig blockstore so bundle is loaded --- chain/consensus/filcns/upgrades.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/chain/consensus/filcns/upgrades.go b/chain/consensus/filcns/upgrades.go index 260bac8dd..26d77145a 100644 --- a/chain/consensus/filcns/upgrades.go +++ b/chain/consensus/filcns/upgrades.go @@ -1754,15 +1754,15 @@ func upgradeActorsV11Common( config migration.Config, ) (cid.Cid, error) { writeStore := blockstore.NewAutobatch(ctx, sm.ChainStore().StateBlockstore(), units.GiB/4) - store := store.ActorStore(ctx, writeStore) + adtStore := store.ActorStore(ctx, writeStore) // ensure that the manifest is loaded in the blockstore - if err := bundle.LoadBundles(ctx, sm.ChainStore().StateBlockstore(), actorstypes.Version11); err != nil { + if err := bundle.LoadBundles(ctx, writeStore, actorstypes.Version11); err != nil { return cid.Undef, xerrors.Errorf("failed to load manifest bundle: %w", err) } // Load the state root. var stateRoot types.StateRoot - if err := store.Get(ctx, root, &stateRoot); err != nil { + if err := adtStore.Get(ctx, root, &stateRoot); err != nil { return cid.Undef, xerrors.Errorf("failed to decode state root: %w", err) } @@ -1779,14 +1779,14 @@ func upgradeActorsV11Common( } // Perform the migration - newHamtRoot, err := nv19.MigrateStateTree(ctx, store, manifest, stateRoot.Actors, epoch, config, + newHamtRoot, err := nv19.MigrateStateTree(ctx, adtStore, manifest, stateRoot.Actors, epoch, config, migrationLogger{}, cache) if err != nil { return cid.Undef, xerrors.Errorf("upgrading to actors v10: %w", err) } // Persist the result. - newRoot, err := store.Put(ctx, &types.StateRoot{ + newRoot, err := adtStore.Put(ctx, &types.StateRoot{ Version: types.StateTreeVersion5, Actors: newHamtRoot, Info: stateRoot.Info, From db374db3063269e49b66be835e3e0bf3f1d69def Mon Sep 17 00:00:00 2001 From: zenground0 Date: Mon, 10 Apr 2023 23:14:25 -0600 Subject: [PATCH 09/13] Debug --- node/bundle/bundle.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/node/bundle/bundle.go b/node/bundle/bundle.go index abdb34e69..a55cad9f1 100644 --- a/node/bundle/bundle.go +++ b/node/bundle/bundle.go @@ -3,6 +3,7 @@ package bundle import ( "bytes" "context" + "fmt" "io" "os" @@ -53,6 +54,7 @@ func LoadBundles(ctx context.Context, bs blockstore.Blockstore, versions ...acto // All manifests are registered on start, so this must succeed. return xerrors.Errorf("unknown actor version v%d", av) } + fmt.Printf("manifest cid: %s\n", manifestCid) if haveManifest, err := bs.Has(ctx, manifestCid); err != nil { return xerrors.Errorf("blockstore error when loading manifest %s: %w", manifestCid, err) From bb5f69328d61514aaa6efafbea8cd772ab8df7fd Mon Sep 17 00:00:00 2001 From: zenground0 Date: Mon, 10 Apr 2023 23:18:23 -0600 Subject: [PATCH 10/13] Fix --- chain/consensus/filcns/upgrades.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/consensus/filcns/upgrades.go b/chain/consensus/filcns/upgrades.go index 26d77145a..0e68c6b12 100644 --- a/chain/consensus/filcns/upgrades.go +++ b/chain/consensus/filcns/upgrades.go @@ -1773,7 +1773,7 @@ func upgradeActorsV11Common( ) } - manifest, ok := actors.GetManifest(actorstypes.Version10) + manifest, ok := actors.GetManifest(actorstypes.Version11) if !ok { return cid.Undef, xerrors.Errorf("no manifest CID for v11 upgrade") } From e75c2e42083b3ff741aed35419da2593face73fe Mon Sep 17 00:00:00 2001 From: zenground0 Date: Mon, 10 Apr 2023 23:32:03 -0600 Subject: [PATCH 11/13] Make butterfly upgrades happen --- build/params_butterfly.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/build/params_butterfly.go b/build/params_butterfly.go index eb7903692..63ab9a706 100644 --- a/build/params_butterfly.go +++ b/build/params_butterfly.go @@ -51,13 +51,11 @@ const UpgradeChocolateHeight = -17 const UpgradeOhSnapHeight = -18 const UpgradeSkyrHeight = -19 const UpgradeSharkHeight = abi.ChainEpoch(-20) -const UpgradeHyggeHeight = abi.ChainEpoch(600) +const UpgradeHyggeHeight = abi.ChainEpoch(-21) -// ?????????? -const UpgradeLightningHeight = 999999999999999 +const UpgradeLightningHeight = 600 -// ?????????????????? -const UpgradeThunderHeight = UpgradeLightningHeight + 1 +const UpgradeThunderHeight = UpgradeLightningHeight + 1440 var SupportedProofTypes = []abi.RegisteredSealProof{ abi.RegisteredSealProof_StackedDrg512MiBV1, From 8f4824d2b790155cbebac0e549caf828a8f415c2 Mon Sep 17 00:00:00 2001 From: zenground0 Date: Tue, 11 Apr 2023 09:08:24 -0600 Subject: [PATCH 12/13] Another ffi experiment --- extern/filecoin-ffi | 2 +- go.sum | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index cec06f7f9..cec06a79d 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit cec06f7f9d4c8ffd38caf4f7ac97381b453575ef +Subproject commit cec06a79dc858f221f6542cff264b92b4f99c25d diff --git a/go.sum b/go.sum index 6bc69c5b3..8cbbf7e54 100644 --- a/go.sum +++ b/go.sum @@ -354,7 +354,6 @@ github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.6/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= github.com/filecoin-project/go-state-types v0.1.10/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= -github.com/filecoin-project/go-state-types v0.11.0-alpha-2.0.20230331142914-3c737db98e58/go.mod h1:aLIas+W8BWAfpLWEPUOGMPBdhcVwoCG4pIQSQk26024= github.com/filecoin-project/go-state-types v0.11.0-rc1 h1:UvifFHZ+S/jbshCjRHgAUuZ1qAEyeItsNywh2tsSpY4= github.com/filecoin-project/go-state-types v0.11.0-rc1/go.mod h1:SyNPwTsU7I22gL2r0OAPcImvLoTVfgRwdK/Y5rR1zz8= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= @@ -1922,6 +1921,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zondax/hid v0.9.0/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= github.com/zondax/hid v0.9.1 h1:gQe66rtmyZ8VeGFcOpbuH3r7erYtNEAezCAYu8LdkJo= github.com/zondax/hid v0.9.1/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= @@ -2055,7 +2055,6 @@ golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= @@ -2170,6 +2169,7 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220418201149-a630d4f3e7a2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2297,11 +2297,12 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211209171907-798191bca915/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From d91540ae1fbe1ec3535a9e1a4e0a8fdb0ecbe84b Mon Sep 17 00:00:00 2001 From: zenground0 Date: Tue, 11 Apr 2023 09:14:18 -0600 Subject: [PATCH 13/13] Fix copy paste error --- cmd/lotus-shed/migrations.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-shed/migrations.go b/cmd/lotus-shed/migrations.go index 9f0d92101..e9e726659 100644 --- a/cmd/lotus-shed/migrations.go +++ b/cmd/lotus-shed/migrations.go @@ -258,7 +258,7 @@ func checkNv19Invariants(ctx context.Context, oldStateRootCid cid.Cid, newStateR return xerrors.Errorf("failed to decode state root: %w", err) } - actorCodeCids, err := actors.GetActorCodeIDs(actorstypes.Version10) + actorCodeCids, err := actors.GetActorCodeIDs(actorstypes.Version11) if err != nil { return err }