From 6db37b72a85bf056630d757f591d11c92f8c65c9 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 24 Sep 2020 11:05:21 -0700 Subject: [PATCH 01/29] update ffi to code with blst fixes --- extern/filecoin-ffi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index f640612a1..57e38efe4 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit f640612a1a1f7a2dd8b3a49e1531db0aa0f63447 +Subproject commit 57e38efe4943f09d3127dcf6f0edd614e6acf68e From 306c098d304cb853351a9fe7d8ccf8ccb2bb1caa Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 24 Sep 2020 11:32:38 -0700 Subject: [PATCH 02/29] also update our vendored blst repo --- extern/fil-blst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/fil-blst b/extern/fil-blst index 5f93488fc..8609119cf 160000 --- a/extern/fil-blst +++ b/extern/fil-blst @@ -1 +1 @@ -Subproject commit 5f93488fc0dbfb450f2355269f18fc67010d59bb +Subproject commit 8609119cf4595d1741139c24378fcd8bc4f1c475 From 585b8cc51dee241dd0d340bd3df0502b10e324a6 Mon Sep 17 00:00:00 2001 From: Arsenii Petrovich Date: Fri, 25 Sep 2020 03:54:29 +0300 Subject: [PATCH 03/29] Add glif node to bootstrap peers --- build/bootstrap/bootstrappers.pi | 1 + 1 file changed, 1 insertion(+) diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi index 465f3b5e9..481ede6ab 100644 --- a/build/bootstrap/bootstrappers.pi +++ b/build/bootstrap/bootstrappers.pi @@ -4,3 +4,4 @@ /dns4/bootstrap-4.testnet.fildev.network/tcp/1347/p2p/12D3KooWPkL9LrKRQgHtq7kn9ecNhGU9QaziG8R5tX8v9v7t3h34 /dns4/bootstrap-3.testnet.fildev.network/tcp/1347/p2p/12D3KooWKYSsbpgZ3HAjax5M1BXCwXLa6gVkUARciz7uN3FNtr7T /dns4/bootstrap-5.testnet.fildev.network/tcp/1347/p2p/12D3KooWQYzqnLASJAabyMpPb1GcWZvNSe7JDcRuhdRqonFoiK9W +/dns4/node.glif.io/tcp/1235/p2p/12D3KooWBF8cpp65hp2u9LK5mh19x67ftAam84z9LsfaquTDSBpt From 89bfe84f36bc8264a50cc6f9982774e06feac7e8 Mon Sep 17 00:00:00 2001 From: MaoBisheng-IPFSUNION Date: Fri, 25 Sep 2020 10:18:12 +0800 Subject: [PATCH 04/29] add new booststrappers we hope to add two booststrappers for the filecoin network to improve the network synchronization --- build/bootstrap/bootstrappers.pi | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi index 9c619b1bb..e64bd9dae 100644 --- a/build/bootstrap/bootstrappers.pi +++ b/build/bootstrap/bootstrappers.pi @@ -4,3 +4,6 @@ /dns4/lotus-bootstrap-1.fra.fil-test.net/tcp/1347/p2p/12D3KooWLLpNYoKdf9NgcWudBhXLdTcXncqAsTzozw1scMMu6nS5 /dns4/lotus-bootstrap-0.sin.fil-test.net/tcp/1347/p2p/12D3KooWCNL9vXaXwNs3Bu8uRAJK4pxpCyPeM7jZLSDpJma1wrV8 /dns4/lotus-bootstrap-1.sin.fil-test.net/tcp/1347/p2p/12D3KooWNGGxFda1eC5U2YKAgs4ypoFHn3Z3xHCsjmFdrCcytoxm + +/dns4/bootstrap-0.starpool.in/tcp/12757/p2p/12D3KooWGHpBMeZbestVEWkfdnC9u7p6uFHXL1n7m1ZBqsEmiUzz +/dns4/bootstrap-1.starpool.in/tcp/12757/p2p/12D3KooWQZrGH1PxSNZPum99M1zNvjNFM33d1AAu5DcvdHptuU7u From 4ff38aa8564a3c86391bc6886e5e37d0c7ca286e Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Fri, 25 Sep 2020 02:35:34 -0700 Subject: [PATCH 05/29] feat(markets): update to 0.6.3 --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index cfdf4cb1d..ca7e0760d 100644 --- a/go.mod +++ b/go.mod @@ -25,9 +25,9 @@ require ( github.com/filecoin-project/go-bitfield v0.2.0 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 - github.com/filecoin-project/go-data-transfer v0.6.5 + github.com/filecoin-project/go-data-transfer v0.6.6 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.6.2 + github.com/filecoin-project/go-fil-markets v0.6.3 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 @@ -60,7 +60,7 @@ require ( github.com/ipfs/go-ds-measure v0.1.0 github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.6 - github.com/ipfs/go-graphsync v0.2.0 + github.com/ipfs/go-graphsync v0.2.1 github.com/ipfs/go-ipfs-blockstore v1.0.1 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-ds-help v1.0.0 diff --git a/go.sum b/go.sum index 3b31e6c3b..1ca615b59 100644 --- a/go.sum +++ b/go.sum @@ -222,12 +222,12 @@ github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:a github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-data-transfer v0.6.5 h1:oP20la8Z0CLrw0uqvt6xVgw6rOevZeGJ9GNQeC0OCSU= -github.com/filecoin-project/go-data-transfer v0.6.5/go.mod h1:I9Ylb/UiZyqnI41wUoCXq/le0nDLhlwpFQCtNPxEPOA= +github.com/filecoin-project/go-data-transfer v0.6.6 h1:2TccLSxPYJENcYRdov2WvpTvQ1qUMrPkWe8sBrfj36g= +github.com/filecoin-project/go-data-transfer v0.6.6/go.mod h1:C++k1U6+jMQODOaen5OPDo9XQbth9Yq3ie94vNjBJbk= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.6.2 h1:9Z57KeaQSa1liCmT1pH6SIjrn9mGTDFJXmR2WQVuaiY= -github.com/filecoin-project/go-fil-markets v0.6.2/go.mod h1:wtN4Hc/1hoVCpWhSWYxwUxH3PQtjSkWWuC1nQjiIWog= +github.com/filecoin-project/go-fil-markets v0.6.3 h1:3kTxfquGvk3zQY+hJH1kEA28tRQ47phqSRqOI4+YcQM= +github.com/filecoin-project/go-fil-markets v0.6.3/go.mod h1:Ug1yhGhzTYC6qrpKsR2QpU8QRCeBpwkTA9RICVKuOMM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= @@ -502,8 +502,8 @@ github.com/ipfs/go-filestore v1.0.0/go.mod h1:/XOCuNtIe2f1YPbiXdYvD0BKLA0JR1MgPi github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0= github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM= github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= -github.com/ipfs/go-graphsync v0.2.0 h1:x94MvHLNuRwBlZzVal7tR1RYK7T7H6bqQLPopxDbIF0= -github.com/ipfs/go-graphsync v0.2.0/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= +github.com/ipfs/go-graphsync v0.2.1 h1:MdehhqBSuTI2LARfKLkpYnt0mUrqHs/mtuDnESXHBfU= +github.com/ipfs/go-graphsync v0.2.1/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= From 9dfce400a7a5234ace883ae61648f94eb915516f Mon Sep 17 00:00:00 2001 From: MaoBisheng-IPFSUNION Date: Fri, 25 Sep 2020 17:36:57 +0800 Subject: [PATCH 06/29] delate --- build/bootstrap/bootstrappers.pi | 1 - 1 file changed, 1 deletion(-) diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi index e64bd9dae..802f1471a 100644 --- a/build/bootstrap/bootstrappers.pi +++ b/build/bootstrap/bootstrappers.pi @@ -4,6 +4,5 @@ /dns4/lotus-bootstrap-1.fra.fil-test.net/tcp/1347/p2p/12D3KooWLLpNYoKdf9NgcWudBhXLdTcXncqAsTzozw1scMMu6nS5 /dns4/lotus-bootstrap-0.sin.fil-test.net/tcp/1347/p2p/12D3KooWCNL9vXaXwNs3Bu8uRAJK4pxpCyPeM7jZLSDpJma1wrV8 /dns4/lotus-bootstrap-1.sin.fil-test.net/tcp/1347/p2p/12D3KooWNGGxFda1eC5U2YKAgs4ypoFHn3Z3xHCsjmFdrCcytoxm - /dns4/bootstrap-0.starpool.in/tcp/12757/p2p/12D3KooWGHpBMeZbestVEWkfdnC9u7p6uFHXL1n7m1ZBqsEmiUzz /dns4/bootstrap-1.starpool.in/tcp/12757/p2p/12D3KooWQZrGH1PxSNZPum99M1zNvjNFM33d1AAu5DcvdHptuU7u From bdc782617f1547b1e9631ba34ba8fba917ce767e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 25 Sep 2020 11:27:30 -0700 Subject: [PATCH 07/29] return an error when we fail to find a sector when checking sector expiration Returning nil, nil is a footgun. fix: https://github.com/filecoin-project/lotus/issues/3984 --- chain/actors/builtin/miner/v0.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 9cdfc25bc..f5aa7849d 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -6,6 +6,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" @@ -96,9 +97,7 @@ func (s *state0) NumLiveSectors() (uint64, error) { // GetSectorExpiration returns the effective expiration of the given sector. // -// If the sector isn't found or has already been terminated, this method returns -// nil and no error. If the sector does not expire early, the Early expiration -// field is 0. +// If the sector does not expire early, the Early expiration field is 0. func (s *state0) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, error) { dls, err := s.State.LoadDeadlines(s.store) if err != nil { @@ -161,7 +160,7 @@ func (s *state0) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, e return nil, err } if out.Early == 0 && out.OnTime == 0 { - return nil, nil + return nil, xerrors.Errorf("failed to find sector %d", num) } return &out, nil } From ddcbcdea4809eb2e0c153207f9c8d1116d9cd1e8 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 25 Sep 2020 12:16:35 -0700 Subject: [PATCH 08/29] test sector status on expiring sectors --- api/test/ccupgrade.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/api/test/ccupgrade.go b/api/test/ccupgrade.go index b281f30a0..f58f1ff6e 100644 --- a/api/test/ccupgrade.go +++ b/api/test/ccupgrade.go @@ -94,6 +94,22 @@ func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) { require.Less(t, 50000, int(exp.OnTime)) } + dlInfo, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + + // Sector should expire. + for { + // Wait for the sector to expire. + status, err := miner.SectorsStatus(ctx, CC, true) + require.NoError(t, err) + if status.OnTime == 0 && status.Early == 0 { + break + } + t.Log("waiting for sector to expire") + // wait one deadline per loop. + time.Sleep(time.Duration(dlInfo.WPoStChallengeWindow) * blocktime) + } + fmt.Println("shutting down mining") atomic.AddInt64(&mine, -1) <-done From 60e43ccbb13a88473f91b80cf645758523658ad2 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 25 Sep 2020 15:45:27 -0400 Subject: [PATCH 09/29] Add an envvar to set address network version --- build/params_shared_funcs.go | 6 ++++++ build/params_shared_vals.go | 12 +++++++++++ chain/types_test.go | 40 ++++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 2 ++ 5 files changed, 61 insertions(+), 1 deletion(-) diff --git a/build/params_shared_funcs.go b/build/params_shared_funcs.go index 0e9739914..28567d3d1 100644 --- a/build/params_shared_funcs.go +++ b/build/params_shared_funcs.go @@ -3,6 +3,8 @@ package build import ( "sort" + "github.com/filecoin-project/go-address" + "github.com/libp2p/go-libp2p-core/protocol" "github.com/filecoin-project/go-state-types/abi" @@ -44,3 +46,7 @@ func UseNewestNetwork() bool { } return false } + +func SetAddressNetwork(n address.Network) { + address.CurrentNetwork = n +} diff --git a/build/params_shared_vals.go b/build/params_shared_vals.go index 3ee9f52ec..5dd23de44 100644 --- a/build/params_shared_vals.go +++ b/build/params_shared_vals.go @@ -4,6 +4,9 @@ package build import ( "math/big" + "os" + + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/network" @@ -60,6 +63,11 @@ const TicketRandomnessLookback = abi.ChainEpoch(1) const WinningPoStSectorSetLookback = abi.ChainEpoch(10) +// ///// +// Address + +const AddressMainnetEnvVar = "_mainnet_" + // ///// // Devnet settings @@ -75,6 +83,10 @@ var InitialRewardBalance *big.Int func init() { InitialRewardBalance = big.NewInt(int64(FilAllocStorageMining)) InitialRewardBalance = InitialRewardBalance.Mul(InitialRewardBalance, big.NewInt(int64(FilecoinPrecision))) + + if os.Getenv("LOTUS_ADDRESS_TYPE") == AddressMainnetEnvVar { + SetAddressNetwork(address.Mainnet) + } } // Sync diff --git a/chain/types_test.go b/chain/types_test.go index 7d68da68d..b47471c9d 100644 --- a/chain/types_test.go +++ b/chain/types_test.go @@ -1,9 +1,12 @@ package chain import ( + "crypto/rand" "encoding/json" "testing" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" ) @@ -35,3 +38,40 @@ func TestSignedMessageJsonRoundtrip(t *testing.T) { t.Fatal(err) } } + +func TestAddressType(t *testing.T) { + build.SetAddressNetwork(address.Testnet) + addr, err := makeRandomAddress() + if err != nil { + t.Fatal(err) + } + + if string(addr[0]) != address.TestnetPrefix { + t.Fatalf("address should start with %s", address.TestnetPrefix) + } + + build.SetAddressNetwork(address.Mainnet) + addr, err = makeRandomAddress() + if err != nil { + t.Fatal(err) + } + + if string(addr[0]) != address.MainnetPrefix { + t.Fatalf("address should start with %s", address.MainnetPrefix) + } +} + +func makeRandomAddress() (string, error) { + bytes := make([]byte, 32) + _, err := rand.Read(bytes) + if err != nil { + return "", err + } + + addr, err := address.NewActorAddress(bytes) + if err != nil { + return "", err + } + + return addr.String(), nil +} diff --git a/go.mod b/go.mod index ca7e0760d..cfa6bf47f 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/elastic/go-sysinfo v1.3.0 github.com/fatih/color v1.8.0 github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200716204036-cddc56607e1d - github.com/filecoin-project/go-address v0.0.3 + github.com/filecoin-project/go-address v0.0.4 github.com/filecoin-project/go-bitfield v0.2.0 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 diff --git a/go.sum b/go.sum index 1ca615b59..7e50fc200 100644 --- a/go.sum +++ b/go.sum @@ -214,6 +214,8 @@ github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGj github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/filecoin-project/go-address v0.0.3 h1:eVfbdjEbpbzIrbiSa+PiGUY+oDK9HnUn+M1R/ggoHf8= github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= +github.com/filecoin-project/go-address v0.0.4 h1:gSNMv0qWwH16fGQs7ycOUrDjY6YCSsgLUl0I0KLjo8w= +github.com/filecoin-project/go-address v0.0.4/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 h1:t6qDiuGYYngDqaLc2ZUvdtAg4UNxPeOYaXhBWSNsVaM= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoCcSuHN1XcbN0c6KBh7yvP5fs= github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7mkT0qCJxzGI2Q= From a13d10e42d70018652e9c4926bad9dd155514950 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Fri, 25 Sep 2020 20:08:28 +0000 Subject: [PATCH 10/29] add logging to chain export --- chain/store/store.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/chain/store/store.go b/chain/store/store.go index fce8a650f..1dbf69547 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -15,9 +15,11 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/journal" bstore "github.com/filecoin-project/lotus/lib/blockstore" @@ -1183,6 +1185,7 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo } blocksToWalk := ts.Cids() + currentMinHeight := ts.Height() walkChain := func(blk cid.Cid) error { if !seen.Visit(blk) { @@ -1203,6 +1206,13 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo return xerrors.Errorf("unmarshaling block header (cid=%s): %w", blk, err) } + if currentMinHeight > b.Height { + currentMinHeight = b.Height + if currentMinHeight%builtin.EpochsInDay == 0 { + log.Infow("export", "height", currentMinHeight) + } + } + var cids []cid.Cid if !skipOldMsgs || b.Height > ts.Height()-inclRecentRoots { mcids, err := recurseLinks(cs.bs, walked, b.Messages, []cid.Cid{b.Messages}) @@ -1251,6 +1261,9 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo return nil } + log.Infow("export started") + exportStart := build.Clock.Now() + for len(blocksToWalk) > 0 { next := blocksToWalk[0] blocksToWalk = blocksToWalk[1:] @@ -1259,6 +1272,8 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo } } + log.Infow("export finished", "duration", build.Clock.Now().Sub(exportStart).Seconds()) + return nil } From 4eec4a01410c19a5687fec620fca418551ce9803 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 25 Sep 2020 23:28:12 +0200 Subject: [PATCH 11/29] Move policy change to seal bench Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 2516bfd26..e409dfe5a 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -76,8 +76,6 @@ func main() { log.Info("Starting lotus-bench") - policy.AddSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) - app := &cli.App{ Name: "lotus-bench", Usage: "Benchmark performance of lotus on your hardware", @@ -147,6 +145,8 @@ var sealBenchCmd = &cli.Command{ }, }, Action: func(c *cli.Context) error { + policy.AddSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) + if c.Bool("no-gpu") { err := os.Setenv("BELLMAN_NO_GPU", "1") if err != nil { From d8431ff611be8ae5d47a4b05504e10ffd48c1673 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 25 Sep 2020 17:33:25 -0400 Subject: [PATCH 12/29] Fix AddSupportedProofTypes --- chain/actors/policy/policy.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/chain/actors/policy/policy.go b/chain/actors/policy/policy.go index eec52e855..b8205177e 100644 --- a/chain/actors/policy/policy.go +++ b/chain/actors/policy/policy.go @@ -22,12 +22,10 @@ func SetSupportedProofTypes(types ...abi.RegisteredSealProof) { // AddSupportedProofTypes sets supported proof types, across all actor versions. // This should only be used for testing. func AddSupportedProofTypes(types ...abi.RegisteredSealProof) { - newTypes := make(map[abi.RegisteredSealProof]struct{}, len(types)) for _, t := range types { - newTypes[t] = struct{}{} + // Set for all miner versions. + miner0.SupportedProofTypes[t] = struct{}{} } - // Set for all miner versions. - miner0.SupportedProofTypes = newTypes } // SetPreCommitChallengeDelay sets the pre-commit challenge delay across all From a5f13a5b31cb555eca3d1e9a63fb69fb93ef5023 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 25 Sep 2020 14:36:36 -0700 Subject: [PATCH 13/29] Test supported proof types. --- chain/actors/policy/policy_test.go | 36 ++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 chain/actors/policy/policy_test.go diff --git a/chain/actors/policy/policy_test.go b/chain/actors/policy/policy_test.go new file mode 100644 index 000000000..be64362a2 --- /dev/null +++ b/chain/actors/policy/policy_test.go @@ -0,0 +1,36 @@ +package policy + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/go-state-types/abi" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" +) + +func TestSupportedProofTypes(t *testing.T) { + var oldTypes []abi.RegisteredSealProof + for t := range miner0.SupportedProofTypes { + oldTypes = append(oldTypes, t) + } + t.Cleanup(func() { + SetSupportedProofTypes(oldTypes...) + }) + + SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) + require.EqualValues(t, + miner0.SupportedProofTypes, + map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, + }, + ) + AddSupportedProofTypes(abi.RegisteredSealProof_StackedDrg8MiBV1) + require.EqualValues(t, + miner0.SupportedProofTypes, + map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, + abi.RegisteredSealProof_StackedDrg8MiBV1: {}, + }, + ) +} From 8545c08f30bdb25add71604f5f6b5f12ed862966 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 25 Sep 2020 23:47:59 -0700 Subject: [PATCH 14/29] add json output to state compute --- cli/state.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cli/state.go b/cli/state.go index d96c93c54..3d37e6fb7 100644 --- a/cli/state.go +++ b/cli/state.go @@ -818,6 +818,10 @@ var stateComputeStateCmd = &cli.Command{ Name: "html", Usage: "generate html report", }, + &cli.BoolFlag{ + Name: "json", + Usage: "generate json output", + }, }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -862,6 +866,15 @@ var stateComputeStateCmd = &cli.Command{ return err } + if cctx.Bool("json") { + out, err := json.Marshal(stout) + if err != nil { + return err + } + fmt.Println(string(out)) + return nil + } + if cctx.Bool("html") { codeCache := map[address.Address]cid.Cid{} getCode := func(addr address.Address) (cid.Cid, error) { From ef28ebb14a9e92833d6eb9cfab5f6f59279e516a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 24 Sep 2020 23:30:11 +0200 Subject: [PATCH 15/29] Ignition upgrades, much excite. --- build/params_2k.go | 2 + build/params_shared_vals.go | 6 +- build/params_testnet.go | 8 + chain/actors/builtin/builtin.go | 2 +- chain/stmgr/forks.go | 377 +++++++++++++++++++++++++++++--- chain/stmgr/forks_test.go | 18 +- chain/stmgr/stmgr.go | 194 +++++++++++++--- chain/stmgr/utils.go | 20 +- cli/state.go | 24 +- cmd/lotus-shed/balances.go | 138 ++++++++++++ go.mod | 2 +- go.sum | 4 +- node/modules/chain.go | 5 + node/modules/storageminer.go | 3 + storage/wdpost_run.go | 4 + 15 files changed, 708 insertions(+), 99 deletions(-) diff --git a/build/params_2k.go b/build/params_2k.go index 3edd0fb82..3682f7be1 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -12,6 +12,8 @@ const UpgradeBreezeHeight = -1 const BreezeGasTampingDuration = 0 const UpgradeSmokeHeight = -1 +const UpgradeIgnitionHeight = -2 +const UpgradeLiftoffHeight = -3 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, diff --git a/build/params_shared_vals.go b/build/params_shared_vals.go index 3ee9f52ec..95c347281 100644 --- a/build/params_shared_vals.go +++ b/build/params_shared_vals.go @@ -22,8 +22,8 @@ const UnixfsLinksPerLevel = 1024 // Consensus / Network const AllowableClockDriftSecs = uint64(1) -const NewestNetworkVersion = network.Version2 -const ActorUpgradeNetworkVersion = network.Version3 +const NewestNetworkVersion = network.Version3 +const ActorUpgradeNetworkVersion = network.Version4 // Epochs const ForkLengthThreshold = Finality @@ -63,6 +63,8 @@ const WinningPoStSectorSetLookback = abi.ChainEpoch(10) // ///// // Devnet settings +var Devnet = true + const FilBase = uint64(2_000_000_000) const FilAllocStorageMining = uint64(1_100_000_000) diff --git a/build/params_testnet.go b/build/params_testnet.go index 13d2ff62e..960f3a9b6 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -21,12 +21,20 @@ const BreezeGasTampingDuration = 120 const UpgradeSmokeHeight = 51000 +const UpgradeIgnitionHeight = 94000 + +// This signals our tentative epoch for mainnet launch. Can make it later, but not earlier. +// Miners, clients, developers, custodians all need time to prepare. +// We still have upgrades and state changes to do, but can happen after signaling timing here. +const UpgradeLiftoffHeight = 148888 + func init() { policy.SetConsensusMinerMinPower(abi.NewStoragePower(10 << 40)) policy.SetSupportedProofTypes( abi.RegisteredSealProof_StackedDrg32GiBV1, abi.RegisteredSealProof_StackedDrg64GiBV1, ) + Devnet = false } const BlockDelaySecs = uint64(builtin0.EpochDurationSeconds) diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index 4079e694a..a85b4da65 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -21,7 +21,7 @@ const ( // Converts a network version into a specs-actors version. func VersionForNetwork(version network.Version) Version { switch version { - case network.Version0, network.Version1, network.Version2: + case network.Version0, network.Version1, network.Version2, network.Version3: return Version0 default: panic(fmt.Sprintf("unsupported network version %d", version)) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index f9418c4d8..872c70b1e 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -1,44 +1,63 @@ package stmgr import ( + "bytes" "context" + "encoding/binary" + "math" + + multisig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + + "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" + + "github.com/filecoin-project/lotus/chain/state" + + "github.com/filecoin-project/specs-actors/actors/migration/nv3" + + "github.com/ipfs/go-cid" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" cbor "github.com/ipfs/go-ipld-cbor" "golang.org/x/xerrors" ) -var ForksAtHeight = map[abi.ChainEpoch]func(context.Context, *StateManager, types.StateTree, *types.TipSet) error{ - build.UpgradeBreezeHeight: UpgradeFaucetBurnRecovery, +var ForksAtHeight = map[abi.ChainEpoch]func(context.Context, *StateManager, ExecCallback, cid.Cid, *types.TipSet) (cid.Cid, error){ + build.UpgradeBreezeHeight: UpgradeFaucetBurnRecovery, + build.UpgradeIgnitionHeight: UpgradeIgnition, + build.UpgradeLiftoffHeight: UpgradeLiftoff, } -func (sm *StateManager) handleStateForks(ctx context.Context, st types.StateTree, height abi.ChainEpoch, ts *types.TipSet) (err error) { +func (sm *StateManager) handleStateForks(ctx context.Context, root cid.Cid, height abi.ChainEpoch, cb ExecCallback, ts *types.TipSet) (cid.Cid, error) { + retCid := root + var err error f, ok := ForksAtHeight[height] if ok { - err := f(ctx, sm, st, ts) + retCid, err = f(ctx, sm, cb, root, ts) if err != nil { - return err + return cid.Undef, err } } - return nil + return retCid, nil } type forEachTree interface { ForEach(func(address.Address, *types.Actor) error) error } -func doTransfer(tree types.StateTree, from, to address.Address, amt abi.TokenAmount) error { +func doTransfer(cb ExecCallback, tree types.StateTree, from, to address.Address, amt abi.TokenAmount) error { fromAct, err := tree.GetActor(from) if err != nil { return xerrors.Errorf("failed to get 'from' actor for transfer: %w", err) @@ -64,10 +83,43 @@ func doTransfer(tree types.StateTree, from, to address.Address, amt abi.TokenAmo return xerrors.Errorf("failed to persist to actor: %w", err) } + if cb != nil { + // record the transfer in execution traces + + fakeMsg := &types.Message{ + From: from, + To: to, + Value: amt, + Nonce: math.MaxUint64, + } + fakeRct := &types.MessageReceipt{ + ExitCode: 0, + Return: nil, + GasUsed: 0, + } + + if err := cb(fakeMsg.Cid(), fakeMsg, &vm.ApplyRet{ + MessageReceipt: *fakeRct, + ActorErr: nil, + ExecutionTrace: types.ExecutionTrace{ + Msg: fakeMsg, + MsgRct: fakeRct, + Error: "", + Duration: 0, + GasCharges: nil, + Subcalls: nil, + }, + Duration: 0, + GasCosts: vm.ZeroGasOutputs(), + }); err != nil { + return xerrors.Errorf("recording transfer: %w", err) + } + } + return nil } -func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types.StateTree, ts *types.TipSet) error { +func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { // Some initial parameters FundsForMiners := types.FromFil(1_000_000) LookbackEpoch := abi.ChainEpoch(32000) @@ -94,22 +146,22 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types // Grab lookback state for account checks lbts, err := sm.ChainStore().GetTipsetByHeight(ctx, LookbackEpoch, ts, false) if err != nil { - return xerrors.Errorf("failed to get tipset at lookback height: %w", err) + return cid.Undef, xerrors.Errorf("failed to get tipset at lookback height: %w", err) } lbtree, err := sm.ParentState(lbts) if err != nil { - return xerrors.Errorf("loading state tree failed: %w", err) + return cid.Undef, xerrors.Errorf("loading state tree failed: %w", err) } ReserveAddress, err := address.NewFromString("t090") if err != nil { - return xerrors.Errorf("failed to parse reserve address: %w", err) + return cid.Undef, xerrors.Errorf("failed to parse reserve address: %w", err) } - fetree, ok := tree.(forEachTree) - if !ok { - return xerrors.Errorf("fork transition state tree doesnt support ForEach (%T)", tree) + tree, err := sm.StateTree(root) + if err != nil { + return cid.Undef, xerrors.Errorf("getting state tree: %w", err) } type transfer struct { @@ -121,7 +173,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types var transfers []transfer // Take all excess funds away, put them into the reserve account - err = fetree.ForEach(func(addr address.Address, act *types.Actor) error { + err = tree.ForEach(func(addr address.Address, act *types.Actor) error { switch act.Code { case builtin0.AccountActorCodeID, builtin0.MultisigActorCodeID, builtin0.PaymentChannelActorCodeID: sysAcc, err := isSystemAccount(addr) @@ -163,13 +215,13 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types return nil }) if err != nil { - return xerrors.Errorf("foreach over state tree failed: %w", err) + return cid.Undef, xerrors.Errorf("foreach over state tree failed: %w", err) } // Execute transfers from previous step for _, t := range transfers { - if err := doTransfer(tree, t.From, t.To, t.Amt); err != nil { - return xerrors.Errorf("transfer %s %s->%s failed: %w", t.Amt, t.From, t.To, err) + if err := doTransfer(cb, tree, t.From, t.To, t.Amt); err != nil { + return cid.Undef, xerrors.Errorf("transfer %s %s->%s failed: %w", t.Amt, t.From, t.To, err) } } @@ -177,19 +229,19 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types var ps power0.State powAct, err := tree.GetActor(builtin0.StoragePowerActorAddr) if err != nil { - return xerrors.Errorf("failed to load power actor: %w", err) + return cid.Undef, xerrors.Errorf("failed to load power actor: %w", err) } cst := cbor.NewCborStore(sm.ChainStore().Blockstore()) if err := cst.Get(ctx, powAct.Head, &ps); err != nil { - return xerrors.Errorf("failed to get power actor state: %w", err) + return cid.Undef, xerrors.Errorf("failed to get power actor state: %w", err) } totalPower := ps.TotalBytesCommitted var transfersBack []transfer // Now, we return some funds to places where they are needed - err = fetree.ForEach(func(addr address.Address, act *types.Actor) error { + err = tree.ForEach(func(addr address.Address, act *types.Actor) error { lbact, err := lbtree.GetActor(addr) if err != nil { if !xerrors.Is(err, types.ErrActorNotFound) { @@ -267,53 +319,310 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types return nil }) if err != nil { - return xerrors.Errorf("foreach over state tree failed: %w", err) + return cid.Undef, xerrors.Errorf("foreach over state tree failed: %w", err) } for _, t := range transfersBack { - if err := doTransfer(tree, t.From, t.To, t.Amt); err != nil { - return xerrors.Errorf("transfer %s %s->%s failed: %w", t.Amt, t.From, t.To, err) + if err := doTransfer(cb, tree, t.From, t.To, t.Amt); err != nil { + return cid.Undef, xerrors.Errorf("transfer %s %s->%s failed: %w", t.Amt, t.From, t.To, err) } } // transfer all burnt funds back to the reserve account burntAct, err := tree.GetActor(builtin0.BurntFundsActorAddr) if err != nil { - return xerrors.Errorf("failed to load burnt funds actor: %w", err) + return cid.Undef, xerrors.Errorf("failed to load burnt funds actor: %w", err) } - if err := doTransfer(tree, builtin0.BurntFundsActorAddr, ReserveAddress, burntAct.Balance); err != nil { - return xerrors.Errorf("failed to unburn funds: %w", err) + if err := doTransfer(cb, tree, builtin0.BurntFundsActorAddr, ReserveAddress, burntAct.Balance); err != nil { + return cid.Undef, xerrors.Errorf("failed to unburn funds: %w", err) } // Top up the reimbursement service reimbAddr, err := address.NewFromString("t0111") if err != nil { - return xerrors.Errorf("failed to parse reimbursement service address") + return cid.Undef, xerrors.Errorf("failed to parse reimbursement service address") } reimb, err := tree.GetActor(reimbAddr) if err != nil { - return xerrors.Errorf("failed to load reimbursement account actor: %w", err) + return cid.Undef, xerrors.Errorf("failed to load reimbursement account actor: %w", err) } difference := types.BigSub(DesiredReimbursementBalance, reimb.Balance) - if err := doTransfer(tree, ReserveAddress, reimbAddr, difference); err != nil { - return xerrors.Errorf("failed to top up reimbursement account: %w", err) + if err := doTransfer(cb, tree, ReserveAddress, reimbAddr, difference); err != nil { + return cid.Undef, xerrors.Errorf("failed to top up reimbursement account: %w", err) } // Now, a final sanity check to make sure the balances all check out total := abi.NewTokenAmount(0) - err = fetree.ForEach(func(addr address.Address, act *types.Actor) error { + err = tree.ForEach(func(addr address.Address, act *types.Actor) error { total = types.BigAdd(total, act.Balance) return nil }) if err != nil { - return xerrors.Errorf("checking final state balance failed: %w", err) + return cid.Undef, xerrors.Errorf("checking final state balance failed: %w", err) } exp := types.FromFil(build.FilBase) if !exp.Equals(total) { - return xerrors.Errorf("resultant state tree account balance was not correct: %s", total) + return cid.Undef, xerrors.Errorf("resultant state tree account balance was not correct: %s", total) + } + + return tree.Flush(ctx) +} + +func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { + store := sm.cs.Store(ctx) + + nst, err := nv3.MigrateStateTree(ctx, store, root, build.UpgradeIgnitionHeight) + if err != nil { + return cid.Undef, xerrors.Errorf("migrating actors state: %w", err) + } + + tree, err := sm.StateTree(nst) + if err != nil { + return cid.Undef, xerrors.Errorf("getting state tree: %w", err) + } + + err = setNetworkName(ctx, store, tree, "ignition") + if err != nil { + return cid.Undef, xerrors.Errorf("setting network name: %w", err) + } + + split1, err := address.NewFromString("t0115") + if err != nil { + return cid.Undef, xerrors.Errorf("first split address: %w", err) + } + + split2, err := address.NewFromString("t0116") + if err != nil { + return cid.Undef, xerrors.Errorf("second split address: %w", err) + } + + err = resetGenesisMsigs(ctx, sm, store, tree) + if err != nil { + return cid.Undef, xerrors.Errorf("resetting genesis msig start epochs: %w", err) + } + + err = splitGenesisMultisig(ctx, cb, split1, store, tree, 50) + if err != nil { + return cid.Undef, xerrors.Errorf("splitting first msig: %w", err) + } + + err = splitGenesisMultisig(ctx, cb, split2, store, tree, 50) + if err != nil { + return cid.Undef, xerrors.Errorf("splitting second msig: %w", err) + } + + err = nv3.CheckStateTree(ctx, store, nst, build.UpgradeIgnitionHeight, builtin0.TotalFilecoin) + if err != nil { + return cid.Undef, xerrors.Errorf("sanity check after ignition upgrade failed: %w", err) + } + + return tree.Flush(ctx) +} + +func UpgradeLiftoff(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { + tree, err := sm.StateTree(root) + if err != nil { + return cid.Undef, xerrors.Errorf("getting state tree: %w", err) + } + + err = setNetworkName(ctx, sm.cs.Store(ctx), tree, "mainnet") + if err != nil { + return cid.Undef, xerrors.Errorf("setting network name: %w", err) + } + + return tree.Flush(ctx) +} + +func setNetworkName(ctx context.Context, store adt0.Store, tree *state.StateTree, name string) error { + ia, err := tree.GetActor(builtin0.InitActorAddr) + if err != nil { + return xerrors.Errorf("getting init actor: %w", err) + } + + var initState init0.State + if err := store.Get(ctx, ia.Head, &initState); err != nil { + return xerrors.Errorf("reading init state: %w", err) + } + + initState.NetworkName = name + + ia.Head, err = store.Put(ctx, &initState) + if err != nil { + return xerrors.Errorf("writing new init state: %w", err) + } + + if err := tree.SetActor(builtin0.InitActorAddr, ia); err != nil { + return xerrors.Errorf("setting init actor: %w", err) + } + + return nil +} + +func splitGenesisMultisig(ctx context.Context, cb ExecCallback, addr address.Address, store adt0.Store, tree *state.StateTree, portions uint64) error { + if portions < 1 { + return xerrors.Errorf("cannot split into 0 portions") + } + + mact, err := tree.GetActor(addr) + if err != nil { + return xerrors.Errorf("getting msig actor: %w", err) + } + + mst, err := multisig.Load(store, mact) + if err != nil { + return xerrors.Errorf("getting msig state: %w", err) + } + + signers, err := mst.Signers() + if err != nil { + return xerrors.Errorf("getting msig signers: %w", err) + } + + thresh, err := mst.Threshold() + if err != nil { + return xerrors.Errorf("getting msig threshold: %w", err) + } + + ibal, err := mst.InitialBalance() + if err != nil { + return xerrors.Errorf("getting msig initial balance: %w", err) + } + + se, err := mst.StartEpoch() + if err != nil { + return xerrors.Errorf("getting msig start epoch: %w", err) + } + + ud, err := mst.UnlockDuration() + if err != nil { + return xerrors.Errorf("getting msig unlock duration: %w", err) + } + + pending, err := adt0.MakeEmptyMap(store).Root() + if err != nil { + return xerrors.Errorf("failed to create empty map: %w", err) + } + + newIbal := big.Div(ibal, types.NewInt(portions)) + newState := &multisig0.State{ + Signers: signers, + NumApprovalsThreshold: thresh, + NextTxnID: 0, + InitialBalance: newIbal, + StartEpoch: se, + UnlockDuration: ud, + PendingTxns: pending, + } + + scid, err := store.Put(ctx, newState) + if err != nil { + return xerrors.Errorf("storing new state: %w", err) + } + + newActor := types.Actor{ + Code: builtin0.MultisigActorCodeID, + Head: scid, + Nonce: 0, + Balance: big.Zero(), + } + + i := uint64(0) + for i < portions { + keyAddr, err := makeKeyAddr(addr, i) + if err != nil { + return xerrors.Errorf("creating key address: %w", err) + } + + idAddr, err := tree.RegisterNewAddress(keyAddr) + if err != nil { + return xerrors.Errorf("registering new address: %w", err) + } + + err = tree.SetActor(idAddr, &newActor) + if err != nil { + return xerrors.Errorf("setting new msig actor state: %w", err) + } + + if err := doTransfer(cb, tree, addr, idAddr, newIbal); err != nil { + return xerrors.Errorf("transferring split msig balance: %w", err) + } + + i++ + } + + return nil +} + +func makeKeyAddr(splitAddr address.Address, count uint64) (address.Address, error) { + var b bytes.Buffer + if err := splitAddr.MarshalCBOR(&b); err != nil { + return address.Undef, xerrors.Errorf("marshalling split address: %w", err) + } + + if err := binary.Write(&b, binary.BigEndian, count); err != nil { + return address.Undef, xerrors.Errorf("writing count into a buffer: %w", err) + } + + if err := binary.Write(&b, binary.BigEndian, []byte("Ignition upgrade")); err != nil { + return address.Undef, xerrors.Errorf("writing fork name into a buffer: %w", err) + } + + addr, err := address.NewActorAddress(b.Bytes()) + if err != nil { + return address.Undef, xerrors.Errorf("create actor address: %w", err) + } + + return addr, nil +} + +func resetGenesisMsigs(ctx context.Context, sm *StateManager, store adt0.Store, tree *state.StateTree) error { + gb, err := sm.cs.GetGenesis() + if err != nil { + return xerrors.Errorf("getting genesis block: %w", err) + } + + gts, err := types.NewTipSet([]*types.BlockHeader{gb}) + if err != nil { + return xerrors.Errorf("getting genesis tipset: %w", err) + } + + cst := cbor.NewCborStore(sm.cs.Blockstore()) + genesisTree, err := state.LoadStateTree(cst, gts.ParentState()) + if err != nil { + return xerrors.Errorf("loading state tree: %w", err) + } + + err = genesisTree.ForEach(func(addr address.Address, genesisActor *types.Actor) error { + if genesisActor.Code == builtin0.MultisigActorCodeID { + currActor, err := tree.GetActor(addr) + if err != nil { + return xerrors.Errorf("loading actor: %w", err) + } + + var currState multisig0.State + if err := store.Get(ctx, currActor.Head, &currState); err != nil { + return xerrors.Errorf("reading multisig state: %w", err) + } + + currState.StartEpoch = build.UpgradeLiftoffHeight + + currActor.Head, err = store.Put(ctx, &currState) + if err != nil { + return xerrors.Errorf("writing new multisig state: %w", err) + } + + if err := tree.SetActor(addr, currActor); err != nil { + return xerrors.Errorf("setting multisig actor: %w", err) + } + } + return nil + }) + + if err != nil { + return xerrors.Errorf("iterating over genesis actors: %w", err) } return nil diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index 516058a81..a3423ccdd 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -25,6 +25,7 @@ import ( _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" + "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log" cbg "github.com/whyrusleeping/cbor-gen" @@ -114,33 +115,38 @@ func TestForkHeightTriggers(t *testing.T) { t.Fatal(err) } - stmgr.ForksAtHeight[testForkHeight] = func(ctx context.Context, sm *StateManager, st types.StateTree, ts *types.TipSet) error { + stmgr.ForksAtHeight[testForkHeight] = func(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { cst := cbor.NewCborStore(sm.ChainStore().Blockstore()) + st, err := sm.StateTree(root) + if err != nil { + return cid.Undef, xerrors.Errorf("getting state tree: %w", err) + } + act, err := st.GetActor(taddr) if err != nil { - return err + return cid.Undef, err } var tas testActorState if err := cst.Get(ctx, act.Head, &tas); err != nil { - return xerrors.Errorf("in fork handler, failed to run get: %w", err) + return cid.Undef, xerrors.Errorf("in fork handler, failed to run get: %w", err) } tas.HasUpgraded = 55 ns, err := cst.Put(ctx, &tas) if err != nil { - return err + return cid.Undef, err } act.Head = ns if err := st.SetActor(taddr, act); err != nil { - return err + return cid.Undef, err } - return nil + return st.Flush(ctx) } inv.Register(builtin.PaymentChannelActorCodeID, &testActor{}, &testActorState{}) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index eaf9215db..e800ce665 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -41,12 +41,13 @@ var log = logging.Logger("statemgr") type StateManager struct { cs *store.ChainStore - stCache map[string][]cid.Cid - compWait map[string]chan struct{} - stlk sync.Mutex - genesisMsigLk sync.Mutex - newVM func(context.Context, *vm.VMOpts) (*vm.VM, error) - genInfo *genesisInfo + stCache map[string][]cid.Cid + compWait map[string]chan struct{} + stlk sync.Mutex + genesisMsigLk sync.Mutex + newVM func(context.Context, *vm.VMOpts) (*vm.VM, error) + preIgnitionGenInfos *genesisInfo + postIgnitionGenInfos *genesisInfo } func NewStateManager(cs *store.ChainStore) *StateManager { @@ -123,9 +124,8 @@ func (sm *StateManager) TipSetState(ctx context.Context, ts *types.TipSet) (st c return st, rec, nil } -func (sm *StateManager) ExecutionTrace(ctx context.Context, ts *types.TipSet) (cid.Cid, []*api.InvocResult, error) { - var trace []*api.InvocResult - st, _, err := sm.computeTipSetState(ctx, ts, func(mcid cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { +func traceFunc(trace *[]*api.InvocResult) func(mcid cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { + return func(mcid cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { ir := &api.InvocResult{ Msg: msg, MsgRct: &ret.MessageReceipt, @@ -135,9 +135,14 @@ func (sm *StateManager) ExecutionTrace(ctx context.Context, ts *types.TipSet) (c if ret.ActorErr != nil { ir.Error = ret.ActorErr.Error() } - trace = append(trace, ir) + *trace = append(*trace, ir) return nil - }) + } +} + +func (sm *StateManager) ExecutionTrace(ctx context.Context, ts *types.TipSet) (cid.Cid, []*api.InvocResult, error) { + var trace []*api.InvocResult + st, _, err := sm.computeTipSetState(ctx, ts, traceFunc(&trace)) if err != nil { return cid.Undef, nil, err } @@ -149,20 +154,24 @@ type ExecCallback func(cid.Cid, *types.Message, *vm.ApplyRet) error func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEpoch, pstate cid.Cid, bms []store.BlockMessages, epoch abi.ChainEpoch, r vm.Rand, cb ExecCallback, baseFee abi.TokenAmount, ts *types.TipSet) (cid.Cid, cid.Cid, error) { - vmopt := &vm.VMOpts{ - StateBase: pstate, - Epoch: epoch, - Rand: r, - Bstore: sm.cs.Blockstore(), - Syscalls: sm.cs.VMSys(), - CircSupplyCalc: sm.GetCirculatingSupply, - NtwkVersion: sm.GetNtwkVersion, - BaseFee: baseFee, + makeVmWithBaseState := func(base cid.Cid) (*vm.VM, error) { + vmopt := &vm.VMOpts{ + StateBase: base, + Epoch: epoch, + Rand: r, + Bstore: sm.cs.Blockstore(), + Syscalls: sm.cs.VMSys(), + CircSupplyCalc: sm.GetCirculatingSupply, + NtwkVersion: sm.GetNtwkVersion, + BaseFee: baseFee, + } + + return sm.newVM(ctx, vmopt) } - vmi, err := sm.newVM(ctx, vmopt) + vmi, err := makeVmWithBaseState(pstate) if err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("instantiating VM failed: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err) } runCron := func() error { @@ -202,19 +211,32 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp for i := parentEpoch; i < epoch; i++ { // handle state forks // XXX: The state tree - err = sm.handleStateForks(ctx, vmi.StateTree(), i, ts) + newState, err := sm.handleStateForks(ctx, pstate, i, cb, ts) if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("error handling state forks: %w", err) } + if pstate != newState { + vmi, err = makeVmWithBaseState(newState) + if err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err) + } + } + if i > parentEpoch { // run cron for null rounds if any if err := runCron(); err != nil { return cid.Cid{}, cid.Cid{}, err } + + newState, err = vmi.Flush(ctx) + if err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("flushing vm: %w", err) + } } vmi.SetBlockHeight(i + 1) + pstate = newState } var receipts []cbg.CBORMarshaler @@ -904,7 +926,7 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { gi.genesisMsigs = append(gi.genesisMsigs, ns) } - sm.genInfo = &gi + sm.preIgnitionGenInfos = &gi return nil } @@ -912,7 +934,7 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { // sets up information about the actors in the genesis state // For testnet we use a hardcoded set of multisig states, instead of what's actually in the genesis multisigs // We also do not consider ANY account actors (including the faucet) -func (sm *StateManager) setupGenesisActorsTestnet(ctx context.Context) error { +func (sm *StateManager) setupPreIgnitionGenesisActorsTestnet(ctx context.Context) error { gi := genesisInfo{} @@ -981,7 +1003,87 @@ func (sm *StateManager) setupGenesisActorsTestnet(ctx context.Context) error { gi.genesisMsigs = append(gi.genesisMsigs, ns) } - sm.genInfo = &gi + sm.preIgnitionGenInfos = &gi + + return nil +} + +// sets up information about the actors in the genesis state, post the ignition fork +func (sm *StateManager) setupPostIgnitionGenesisActors(ctx context.Context) error { + + gi := genesisInfo{} + + gb, err := sm.cs.GetGenesis() + if err != nil { + return xerrors.Errorf("getting genesis block: %w", err) + } + + gts, err := types.NewTipSet([]*types.BlockHeader{gb}) + if err != nil { + return xerrors.Errorf("getting genesis tipset: %w", err) + } + + st, _, err := sm.TipSetState(ctx, gts) + if err != nil { + return xerrors.Errorf("getting genesis tipset state: %w", err) + } + + cst := cbor.NewCborStore(sm.cs.Blockstore()) + sTree, err := state.LoadStateTree(cst, st) + if err != nil { + return xerrors.Errorf("loading state tree: %w", err) + } + + // Unnecessary, should be removed + gi.genesisMarketFunds, err = getFilMarketLocked(ctx, sTree) + if err != nil { + return xerrors.Errorf("setting up genesis market funds: %w", err) + } + + // Unnecessary, should be removed + gi.genesisPledge, err = getFilPowerLocked(ctx, sTree) + if err != nil { + return xerrors.Errorf("setting up genesis pledge: %w", err) + } + + totalsByEpoch := make(map[abi.ChainEpoch]abi.TokenAmount) + + // 6 months + sixMonths := abi.ChainEpoch(183 * builtin0.EpochsInDay) + totalsByEpoch[sixMonths] = big.NewInt(49_929_341) + totalsByEpoch[sixMonths] = big.Add(totalsByEpoch[sixMonths], big.NewInt(32_787_700)) + + // 1 year + oneYear := abi.ChainEpoch(365 * builtin0.EpochsInDay) + totalsByEpoch[oneYear] = big.NewInt(22_421_712) + + // 2 years + twoYears := abi.ChainEpoch(2 * 365 * builtin0.EpochsInDay) + totalsByEpoch[twoYears] = big.NewInt(7_223_364) + + // 3 years + threeYears := abi.ChainEpoch(3 * 365 * builtin0.EpochsInDay) + totalsByEpoch[threeYears] = big.NewInt(87_637_883) + + // 6 years + sixYears := abi.ChainEpoch(6 * 365 * builtin0.EpochsInDay) + totalsByEpoch[sixYears] = big.NewInt(100_000_000) + totalsByEpoch[sixYears] = big.Add(totalsByEpoch[sixYears], big.NewInt(300_000_000)) + + gi.genesisMsigs = make([]msig0.State, 0, len(totalsByEpoch)) + for k, v := range totalsByEpoch { + ns := msig0.State{ + // In the pre-ignition logic, we incorrectly set this value in Fil, not attoFil, an off-by-10^18 error + InitialBalance: big.Mul(v, big.NewInt(int64(build.FilecoinPrecision))), + UnlockDuration: k, + PendingTxns: cid.Undef, + // In the pre-ignition logic, the start epoch was 0. This changes in the fork logic of the Ignition upgrade itself. + StartEpoch: build.UpgradeLiftoffHeight, + } + gi.genesisMsigs = append(gi.genesisMsigs, ns) + } + + sm.postIgnitionGenInfos = &gi return nil } @@ -991,13 +1093,23 @@ func (sm *StateManager) setupGenesisActorsTestnet(ctx context.Context) error { // - For Accounts, it counts max(currentBalance - genesisBalance, 0). func (sm *StateManager) GetFilVested(ctx context.Context, height abi.ChainEpoch, st *state.StateTree) (abi.TokenAmount, error) { vf := big.Zero() - for _, v := range sm.genInfo.genesisMsigs { - au := big.Sub(v.InitialBalance, v.AmountLocked(height)) - vf = big.Add(vf, au) + if height <= build.UpgradeIgnitionHeight { + for _, v := range sm.preIgnitionGenInfos.genesisMsigs { + au := big.Sub(v.InitialBalance, v.AmountLocked(height)) + vf = big.Add(vf, au) + } + } else { + for _, v := range sm.postIgnitionGenInfos.genesisMsigs { + // In the pre-ignition logic, we simply called AmountLocked(height), assuming startEpoch was 0. + // The start epoch changed in the Ignition upgrade. + au := big.Sub(v.InitialBalance, v.AmountLocked(height-v.StartEpoch)) + vf = big.Add(vf, au) + } } // there should not be any such accounts in testnet (and also none in mainnet?) - for _, v := range sm.genInfo.genesisActors { + // continue to use preIgnitionGenInfos, nothing changed at the Ignition epoch + for _, v := range sm.preIgnitionGenInfos.genesisActors { act, err := st.GetActor(v.addr) if err != nil { return big.Zero(), xerrors.Errorf("failed to get actor: %w", err) @@ -1009,8 +1121,10 @@ func (sm *StateManager) GetFilVested(ctx context.Context, height abi.ChainEpoch, } } - vf = big.Add(vf, sm.genInfo.genesisPledge) - vf = big.Add(vf, sm.genInfo.genesisMarketFunds) + // continue to use preIgnitionGenInfos, nothing changed at the Ignition epoch + vf = big.Add(vf, sm.preIgnitionGenInfos.genesisPledge) + // continue to use preIgnitionGenInfos, nothing changed at the Ignition epoch + vf = big.Add(vf, sm.preIgnitionGenInfos.genesisMarketFunds) return vf, nil } @@ -1084,10 +1198,16 @@ func GetFilBurnt(ctx context.Context, st *state.StateTree) (abi.TokenAmount, err func (sm *StateManager) GetCirculatingSupplyDetailed(ctx context.Context, height abi.ChainEpoch, st *state.StateTree) (api.CirculatingSupply, error) { sm.genesisMsigLk.Lock() defer sm.genesisMsigLk.Unlock() - if sm.genInfo == nil { - err := sm.setupGenesisActorsTestnet(ctx) + if sm.preIgnitionGenInfos == nil { + err := sm.setupPreIgnitionGenesisActorsTestnet(ctx) if err != nil { - return api.CirculatingSupply{}, xerrors.Errorf("failed to setup genesis information: %w", err) + return api.CirculatingSupply{}, xerrors.Errorf("failed to setup pre-ignition genesis information: %w", err) + } + } + if sm.postIgnitionGenInfos == nil { + err := sm.setupPostIgnitionGenesisActors(ctx) + if err != nil { + return api.CirculatingSupply{}, xerrors.Errorf("failed to setup post-ignition genesis information: %w", err) } } @@ -1152,6 +1272,10 @@ func (sm *StateManager) GetNtwkVersion(ctx context.Context, height abi.ChainEpoc return network.Version1 } + if height <= build.UpgradeIgnitionHeight { + return network.Version2 + } + return build.NewestNetworkVersion } diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 3493afca3..58e7f480f 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -368,6 +368,16 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch, return cid.Undef, nil, err } + for i := ts.Height(); i < height; i++ { + // handle state forks + base, err = sm.handleStateForks(ctx, base, i, traceFunc(&trace), ts) + if err != nil { + return cid.Undef, nil, xerrors.Errorf("error handling state forks: %w", err) + } + + // TODO: should we also run cron here? + } + r := store.NewChainRand(sm.cs, ts.Cids()) vmopt := &vm.VMOpts{ StateBase: base, @@ -384,16 +394,6 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch, return cid.Undef, nil, err } - for i := ts.Height(); i < height; i++ { - // handle state forks - err = sm.handleStateForks(ctx, vmi.StateTree(), i, ts) - if err != nil { - return cid.Undef, nil, xerrors.Errorf("error handling state forks: %w", err) - } - - // TODO: should we also run cron here? - } - for i, msg := range msgs { // TODO: Use the signed message length for secp messages ret, err := vmi.ApplyMessage(ctx, msg) diff --git a/cli/state.go b/cli/state.go index d96c93c54..b25a5e4a7 100644 --- a/cli/state.go +++ b/cli/state.go @@ -19,6 +19,7 @@ import ( "github.com/multiformats/go-multiaddr" "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multihash" "github.com/urfave/cli/v2" @@ -33,7 +34,9 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/exported" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" ) @@ -834,14 +837,14 @@ var stateComputeStateCmd = &cli.Command{ } h := abi.ChainEpoch(cctx.Uint64("vm-height")) - if h == 0 { - if ts == nil { - head, err := api.ChainHead(ctx) - if err != nil { - return err - } - ts = head + if ts == nil { + head, err := api.ChainHead(ctx) + if err != nil { + return err } + ts = head + } + if h == 0 { h = ts.Height() } @@ -863,13 +866,18 @@ var stateComputeStateCmd = &cli.Command{ } if cctx.Bool("html") { + st, err := state.LoadStateTree(cbor.NewCborStore(apibstore.NewAPIBlockstore(api)), stout.Root) + if err != nil { + return xerrors.Errorf("loading state tree: %w", err) + } + codeCache := map[address.Address]cid.Cid{} getCode := func(addr address.Address) (cid.Cid, error) { if c, found := codeCache[addr]; found { return c, nil } - c, err := api.StateGetActor(ctx, addr, ts.Key()) + c, err := st.GetActor(addr) if err != nil { return cid.Cid{}, err } diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index c156de931..1c89a00cf 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -3,9 +3,16 @@ package main import ( "context" "fmt" + "strconv" + + "github.com/docker/go-units" + lotusbuiltin "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/builtin/power" + "github.com/filecoin-project/lotus/chain/actors/builtin/reward" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" + logging "github.com/ipfs/go-log/v2" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -44,6 +51,7 @@ var auditsCmd = &cli.Command{ Subcommands: []*cli.Command{ chainBalanceCmd, chainBalanceStateCmd, + chainPledgeCmd, }, } @@ -248,3 +256,133 @@ var chainBalanceStateCmd = &cli.Command{ return nil }, } + +var chainPledgeCmd = &cli.Command{ + Name: "stateroot-pledge", + Description: "Calculate sector pledge numbers", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "repo", + Value: "~/.lotus", + }, + }, + ArgsUsage: "[stateroot epoch]", + Action: func(cctx *cli.Context) error { + logging.SetLogLevel("badger", "ERROR") + ctx := context.TODO() + + if !cctx.Args().Present() { + return fmt.Errorf("must pass state root") + } + + sroot, err := cid.Decode(cctx.Args().First()) + if err != nil { + return fmt.Errorf("failed to parse input: %w", err) + } + + epoch, err := strconv.ParseInt(cctx.Args().Get(1), 10, 64) + if err != nil { + return xerrors.Errorf("parsing epoch arg: %w", err) + } + + fsrepo, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return err + } + + lkrepo, err := fsrepo.Lock(repo.FullNode) + if err != nil { + return err + } + + defer lkrepo.Close() //nolint:errcheck + + ds, err := lkrepo.Datastore("/chain") + if err != nil { + return err + } + + mds, err := lkrepo.Datastore("/metadata") + if err != nil { + return err + } + + bs := blockstore.NewBlockstore(ds) + + cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) + + cst := cbor.NewCborStore(bs) + store := adt.WrapStore(ctx, cst) + + sm := stmgr.NewStateManager(cs) + + state, err := state.LoadStateTree(cst, sroot) + if err != nil { + return err + } + + var ( + powerSmoothed lotusbuiltin.FilterEstimate + pledgeCollateral abi.TokenAmount + ) + if act, err := state.GetActor(power.Address); err != nil { + return xerrors.Errorf("loading miner actor: %w", err) + } else if s, err := power.Load(store, act); err != nil { + return xerrors.Errorf("loading power actor state: %w", err) + } else if p, err := s.TotalPowerSmoothed(); err != nil { + return xerrors.Errorf("failed to determine total power: %w", err) + } else if c, err := s.TotalLocked(); err != nil { + return xerrors.Errorf("failed to determine pledge collateral: %w", err) + } else { + powerSmoothed = p + pledgeCollateral = c + } + + circ, err := sm.GetCirculatingSupplyDetailed(ctx, abi.ChainEpoch(epoch), state) + if err != nil { + return err + } + + fmt.Println("(real) circulating supply: ", types.FIL(circ.FilCirculating)) + if circ.FilCirculating.LessThan(big.Zero()) { + circ.FilCirculating = big.Zero() + } + + rewardActor, err := state.GetActor(reward.Address) + if err != nil { + return xerrors.Errorf("loading miner actor: %w", err) + } + + rewardState, err := reward.Load(store, rewardActor) + if err != nil { + return xerrors.Errorf("loading reward actor state: %w", err) + } + + fmt.Println("FilVested", types.FIL(circ.FilVested)) + fmt.Println("FilMined", types.FIL(circ.FilMined)) + fmt.Println("FilBurnt", types.FIL(circ.FilBurnt)) + fmt.Println("FilLocked", types.FIL(circ.FilLocked)) + fmt.Println("FilCirculating", types.FIL(circ.FilCirculating)) + + for _, sectorWeight := range []abi.StoragePower{ + types.NewInt(32 << 30), + types.NewInt(64 << 30), + types.NewInt(32 << 30 * 10), + types.NewInt(64 << 30 * 10), + } { + initialPledge, err := rewardState.InitialPledgeForPower( + sectorWeight, + pledgeCollateral, + &powerSmoothed, + circ.FilCirculating, + ) + if err != nil { + return xerrors.Errorf("calculating initial pledge: %w", err) + } + + fmt.Println("IP ", units.HumanSize(float64(sectorWeight.Uint64())), types.FIL(initialPledge)) + } + + return nil + }, +} diff --git a/go.mod b/go.mod index ca7e0760d..b0de7dfd6 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370 github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/specs-actors v0.9.10 + github.com/filecoin-project/specs-actors v0.9.11 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 github.com/filecoin-project/test-vectors/schema v0.0.1 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index 1ca615b59..6412fe743 100644 --- a/go.sum +++ b/go.sum @@ -254,8 +254,8 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= -github.com/filecoin-project/specs-actors v0.9.10 h1:gU0TrRhgkCsBEOP42sGDE7RQuR0Cov9hJhBqq+RJmjU= -github.com/filecoin-project/specs-actors v0.9.10/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= +github.com/filecoin-project/specs-actors v0.9.11 h1:TnpG7HAeiUrfj0mJM7UaPW0P2137H62RGof7ftT5Mas= +github.com/filecoin-project/specs-actors v0.9.11/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.1 h1:5fNF76nl4qolEvcIsjc0kUADlTMVHO73tW4kXXPnsus= diff --git a/node/modules/chain.go b/node/modules/chain.go index 5eda51078..66f54a76a 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -18,6 +18,7 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/exchange" @@ -157,6 +158,10 @@ func SetGenesis(cs *store.ChainStore, g Genesis) (dtypes.AfterGenesisSet, error) } func NetworkName(mctx helpers.MetricsCtx, lc fx.Lifecycle, cs *store.ChainStore, _ dtypes.AfterGenesisSet) (dtypes.NetworkName, error) { + if !build.Devnet { + return "testnetnet", nil + } + ctx := helpers.LifecycleCtx(mctx, lc) netName, err := stmgr.GetNetworkName(ctx, stmgr.NewStateManager(cs), cs.GetHeaviestTipSet().ParentState()) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 9a94a56a5..de466b004 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -109,6 +109,9 @@ func MinerID(ma dtypes.MinerAddress) (dtypes.MinerID, error) { } func StorageNetworkName(ctx helpers.MetricsCtx, a lapi.FullNode) (dtypes.NetworkName, error) { + if !build.Devnet { + return "testnetnet", nil + } return a.StateNetworkName(ctx) } diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 9a497f879..59ffcb74c 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -371,6 +371,10 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty return j }) + if ts.Height() > build.UpgradeIgnitionHeight { + return // FORK: declaring faults after ignition upgrade makes no sense + } + if faults, sigmsg, err = s.checkNextFaults(context.TODO(), declDeadline, partitions); err != nil { // TODO: This is also potentially really bad, but we try to post anyways log.Errorf("checking sector faults: %v", err) From 12e97dbea743c02031ffbcd4f9800cc2d414e2f1 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sat, 26 Sep 2020 02:59:24 -0400 Subject: [PATCH 16/29] Fix docs and linter --- chain/stmgr/forks.go | 4 ---- documentation/en/api-methods.md | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 872c70b1e..252b731d7 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -53,10 +53,6 @@ func (sm *StateManager) handleStateForks(ctx context.Context, root cid.Cid, heig return retCid, nil } -type forEachTree interface { - ForEach(func(address.Address, *types.Actor) error) error -} - func doTransfer(cb ExecCallback, tree types.StateTree, from, to address.Address, amt abi.TokenAmount) error { fromAct, err := tree.GetActor(from) if err != nil { diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index e489fcb0f..ed082ccbf 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -3825,7 +3825,7 @@ Inputs: ] ``` -Response: `2` +Response: `3` ### StateReadState StateReadState returns the indicated actor's state. From 567261e2c7bc5b6cb764ab0ef3c9f1f30600448d Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 25 Sep 2020 23:59:59 -0700 Subject: [PATCH 17/29] set upgrade heights for testground builds --- build/params_testground.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build/params_testground.go b/build/params_testground.go index 954b5ccfd..07cc88688 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -74,7 +74,9 @@ var ( UpgradeBreezeHeight abi.ChainEpoch = -1 BreezeGasTampingDuration abi.ChainEpoch = 0 - UpgradeSmokeHeight abi.ChainEpoch = -1 + UpgradeSmokeHeight abi.ChainEpoch = -1 + UpgradeIgnitionHeight abi.ChainEpoch = -2 + UpgradeLiftoffHeight abi.ChainEpoch = -3 DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, @@ -82,4 +84,6 @@ var ( NewestNetworkVersion = network.Version2 ActorUpgradeNetworkVersion = network.Version3 + + Devnet = true ) From 45eadc1b3aeddf794b8f29186863e43e6e25034b Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sat, 26 Sep 2020 01:45:22 -0400 Subject: [PATCH 18/29] Lotus version 0.8.0 --- CHANGELOG.md | 57 ++++++++++++++++++++++++++++++++++++++++++++++++ build/version.go | 2 +- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16ced709b..ac687675e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,62 @@ # Lotus changelog +# 0.8.0 / 2020-09-26 + +This consensus-breaking release of Lotus introduces an upgrade to the network. The changes that break consensus are: + +- Upgrading to specs-actors v0.9.11, which reduces WindowPoSt faults per [FIP 0002](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0002.md) to reduce cost for honest miners with occasional faults (see https://github.com/filecoin-project/specs-actors/pull/1181) +- Revisions to some cryptoeconomics and network params + +This release also updates go-fil-markets to fix an incompatibility issue between v0.7.2 and earlier versions. + +## Changes + +#### Dependencies + +- Update spec actors to 0.9.11 (https://github.com/filecoin-project/lotus/pull/4039) +- Update markets to 0.6.3 (https://github.com/filecoin-project/lotus/pull/4013) + +#### Core Lotus + +- Network upgrade (https://github.com/filecoin-project/lotus/pull/4039) +- Fix AddSupportedProofTypes (https://github.com/filecoin-project/lotus/pull/4033) +- Return an error when we fail to find a sector when checking sector expiration (https://github.com/filecoin-project/lotus/pull/4026) +- Batch blockstore copies after block validation (https://github.com/filecoin-project/lotus/pull/3980) +- Remove a misleading miner actor abstraction (https://github.com/filecoin-project/lotus/pull/3977) +- Fix out-of-bounds when loading all sector infos (https://github.com/filecoin-project/lotus/pull/3976) +- Fix break condition in the miner (https://github.com/filecoin-project/lotus/pull/3953) + +#### UX + +- Correct helptext around miners setting ask (https://github.com/filecoin-project/lotus/pull/4009) +- Make sync wait nicer (https://github.com/filecoin-project/lotus/pull/3991) + +#### Tooling and validation + +- Small adjustments following network upgradability changes (https://github.com/filecoin-project/lotus/pull/3996) +- Add some more big pictures stats to stateroot stat (https://github.com/filecoin-project/lotus/pull/3995) +- Add some actors policy setters for testing (https://github.com/filecoin-project/lotus/pull/3975) + +## Contributors + +The following contributors had 5 or more commits go into this release. +We are grateful for every contribution! + +| Contributor | Commits | Lines ± | +|--------------------|---------|---------------| +| arajasek | 66 | +3140/-1261 | +| Stebalien | 64 | +3797/-3434 | +| magik6k | 48 | +1892/-976 | +| raulk | 40 | +2412/-1549 | +| vyzo | 22 | +287/-196 | +| alanshaw | 15 | +761/-146 | +| whyrusleeping | 15 | +736/-52 | +| hannahhoward | 14 | +1237/837- | +| anton | 6 | +32/-8 | +| travisperson | 5 | +502/-6 | +| Frank | 5 | +78/-39 | +| Jennifer | 5 | +148/-41 | + # 0.7.2 / 2020-09-23 This optional release of Lotus introduces a major refactor around how a Lotus node interacts with code from the specs-actors repo. We now use interfaces to read the state of actors, which is required to be able to reason about different versions of actors code at the same time. diff --git a/build/version.go b/build/version.go index cfc8c3ab9..77b98f008 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.7.2" +const BuildVersion = "0.8.0" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit From 81a30cbf062fb0e031dfdfc0826f61faeacc6b22 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Sat, 26 Sep 2020 16:01:10 +0800 Subject: [PATCH 19/29] Add one more node located in China --- build/bootstrap/bootstrappers.pi | 1 + 1 file changed, 1 insertion(+) diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi index 465f3b5e9..1c8b77709 100644 --- a/build/bootstrap/bootstrappers.pi +++ b/build/bootstrap/bootstrappers.pi @@ -4,3 +4,4 @@ /dns4/bootstrap-4.testnet.fildev.network/tcp/1347/p2p/12D3KooWPkL9LrKRQgHtq7kn9ecNhGU9QaziG8R5tX8v9v7t3h34 /dns4/bootstrap-3.testnet.fildev.network/tcp/1347/p2p/12D3KooWKYSsbpgZ3HAjax5M1BXCwXLa6gVkUARciz7uN3FNtr7T /dns4/bootstrap-5.testnet.fildev.network/tcp/1347/p2p/12D3KooWQYzqnLASJAabyMpPb1GcWZvNSe7JDcRuhdRqonFoiK9W +/dns4/lotus-bootstrap.forceup.cn/tcp/41778/p2p/12D3KooWFQsv3nRMUevZNWWsY1Wu6NUzUbawnWU5NcRhgKuJA37C From bddd6dd8a8aaa89d38a5ccebc3b066224ca6b510 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Sat, 26 Sep 2020 11:06:16 -0500 Subject: [PATCH 20/29] fix GetPower with no miner address --- chain/stmgr/utils.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 58e7f480f..bac5a31f5 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -102,6 +102,7 @@ func GetPowerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr addres } var mpow power.Claim + var minpow bool if maddr != address.Undef { var found bool mpow, found, err = pas.MinerPower(maddr) @@ -109,11 +110,11 @@ func GetPowerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr addres // TODO: return an error when not found? return power.Claim{}, power.Claim{}, false, err } - } - minpow, err := pas.MinerNominalPowerMeetsConsensusMinimum(maddr) - if err != nil { - return power.Claim{}, power.Claim{}, false, err + minpow, err = pas.MinerNominalPowerMeetsConsensusMinimum(maddr) + if err != nil { + return power.Claim{}, power.Claim{}, false, err + } } return mpow, tpow, minpow, nil From 1c5cb50da340b19f287b211a40238e8782378cad Mon Sep 17 00:00:00 2001 From: Travis Person Date: Sat, 26 Sep 2020 17:09:16 +0000 Subject: [PATCH 21/29] Add back network power to stats --- tools/stats/metrics.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index 22069c3d0..dd51ee69f 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/reward" @@ -252,6 +253,14 @@ func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointLis p := NewPoint("network.balance", netBalFilFloat) pl.AddPoint(p) + totalPower, err := api.StateMinerPower(ctx, address.Address{}, tipset.Key()) + if err != nil { + return err + } + + p = NewPoint("chain.power", totalPower.TotalPower.QualityAdjPower.Int64()) + pl.AddPoint(p) + miners, err := api.StateListMiners(ctx, tipset.Key()) if err != nil { return err From 70071e273d10f07010e70673f0d878fbb46f2262 Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Sun, 27 Sep 2020 09:58:37 +0800 Subject: [PATCH 22/29] optimize tipset Equals func --- chain/types/tipset.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chain/types/tipset.go b/chain/types/tipset.go index 44d41c29d..5d34ec89d 100644 --- a/chain/types/tipset.go +++ b/chain/types/tipset.go @@ -167,6 +167,10 @@ func (ts *TipSet) Equals(ots *TipSet) bool { return false } + if ts.height != ots.height { + return false + } + if len(ts.blks) != len(ots.blks) { return false } From 04876c663e16eb381adb6695f32db1c7f99493ab Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Sun, 27 Sep 2020 10:17:06 +0800 Subject: [PATCH 23/29] modify tipset Equals --- chain/types/tipset.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chain/types/tipset.go b/chain/types/tipset.go index 5d34ec89d..07eff3734 100644 --- a/chain/types/tipset.go +++ b/chain/types/tipset.go @@ -171,12 +171,12 @@ func (ts *TipSet) Equals(ots *TipSet) bool { return false } - if len(ts.blks) != len(ots.blks) { + if len(ts.cids) != len(ots.cids) { return false } - for i, b := range ts.blks { - if b.Cid() != ots.blks[i].Cid() { + for i, cid := range ts.cids { + if cid != ots.cids[i] { return false } } From 7a14455ac8253e0a9fd23fd358baf541efe3ca8e Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Sun, 27 Sep 2020 15:01:42 +0800 Subject: [PATCH 24/29] miner debug where injectNulls != 0 --- miner/miner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/miner/miner.go b/miner/miner.go index 1b79f5245..d4e7b2317 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -211,6 +211,8 @@ minerLoop: base = prebase } + base.NullRounds += injectNulls // testing + if base.TipSet.Equals(lastBase.TipSet) && lastBase.NullRounds == base.NullRounds { log.Warnf("BestMiningCandidate from the previous round: %s (nulls:%d)", lastBase.TipSet.Cids(), lastBase.NullRounds) if !m.niceSleep(time.Duration(build.BlockDelaySecs) * time.Second) { @@ -219,8 +221,6 @@ minerLoop: continue } - base.NullRounds += injectNulls // testing - b, err := m.mineOne(ctx, base) if err != nil { log.Errorf("mining block failed: %+v", err) From a2d24b5b33a18418edc63bea9518e9dbed72bfef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=9E=97=E6=AC=A3?= Date: Mon, 28 Sep 2020 13:53:29 +0800 Subject: [PATCH 25/29] add ipfsmain bootstrapper --- build/bootstrap/bootstrappers.pi | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi index 465f3b5e9..51adb3cc7 100644 --- a/build/bootstrap/bootstrappers.pi +++ b/build/bootstrap/bootstrappers.pi @@ -4,3 +4,6 @@ /dns4/bootstrap-4.testnet.fildev.network/tcp/1347/p2p/12D3KooWPkL9LrKRQgHtq7kn9ecNhGU9QaziG8R5tX8v9v7t3h34 /dns4/bootstrap-3.testnet.fildev.network/tcp/1347/p2p/12D3KooWKYSsbpgZ3HAjax5M1BXCwXLa6gVkUARciz7uN3FNtr7T /dns4/bootstrap-5.testnet.fildev.network/tcp/1347/p2p/12D3KooWQYzqnLASJAabyMpPb1GcWZvNSe7JDcRuhdRqonFoiK9W + +/dns4/bootstrap-0.ipfsmain.cn/tcp/34721/p2p/12D3KooWQnwEGNqcM2nAcPtRR9rAX8Hrg4k9kJLCHoTR5chJfz6d +/dns4/bootstrap-1.ipfsmain.cn/tcp/34723/p2p/12D3KooWMKxMkD5DMpSWsW7dBddKxKT7L2GgbNuckz9otxvkvByP \ No newline at end of file From a3145bae07f3658183d95d9a6e56d2f3ae38ce76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 28 Sep 2020 12:17:54 +0200 Subject: [PATCH 26/29] daemon cmd: Add progress bar to chain import --- cmd/lotus/daemon.go | 18 +++++++++++++++++- go.mod | 1 + 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index e0fee6564..93f6e4b8c 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -23,6 +23,7 @@ import ( "go.opencensus.io/stats/view" "go.opencensus.io/tag" "golang.org/x/xerrors" + "gopkg.in/cheggaaa/pb.v1" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -333,6 +334,11 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) error { } defer fi.Close() //nolint:errcheck + st, err := os.Stat(fname) + if err != nil { + return err + } + lr, err := r.Lock(repo.FullNode) if err != nil { return err @@ -354,7 +360,17 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) error { cst := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) log.Info("importing chain from file...") - ts, err := cst.Import(fi) + + bar := pb.New64(st.Size()) + br := bar.NewProxyReader(fi) + bar.ShowTimeLeft = true + bar.ShowPercent = true + bar.Units = pb.U_BYTES + + bar.Start() + ts, err := cst.Import(br) + bar.Finish() + if err != nil { return xerrors.Errorf("importing chain failed: %w", err) } diff --git a/go.mod b/go.mod index 67d87347f..2c0322ecc 100644 --- a/go.mod +++ b/go.mod @@ -125,6 +125,7 @@ require ( golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 + gopkg.in/cheggaaa/pb.v1 v1.0.28 gotest.tools v2.2.0+incompatible launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect ) From f8095296997f6d5d8ba05d66acf291cfa048ae50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 28 Sep 2020 12:28:12 +0200 Subject: [PATCH 27/29] daemon cmd: Support imports straight from http --- cmd/lotus/daemon.go | 64 +++++++++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 19 deletions(-) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 93f6e4b8c..b976fde79 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -3,11 +3,14 @@ package main import ( + "bufio" "context" "encoding/hex" "encoding/json" "fmt" + "io" "io/ioutil" + "net/http" "os" "runtime/pprof" "strings" @@ -101,11 +104,11 @@ var DaemonCmd = &cli.Command{ }, &cli.StringFlag{ Name: "import-chain", - Usage: "on first run, load chain from given file and validate", + Usage: "on first run, load chain from given file or url and validate", }, &cli.StringFlag{ Name: "import-snapshot", - Usage: "import chain state from a given chain export file", + Usage: "import chain state from a given chain export file or url", }, &cli.BoolFlag{ Name: "halt-after-import", @@ -207,11 +210,6 @@ var DaemonCmd = &cli.Command{ issnapshot = true } - chainfile, err := homedir.Expand(chainfile) - if err != nil { - return err - } - if err := ImportChain(r, chainfile, issnapshot); err != nil { return err } @@ -327,16 +325,41 @@ func importKey(ctx context.Context, api api.FullNode, f string) error { return nil } -func ImportChain(r repo.Repo, fname string, snapshot bool) error { - fi, err := os.Open(fname) - if err != nil { - return err - } - defer fi.Close() //nolint:errcheck +func ImportChain(r repo.Repo, fname string, snapshot bool) (err error) { + var rd io.Reader + var l int64 + if strings.HasPrefix(fname, "http://") || strings.HasPrefix(fname, "https://") { + resp, err := http.Get(fname) //nolint:gosec + if err != nil { + return err + } + defer resp.Body.Close() //nolint:errcheck - st, err := os.Stat(fname) - if err != nil { - return err + if resp.StatusCode != http.StatusOK { + return xerrors.Errorf("non-200 response: %d", resp.StatusCode) + } + + rd = resp.Body + l = resp.ContentLength + } else { + fname, err = homedir.Expand(fname) + if err != nil { + return err + } + + fi, err := os.Open(fname) + if err != nil { + return err + } + defer fi.Close() //nolint:errcheck + + st, err := os.Stat(fname) + if err != nil { + return err + } + + rd = fi + l = st.Size() } lr, err := r.Lock(repo.FullNode) @@ -359,12 +382,15 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) error { cst := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) - log.Info("importing chain from file...") + log.Infof("importing chain from %s...", fname) - bar := pb.New64(st.Size()) - br := bar.NewProxyReader(fi) + bufr := bufio.NewReaderSize(rd, 1<<20) + + bar := pb.New64(l) + br := bar.NewProxyReader(bufr) bar.ShowTimeLeft = true bar.ShowPercent = true + bar.ShowSpeed = true bar.Units = pb.U_BYTES bar.Start() From 6b16d48bad56544d547449e3df8972cf5feb72ee Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 28 Sep 2020 15:56:44 +0200 Subject: [PATCH 28/29] refactor: fetch tipset blocks in parallel --- chain/store/store.go | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index 1dbf69547..e68857e0b 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -10,6 +10,8 @@ import ( "strconv" "sync" + "golang.org/x/sync/errgroup" + "github.com/filecoin-project/go-state-types/crypto" "github.com/minio/blake2b-simd" @@ -467,14 +469,25 @@ func (cs *ChainStore) LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error) { return v.(*types.TipSet), nil } - var blks []*types.BlockHeader - for _, c := range tsk.Cids() { - b, err := cs.GetBlock(c) - if err != nil { - return nil, xerrors.Errorf("get block %s: %w", c, err) - } + // Fetch tipset block headers from blockstore in parallel + var eg errgroup.Group + cids := tsk.Cids() + blks := make([]*types.BlockHeader, 0, len(cids)) + for _, c := range cids { + c := c + eg.Go(func() error { + b, err := cs.GetBlock(c) + if err != nil { + return xerrors.Errorf("get block %s: %w", c, err) + } - blks = append(blks, b) + blks = append(blks, b) + return nil + }) + } + err := eg.Wait() + if err != nil { + return nil, err } ts, err := types.NewTipSet(blks) From cfe6f595b036c332843697413fce5b78104159af Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 28 Sep 2020 16:35:37 +0200 Subject: [PATCH 29/29] fix: unsafe append in LoadTipSet --- chain/store/store.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index e68857e0b..6c93db7a0 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -472,16 +472,16 @@ func (cs *ChainStore) LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error) { // Fetch tipset block headers from blockstore in parallel var eg errgroup.Group cids := tsk.Cids() - blks := make([]*types.BlockHeader, 0, len(cids)) - for _, c := range cids { - c := c + blks := make([]*types.BlockHeader, len(cids)) + for i, c := range cids { + i, c := i, c eg.Go(func() error { b, err := cs.GetBlock(c) if err != nil { return xerrors.Errorf("get block %s: %w", c, err) } - blks = append(blks, b) + blks[i] = b return nil }) }