From fa4ff89dca8cd8b392e79f32f19c023fd77998f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 26 May 2020 19:55:16 +0200 Subject: [PATCH 001/160] Set devnet params --- build/params_shared.go | 2 +- build/params_testnet.go | 7 ++++--- build/version.go | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/build/params_shared.go b/build/params_shared.go index 2b2f3e985..550afc928 100644 --- a/build/params_shared.go +++ b/build/params_shared.go @@ -118,7 +118,7 @@ const VerifSigCacheSize = 32000 // TODO: If this is gonna stay, it should move to specs-actors const BlockMessageLimit = 512 -const BlockGasLimit = 100_000_000 +const BlockGasLimit = 100_000_000_000 var DrandCoeffs = []string{ "82c279cce744450e68de98ee08f9698a01dd38f8e3be3c53f2b840fb9d09ad62a0b6b87981e179e1b14bc9a2d284c985", diff --git a/build/params_testnet.go b/build/params_testnet.go index 5d5e6b81f..917b8059e 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -12,10 +12,11 @@ import ( ) func init() { - power.ConsensusMinerMinPower = big.NewInt(1024 << 30) + power.ConsensusMinerMinPower = big.NewInt(1024 << 20) miner.SupportedProofTypes = map[abi.RegisteredProof]struct{}{ - abi.RegisteredProof_StackedDRG32GiBSeal: {}, - abi.RegisteredProof_StackedDRG64GiBSeal: {}, + abi.RegisteredProof_StackedDRG512MiBSeal: {}, + abi.RegisteredProof_StackedDRG32GiBSeal: {}, + abi.RegisteredProof_StackedDRG64GiBSeal: {}, } } diff --git a/build/version.go b/build/version.go index b704f2e80..c5d96e9af 100644 --- a/build/version.go +++ b/build/version.go @@ -5,7 +5,7 @@ import "fmt" var CurrentCommit string // BuildVersion is the local build version, set by build system -const BuildVersion = "0.3.0" +const BuildVersion = "0.4.0" var UserVersion = BuildVersion + CurrentCommit From b06b0147a90230c6dfb9b83a29b671fa8efcc546 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 22 May 2020 16:07:38 +0200 Subject: [PATCH 002/160] Switch to new drand client This is a preview of a change to new drand client. Signed-off-by: Jakub Sztandera --- build/params_shared.go | 7 +- chain/beacon/drand/drand.go | 152 +++++----------- chain/beacon/drand/drand_test.go | 22 ++- chain/gen/gen.go | 4 + go.mod | 11 +- go.sum | 290 ++++++++++++++++++++++++------- 6 files changed, 295 insertions(+), 191 deletions(-) diff --git a/build/params_shared.go b/build/params_shared.go index 2b2f3e985..7de5a11d0 100644 --- a/build/params_shared.go +++ b/build/params_shared.go @@ -120,9 +120,4 @@ const VerifSigCacheSize = 32000 const BlockMessageLimit = 512 const BlockGasLimit = 100_000_000 -var DrandCoeffs = []string{ - "82c279cce744450e68de98ee08f9698a01dd38f8e3be3c53f2b840fb9d09ad62a0b6b87981e179e1b14bc9a2d284c985", - "82d51308ad346c686f81b8094551597d7b963295cbf313401a93df9baf52d5ae98a87745bee70839a4d6e65c342bd15b", - "94eebfd53f4ba6a3b8304236400a12e73885e5a781509a5c8d41d2e8b476923d8ea6052649b3c17282f596217f96c5de", - "8dc4231e42b4edf39e86ef1579401692480647918275da767d3e558c520d6375ad953530610fd27daf110187877a65d0", -} +var DrandChain = `{"public_key":"kiouk4KP+DNFuuUz9RcmaaJsAtx21r9ZyAiS4SqxRVwikhGIbzW7Vq9tW+qYECTf","period":25,"genesis_time":1590445175,"hash":"E4oySqZUD5PQ2tACqolFSxvsK26UhoLN5r1NtA9LfJs="}` diff --git a/chain/beacon/drand/drand.go b/chain/beacon/drand/drand.go index 94ee2a28d..39c185ee7 100644 --- a/chain/beacon/drand/drand.go +++ b/chain/beacon/drand/drand.go @@ -1,11 +1,12 @@ package drand import ( + "bytes" "context" - "math/rand" "sync" "time" + "github.com/drand/kyber" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/types" @@ -14,32 +15,26 @@ import ( logging "github.com/ipfs/go-log" - dbeacon "github.com/drand/drand/beacon" - "github.com/drand/drand/core" - dkey "github.com/drand/drand/key" - dnet "github.com/drand/drand/net" - dproto "github.com/drand/drand/protobuf/drand" + dchain "github.com/drand/drand/chain" + dclient "github.com/drand/drand/client" ) var log = logging.Logger("drand") var drandServers = []string{ - "nicolas.drand.fil-test.net:443", - "philipp.drand.fil-test.net:443", - "mathilde.drand.fil-test.net:443", - "ludovic.drand.fil-test.net:443", - "gabbi.drand.fil-test.net:443", - "linus.drand.fil-test.net:443", - "jeff.drand.fil-test.net:443", + "https://pl-eu.testnet.drand.sh", + "https://pl-us.testnet.drand.sh", + "https://pl-sin.testnet.drand.sh", } -var drandPubKey *dkey.DistPublic +var drandChain *dchain.Info func init() { - drandPubKey = new(dkey.DistPublic) - err := drandPubKey.FromTOML(&dkey.DistPublicTOML{Coefficients: build.DrandCoeffs}) + + var err error + drandChain, err = dchain.InfoFromJSON(bytes.NewReader([]byte(build.DrandChain))) if err != nil { - panic(err) + panic("could not unmarshal chain info: " + err.Error()) } } @@ -57,13 +52,9 @@ func (dp *drandPeer) IsTLS() bool { } type DrandBeacon struct { - client dnet.Client + client dclient.Client - peers []dnet.Peer - peersIndex int - peersIndexMtx sync.Mutex - - pubkey *dkey.DistPublic + pubkey kyber.Point // seconds interval time.Duration @@ -80,107 +71,50 @@ func NewDrandBeacon(genesisTs, interval uint64) (*DrandBeacon, error) { if genesisTs == 0 { panic("what are you doing this cant be zero") } + client, err := dclient.New( + dclient.WithHTTPEndpoints(drandServers), + dclient.WithChainInfo(drandChain), + dclient.WithCacheSize(1024), + ) + if err != nil { + return nil, xerrors.Errorf("creating drand client") + } + db := &DrandBeacon{ - client: dnet.NewGrpcClient(), + client: client, localCache: make(map[uint64]types.BeaconEntry), } - for _, ds := range drandServers { - db.peers = append(db.peers, &drandPeer{addr: ds, tls: true}) - } - db.peersIndex = rand.Intn(len(db.peers)) - - groupResp, err := db.client.Group(context.TODO(), db.peers[db.peersIndex], &dproto.GroupRequest{}) - if err != nil { - return nil, xerrors.Errorf("failed to get group response from beacon peer: %w", err) - } - - kgroup, err := core.ProtoToGroup(groupResp) - if err != nil { - return nil, xerrors.Errorf("failed to parse group response: %w", err) - } - - // TODO: verify these values are what we expect them to be - if !kgroup.PublicKey.Equal(drandPubKey) { - return nil, xerrors.Errorf("public key does not match") - } - // fmt.Printf("Drand Pubkey:\n%#v\n", kgroup.PublicKey.TOML()) // use to print public key - db.pubkey = drandPubKey - db.interval = kgroup.Period - db.drandGenTime = uint64(kgroup.GenesisTime) + db.pubkey = drandChain.PublicKey + db.interval = drandChain.Period + db.drandGenTime = uint64(drandChain.GenesisTime) db.filRoundTime = interval db.filGenTime = genesisTs - // TODO: the stream currently gives you back *all* values since drand genesis. - // Having the stream in the background is merely an optimization, so not a big deal to disable it for now - // go db.handleStreamingUpdates() - return db, nil } -func (db *DrandBeacon) rotatePeersIndex() { - db.peersIndexMtx.Lock() - nval := rand.Intn(len(db.peers)) - db.peersIndex = nval - db.peersIndexMtx.Unlock() - - log.Warnf("rotated to drand peer %d, %q", nval, db.peers[nval].Address()) -} - -func (db *DrandBeacon) getPeerIndex() int { - db.peersIndexMtx.Lock() - defer db.peersIndexMtx.Unlock() - return db.peersIndex -} - -func (db *DrandBeacon) handleStreamingUpdates() { - for { - p := db.peers[db.getPeerIndex()] - ch, err := db.client.PublicRandStream(context.Background(), p, &dproto.PublicRandRequest{}) - if err != nil { - log.Warnf("failed to get public rand stream to peer %q: %s", p.Address(), err) - log.Warnf("trying again in 10 seconds") - db.rotatePeersIndex() - time.Sleep(time.Second * 10) - continue - } - - for e := range ch { - db.cacheValue(types.BeaconEntry{ - Round: e.Round, - Data: e.Signature, - }) - } - - log.Warnf("drand beacon stream to peer %q broke, reconnecting in 10 seconds", p.Address()) - db.rotatePeersIndex() - time.Sleep(time.Second * 10) - } -} - func (db *DrandBeacon) Entry(ctx context.Context, round uint64) <-chan beacon.Response { - // check cache, it it if there, otherwise query the endpoint - cres := db.getCachedValue(round) - if cres != nil { - out := make(chan beacon.Response, 1) - out <- beacon.Response{Entry: *cres} - close(out) - return out - } - out := make(chan beacon.Response, 1) + if round != 0 { + be := db.getCachedValue(round) + if be != nil { + out <- beacon.Response{Entry: *be} + close(out) + return out + } + } go func() { - p := db.peers[db.getPeerIndex()] - resp, err := db.client.PublicRand(ctx, p, &dproto.PublicRandRequest{Round: round}) + log.Warnw("fetching randomness", "round", round) + resp, err := db.client.Get(ctx, round) var br beacon.Response if err != nil { - db.rotatePeersIndex() - br.Err = xerrors.Errorf("drand peer %q failed publicRand request: %w", p.Address(), err) + br.Err = xerrors.Errorf("drand failed Get request: %w", err) } else { - br.Entry.Round = resp.GetRound() - br.Entry.Data = resp.GetSignature() + br.Entry.Round = resp.Round() + br.Entry.Data = resp.Signature() } out <- br @@ -189,7 +123,6 @@ func (db *DrandBeacon) Entry(ctx context.Context, round uint64) <-chan beacon.Re return out } - func (db *DrandBeacon) cacheValue(e types.BeaconEntry) { db.cacheLk.Lock() defer db.cacheLk.Unlock() @@ -211,13 +144,12 @@ func (db *DrandBeacon) VerifyEntry(curr types.BeaconEntry, prev types.BeaconEntr // TODO handle genesis better return nil } - b := &dbeacon.Beacon{ + b := &dchain.Beacon{ PreviousSig: prev.Data, Round: curr.Round, Signature: curr.Data, } - //log.Warnw("VerifyEntry", "beacon", b) - err := dbeacon.VerifyBeacon(db.pubkey.Key(), b) + err := dchain.VerifyBeacon(db.pubkey, b) if err == nil { db.cacheValue(curr) } diff --git a/chain/beacon/drand/drand_test.go b/chain/beacon/drand/drand_test.go index 2055597bd..b8b089732 100644 --- a/chain/beacon/drand/drand_test.go +++ b/chain/beacon/drand/drand_test.go @@ -1,14 +1,24 @@ package drand import ( + "encoding/json" "fmt" "testing" + + dchain "github.com/drand/drand/chain" + dclient "github.com/drand/drand/client" + "github.com/stretchr/testify/assert" ) -func TestPrintDrandPubkey(t *testing.T) { - bc, err := NewDrandBeacon(1, 1) - if err != nil { - t.Fatal(err) - } - fmt.Printf("Drand Pubkey:\n%#v\n", bc.pubkey.TOML()) +func TestPrintGroupInfo(t *testing.T) { + c, err := dclient.NewHTTPClient(drandServers[0], nil, nil) + assert.NoError(t, err) + cg := c.(interface { + FetchChainInfo(groupHash []byte) (*dchain.Info, error) + }) + chain, err := cg.FetchChainInfo(nil) + assert.NoError(t, err) + cbytes, err := json.Marshal(chain.ToProto()) + assert.NoError(t, err) + fmt.Printf("%s\n", cbytes) } diff --git a/chain/gen/gen.go b/chain/gen/gen.go index c9ebebcef..b0c4cc044 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -219,6 +219,10 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { miners := []address.Address{maddr1, maddr2} beac := beacon.NewMockBeacon(time.Second) + //beac, err := drand.NewDrandBeacon(tpl.Timestamp, build.BlockDelay) + //if err != nil { + //return nil, xerrors.Errorf("creating drand beacon: %w", err) + //} gen := &ChainGen{ bs: bs, diff --git a/go.mod b/go.mod index 7b04d39be..3a4912c92 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,8 @@ require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/coreos/go-systemd/v22 v22.0.0 github.com/docker/go-units v0.4.0 - github.com/drand/drand v0.8.2-0.20200518165838-d61135e6e2c8 + github.com/drand/drand v0.9.2-0.20200526173017-9cffec0d074e + github.com/drand/kyber v1.0.2 github.com/fatih/color v1.8.0 github.com/filecoin-project/chain-validation v0.0.6-0.20200518190139-483332336e8e github.com/filecoin-project/filecoin-ffi v0.0.0-20200427223233-a0014b17f124 @@ -39,7 +40,7 @@ require ( github.com/gorilla/websocket v1.4.2 github.com/hashicorp/go-multierror v1.1.0 github.com/hashicorp/golang-lru v0.5.4 - github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e + github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d github.com/ipfs/go-bitswap v0.2.8 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.3 @@ -61,7 +62,7 @@ require ( github.com/ipfs/go-ipld-cbor v0.0.5-0.20200428170625-a0bd04d3cbdf github.com/ipfs/go-ipld-format v0.2.0 github.com/ipfs/go-log v1.0.4 - github.com/ipfs/go-log/v2 v2.0.8 + github.com/ipfs/go-log/v2 v2.1.0 github.com/ipfs/go-merkledag v0.3.1 github.com/ipfs/go-path v0.0.7 github.com/ipfs/go-unixfs v0.2.4 @@ -109,7 +110,7 @@ require ( go.uber.org/zap v1.15.0 go4.org v0.0.0-20190313082347-94abd6928b1d // indirect golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 - golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 + golang.org/x/time v0.0.0-20191024005414-555d28b269f0 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 gotest.tools v2.2.0+incompatible @@ -119,5 +120,3 @@ require ( replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v1.18.0 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi - -replace github.com/coreos/go-systemd => github.com/coreos/go-systemd/v22 v22.0.0 diff --git a/go.sum b/go.sum index b407f56ba..155a60127 100644 --- a/go.sum +++ b/go.sum @@ -10,12 +10,12 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee h1:8doiS7ib3zi6/K172oDhSKU0dJ/miJramo9NITOMyZQ= github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee/go.mod h1:W0GbEAA4uFNYOGG2cJpmFJ04E6SD1NLELPYZB57/7AY= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -26,23 +26,36 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrU github.com/Stebalien/go-bitfield v0.0.0-20180330043415-076a62f9ce6e/go.mod h1:3oM7gXIttpYDAJXpVNnSCiUMYBLIZ6cb1t+Ip982MRo= github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= -github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0 h1:5hryIiq9gtn+MiLVn0wP37kb/uTeRZgN08WoCsAhIhI= +github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/beevik/ntp v0.2.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= github.com/benbjohnson/clock v1.0.1 h1:lVM1R/o5khtrr7t3qAr+sS6uagZOP+7iprc7gS3V9CE= github.com/benbjohnson/clock v1.0.1/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= @@ -57,6 +70,8 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -64,23 +79,31 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.2.1-0.20180108230905-e214231b295a/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0 h1:XJIw/+VlJ+87J+doOxznsAWIdmWuViOVhkQamW5YV28= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/daaku/go.zipexe v1.0.0 h1:VSOgZtH418pH9L16hC/JrgSNJbbAL26pj7lmD1+CGdY= @@ -93,8 +116,6 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f h1:BOaYiTvg8p9vBUXpklC22XSK/mifLF7lG9jtmYYi3Tc= github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= -github.com/dchest/blake2b v1.0.0 h1:KK9LimVmE0MjRl9095XJmKqZ+iLxWATvlcpVFRtaw6s= -github.com/dchest/blake2b v1.0.0/go.mod h1:U034kXgbJpCle2wSk5ybGIVhOSHCVLMDqOzcPEA0F7s= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= @@ -105,6 +126,7 @@ github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32y github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -112,24 +134,31 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/drand/bls12-381 v0.3.2 h1:RImU8Wckmx8XQx1tp1q04OV73J9Tj6mmpQLYDP7V1XE= github.com/drand/bls12-381 v0.3.2/go.mod h1:dtcLgPtYT38L3NO6mPDYH0nbpc5tjPassDqiniuAt4Y= -github.com/drand/drand v0.8.2-0.20200518165838-d61135e6e2c8 h1:i5Dh4bklVI1mRLflpOJa5igk6xdPJWgZAe8iesZSs0o= -github.com/drand/drand v0.8.2-0.20200518165838-d61135e6e2c8/go.mod h1:ZdzIrSqqEYZvMiS1UuZlJs3WTb9uLz1I9uH0icYPqoE= +github.com/drand/drand v0.9.2-0.20200526173017-9cffec0d074e h1:e9JEgW+3GaPAqskYoEtL98HueHmUskO5tsRqmPs1ZxE= +github.com/drand/drand v0.9.2-0.20200526173017-9cffec0d074e/go.mod h1:xRGkp/+RCz/BWvF1KjaH2AGJ81hrGFqA6Uml1k1zNto= github.com/drand/kyber v1.0.1-0.20200110225416-8de27ed8c0e2/go.mod h1:UpXoA0Upd1N9l4TvRPHr1qAUBBERj6JQ/mnKI3BPEmw= -github.com/drand/kyber v1.0.1-0.20200331114745-30e90cc60f99 h1:BxLbcT0yq9ii6ShXn7U+0oXB2ABfEfw6GutaVPxoj2Y= -github.com/drand/kyber v1.0.1-0.20200331114745-30e90cc60f99/go.mod h1:Rzu9PGFt3q8d7WWdrHmR8dktHucO0dSTWlMYrgqjSpA= +github.com/drand/kyber v1.0.2 h1:dHjtWJZJdn3zBBZ9pqLsLfcR9ScvDvSqzS1sWA8seao= +github.com/drand/kyber v1.0.2/go.mod h1:x6KOpK7avKj0GJ4emhXFP5n7M7W7ChAPmnQh/OL6vRw= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elastic/go-sysinfo v1.3.0 h1:eb2XFGTMlSwG/yyU9Y8jVAYLIzU2sFzWXwo2gmetyrE= github.com/elastic/go-sysinfo v1.3.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= +github.com/ema/qdisc v0.0.0-20190904071900-b82c76788043/go.mod h1:ix4kG2zvdUd8kEKSW0ZTr1XLks0epFpI4j745DXxlNE= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= @@ -176,6 +205,7 @@ github.com/filecoin-project/sector-storage v0.0.0-20200522011946-a59ca7536a95/go github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= +github.com/filecoin-project/specs-actors v0.4.1-0.20200509020627-3c96f54f3d7d/go.mod h1:UW3ft23q6VS8wQoNqLWjENsu9gu1uh6lxOd+H8cwhT8= github.com/filecoin-project/specs-actors v0.5.2/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= github.com/filecoin-project/specs-actors v0.5.4-0.20200521014528-0df536f7e461 h1:yyAoJ9dNboljDWj0uBdJEbdaTak/YhkjYUQt0GzlY0A= github.com/filecoin-project/specs-actors v0.5.4-0.20200521014528-0df536f7e461/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= @@ -183,6 +213,8 @@ github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 h1: github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102/go.mod h1:xJ1/xl9+8zZeSSSFmDC3Wr6uusCTxyYPI0VeNVSFmPE= github.com/filecoin-project/storage-fsm v0.0.0-20200522010518-83fd743db8bc h1:6du9HB53x8frk+Cky9ymGYsRbvXZDE91EsaOJoCb5Ok= github.com/filecoin-project/storage-fsm v0.0.0-20200522010518-83fd743db8bc/go.mod h1:S0u14Wr55mpe22lElCSKbXrhtWg/jquVTTMhefQ8f4Q= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF6M5KiVP0= @@ -190,28 +222,37 @@ github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1/go.mod h1:0eHX/BVySxPc6SE2mZRoppGq7qcE github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= +github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968 h1:s+PDl6lozQ+dEUtUtQnO7+A2iPG3sK1pI4liU+jxn90= +github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/googleapis v1.1.0 h1:kFkMAZBNAn4j7K0GiZr8cRYzejq68VbheufiV3YuyFI= +github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/gogo/status v1.0.3 h1:WkVBY59mw7qUNTr/bLwO7J2vesJ0rQ2C3tMXrTd3w5M= +github.com/gogo/status v1.0.3/go.mod h1:SavQ51ycCLnc7dGyJxp8YAmudx8xqiVrRf+6IXRsugc= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -225,6 +266,8 @@ github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgj github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -234,6 +277,7 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbBY= github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -241,16 +285,23 @@ github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORR github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f h1:KMlcu9X58lhTA/KrfX8Bi1LQSO4pzoVjTiL3h4Jk+Zk= github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= +github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.14.3 h1:OCJlWkOUoTnl0neNGlf4fUm3TmbEtguw7vR+nGtnDjY= github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= github.com/gxed/go-shellwords v1.0.3/go.mod h1:N7paucT91ByIjmVJHhvoarjoQnmsi3Jd3vH7VqgtMxQ= @@ -261,26 +312,44 @@ github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099 h1:vQqOW github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099/go.mod h1:WVPCl0HO/0RAL5+vBH2GMxBomlxBF70MAS78+Lu1//k= github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e h1:3YKHER4nmd7b5qy5t0GWDTwSn4OyRgfAXSmo6VnryBY= github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e/go.mod h1:I8h3MITA53gN9OnWGCgaMa0JWVRdXthWw4M3CPM54OY= +github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= +github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hodgesds/perf-utils v0.0.8/go.mod h1:F6TfvsbtrF88i++hou29dTXlI2sfsJv+gRZDtmTJkAs= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e h1:txQltCyjXAqVVSZDArPEhUTg35hKwVIuXwtQo7eAMNQ= -github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d h1:/WZQPMZNsjZ7IlCpsLGdQBINg5bxKQ1K1sh6awxLtkA= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= @@ -406,10 +475,10 @@ github.com/ipfs/go-log v1.0.4/go.mod h1:oDCg2FkjogeFOhqqb+N39l2RpTNPL6F/StPkB3kP github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= -github.com/ipfs/go-log/v2 v2.0.5 h1:fL4YI+1g5V/b1Yxr1qAiXTMg1H8z9vx/VmJxBuQMHvU= github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= -github.com/ipfs/go-log/v2 v2.0.8 h1:3b3YNopMHlj4AvyhWAx0pDxqSQWYi4/WuWO7yRV6/Qg= github.com/ipfs/go-log/v2 v2.0.8/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= +github.com/ipfs/go-log/v2 v2.1.0 h1:7l5yjP7aLl5xr7RmqEvPZy9Skq7PP2fIP0Pa7lU/UII= +github.com/ipfs/go-log/v2 v2.1.0/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/ipfs/go-merkledag v0.0.3/go.mod h1:Oc5kIXLHokkE1hWGMBHw+oxehkAaTOqtEb7Zbh6BhLA= github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto= github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= @@ -465,15 +534,20 @@ github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZl github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.1.1-0.20190114141812-62fb9bc030d1 h1:qBCV/RLV02TSfQa7tFmxTihnG+u+7JXByOkhlkR5rmQ= github.com/jonboulle/clockwork v0.1.1-0.20190114141812-62fb9bc030d1/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= +github.com/jsimonetti/rtnetlink v0.0.0-20190830100107-3784a6c7c552/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= +github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -486,6 +560,7 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d h1:68u9r4wEvL3gYg2jvAOgROwZ3H+Y3hIDk4tbbmIjcYQ= @@ -499,7 +574,6 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/libp2p/go-addr-util v0.0.1 h1:TpTQm9cXVRVSKsYbgQ7GKc3KbbHVTnbostgGaDEP+88= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2 h1:7cWK5cdA5x72jX0g8iLrQWm5TRJZ6CzGdPEhWj7plWU= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= @@ -509,7 +583,6 @@ github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoR github.com/libp2p/go-conn-security v0.0.1/go.mod h1:bGmu51N0KU9IEjX7kl2PQjgZa40JQWnayTvNMgD/vyk= github.com/libp2p/go-conn-security-multistream v0.0.1/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE= github.com/libp2p/go-conn-security-multistream v0.0.2/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE= -github.com/libp2p/go-conn-security-multistream v0.1.0 h1:aqGmto+ttL/uJgX0JtQI0tD21CIEy5eYd1Hlp0juHY0= github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= github.com/libp2p/go-conn-security-multistream v0.2.0 h1:uNiDjS58vrvJTg9jO6bySd1rMKejieG7v45ekqHbZ1M= github.com/libp2p/go-conn-security-multistream v0.2.0/go.mod h1:hZN4MjlNetKD3Rq5Jb/P5ohUnFLNzEAR4DLSzpn2QLU= @@ -530,7 +603,6 @@ github.com/libp2p/go-libp2p v0.6.0/go.mod h1:mfKWI7Soz3ABX+XEBR61lGbg+ewyMtJHVt0 github.com/libp2p/go-libp2p v0.6.1/go.mod h1:CTFnWXogryAHjXAKEbOf1OWY+VeAP3lDMZkfEI5sT54= github.com/libp2p/go-libp2p v0.7.0/go.mod h1:hZJf8txWeCduQRDC/WSqBGMxaTHCOYHt2xSU1ivxn0k= github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniVO7zIHGMw= -github.com/libp2p/go-libp2p v0.8.1 h1:6AK178W4GmfGxV+L51bd54/fSWEjNR+S0DO0odk/CwI= github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= github.com/libp2p/go-libp2p v0.8.3/go.mod h1:EsH1A+8yoWK+L4iKcbPYu6MPluZ+CHWI9El8cTaefiM= github.com/libp2p/go-libp2p v0.9.2 h1:5rViLwtjkaEWcIBbk6oII39cVjPTElo3F78SSLf9yho= @@ -541,7 +613,6 @@ github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3Pt github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE= github.com/libp2p/go-libp2p-autonat v0.2.0/go.mod h1:DX+9teU4pEEoZUqR1PiMlqliONQdNbfzE1C718tcViI= github.com/libp2p/go-libp2p-autonat v0.2.1/go.mod h1:MWtAhV5Ko1l6QBsHQNSuM6b1sRkXrpk0/LqCr+vCVxI= -github.com/libp2p/go-libp2p-autonat v0.2.2 h1:4dlgcEEugTFWSvdG2UIFxhnOMpX76QaZSRAtXmYB8n4= github.com/libp2p/go-libp2p-autonat v0.2.2/go.mod h1:HsM62HkqZmHR2k1xgX34WuWDzk/nBwNHoeyyT4IWV6A= github.com/libp2p/go-libp2p-autonat v0.2.3 h1:w46bKK3KTOUWDe5mDYMRjJu1uryqBp8HCNDp/TWMqKw= github.com/libp2p/go-libp2p-autonat v0.2.3/go.mod h1:2U6bNWCNsAG9LEbwccBDQbjzQ8Krdjge1jLTE9rdoMM= @@ -549,7 +620,6 @@ github.com/libp2p/go-libp2p-autonat-svc v0.1.0/go.mod h1:fqi8Obl/z3R4PFVLm8xFtZ6 github.com/libp2p/go-libp2p-blankhost v0.0.1/go.mod h1:Ibpbw/7cPPYwFb7PACIWdvxxv0t0XCCI10t7czjAjTc= github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro= github.com/libp2p/go-libp2p-blankhost v0.1.3/go.mod h1:KML1//wiKR8vuuJO0y3LUd1uLv+tlkGTAr3jC0S5cLg= -github.com/libp2p/go-libp2p-blankhost v0.1.4 h1:I96SWjR4rK9irDHcHq3XHN6hawCRTPUADzkJacgZLvk= github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU= github.com/libp2p/go-libp2p-blankhost v0.1.6 h1:CkPp1/zaCrCnBo0AdsQA0O1VkUYoUOtyHOnoa8gKIcE= github.com/libp2p/go-libp2p-blankhost v0.1.6/go.mod h1:jONCAJqEP+Z8T6EQviGL4JsQcLx1LgTGtVqFNY8EMfQ= @@ -559,11 +629,9 @@ github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFk github.com/libp2p/go-libp2p-circuit v0.1.1/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= github.com/libp2p/go-libp2p-circuit v0.1.3/go.mod h1:Xqh2TjSy8DD5iV2cCOMzdynd6h8OTBGoV1AWbWor3qM= github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3xkAcs3gnksxY7osU= -github.com/libp2p/go-libp2p-circuit v0.2.1 h1:BDiBcQxX/ZJJ/yDl3sqZt1bjj4PkZCEi7IEpwxXr13k= github.com/libp2p/go-libp2p-circuit v0.2.1/go.mod h1:BXPwYDN5A8z4OEY9sOfr2DUQMLQvKt/6oku45YUmjIo= github.com/libp2p/go-libp2p-circuit v0.2.2 h1:87RLabJ9lrhoiSDDZyCJ80ZlI5TLJMwfyoGAaWXzWqA= github.com/libp2p/go-libp2p-circuit v0.2.2/go.mod h1:nkG3iE01tR3FoQ2nMm06IUrCpCyJp1Eo4A1xYdpjfs4= -github.com/libp2p/go-libp2p-connmgr v0.1.1 h1:BIul1BPoN1vPAByMh6CeD33NpGjD+PkavmUjTS7uai8= github.com/libp2p/go-libp2p-connmgr v0.1.1/go.mod h1:wZxh8veAmU5qdrfJ0ZBLcU8oJe9L82ciVP/fl1VHjXk= github.com/libp2p/go-libp2p-connmgr v0.2.3 h1:v7skKI9n+0obPpzMIO6aIlOSdQOmhxTf40cbpzqaGMQ= github.com/libp2p/go-libp2p-connmgr v0.2.3/go.mod h1:Gqjg29zI8CwXX21zRxy6gOg8VYu3zVerJRt2KyktzH4= @@ -583,7 +651,6 @@ github.com/libp2p/go-libp2p-core v0.4.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZas github.com/libp2p/go-libp2p-core v0.5.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0= github.com/libp2p/go-libp2p-core v0.5.1/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y= github.com/libp2p/go-libp2p-core v0.5.2/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y= -github.com/libp2p/go-libp2p-core v0.5.3 h1:b9W3w7AZR2n/YJhG8d0qPFGhGhCWKIvPuJgp4hhc4MM= github.com/libp2p/go-libp2p-core v0.5.3/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y= github.com/libp2p/go-libp2p-core v0.5.4/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y= github.com/libp2p/go-libp2p-core v0.5.5/go.mod h1:vj3awlOr9+GMZJFH9s4mpt9RHHgGqeHCopzbYKZdRjM= @@ -645,7 +712,6 @@ github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVd github.com/libp2p/go-libp2p-peerstore v0.2.0/go.mod h1:N2l3eVIeAitSg3Pi2ipSrJYnqhVnMNQZo9nkSCuAbnQ= github.com/libp2p/go-libp2p-peerstore v0.2.1/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA= github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA= -github.com/libp2p/go-libp2p-peerstore v0.2.3 h1:MofRq2l3c15vQpEygTetV+zRRrncz+ktiXW7H2EKoEQ= github.com/libp2p/go-libp2p-peerstore v0.2.3/go.mod h1:K8ljLdFn590GMttg/luh4caB/3g0vKuY01psze0upRw= github.com/libp2p/go-libp2p-peerstore v0.2.4 h1:jU9S4jYN30kdzTpDAR7SlHUD+meDUjTODh4waLWF1ws= github.com/libp2p/go-libp2p-peerstore v0.2.4/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= @@ -679,7 +745,6 @@ github.com/libp2p/go-libp2p-swarm v0.0.6/go.mod h1:s5GZvzg9xXe8sbeESuFpjt8CJPTCa github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4= github.com/libp2p/go-libp2p-swarm v0.2.1/go.mod h1:x07b4zkMFo2EvgPV2bMTlNmdQc8i+74Jjio7xGvsTgU= github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaTNyBcHImCxRpPKU= -github.com/libp2p/go-libp2p-swarm v0.2.3 h1:uVkCb8Blfg7HQ/f30TyHn1g/uCwXsAET7pU0U59gx/A= github.com/libp2p/go-libp2p-swarm v0.2.3/go.mod h1:P2VO/EpxRyDxtChXz/VPVXyTnszHvokHKRhfkEgFKNM= github.com/libp2p/go-libp2p-swarm v0.2.4 h1:94XL76/tFeTdJNcIGugi+1uZo5O/a7y4i21PirwbgZI= github.com/libp2p/go-libp2p-swarm v0.2.4/go.mod h1:/xIpHFPPh3wmSthtxdGbkHZ0OET1h/GGZes8Wku/M5Y= @@ -698,7 +763,6 @@ github.com/libp2p/go-libp2p-transport v0.0.5/go.mod h1:StoY3sx6IqsP6XKoabsPnHCwq github.com/libp2p/go-libp2p-transport-upgrader v0.0.1/go.mod h1:NJpUAgQab/8K6K0m+JmZCe5RUXG10UMEx4kWe9Ipj5c= github.com/libp2p/go-libp2p-transport-upgrader v0.0.4/go.mod h1:RGq+tupk+oj7PzL2kn/m1w6YXxcIAYJYeI90h6BGgUc= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= -github.com/libp2p/go-libp2p-transport-upgrader v0.2.0 h1:5EhPgQhXZNyfL22ERZTUoVp9UVVbNowWNVtELQaKCHk= github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSoMuyojOdjx73Q47cYD7s5+gZOlns= github.com/libp2p/go-libp2p-transport-upgrader v0.3.0 h1:q3ULhsknEQ34eVDhv4YwKS8iet69ffs9+Fir6a7weN4= github.com/libp2p/go-libp2p-transport-upgrader v0.3.0/go.mod h1:i+SKzbRnvXdVbU3D1dwydnTmKRPXiAR/fyvi1dXuL4o= @@ -734,7 +798,6 @@ github.com/libp2p/go-netroute v0.1.2 h1:UHhB35chwgvcRI392znJA3RCBtZ3MpE3ahNCN5MR github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0= github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= -github.com/libp2p/go-openssl v0.0.4 h1:d27YZvLoTyMhIN4njrkr8zMDOM4lfpHIp6A+TK9fovg= github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.5 h1:pQkejVhF0xp08D4CQUcw8t+BFJeXowja6RVcb5p++EA= github.com/libp2p/go-openssl v0.0.5/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= @@ -744,14 +807,12 @@ github.com/libp2p/go-reuseport-transport v0.0.1/go.mod h1:YkbSDrvjUVDL6b8XqriyA2 github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-reuseport-transport v0.0.3 h1:zzOeXnTooCkRvoH+bSXEfXhn76+LAiwoneM0gnXjF2M= github.com/libp2p/go-reuseport-transport v0.0.3/go.mod h1:Spv+MPft1exxARzP2Sruj2Wb5JSyHNncjf1Oi2dEbzM= -github.com/libp2p/go-sockaddr v0.0.2 h1:tCuXfpA9rq7llM/v834RKc/Xvovy/AqM9kHvTV/jY/Q= github.com/libp2p/go-sockaddr v0.0.2/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= github.com/libp2p/go-sockaddr v0.1.0 h1:Y4s3/jNoryVRKEBrkJ576F17CPOaMIzUeCsg7dlTDj0= github.com/libp2p/go-sockaddr v0.1.0/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14= github.com/libp2p/go-stream-muxer v0.1.0/go.mod h1:8JAVsjeRBCWwPoZeH0W1imLOcriqXJyFvB0mR4A04sQ= github.com/libp2p/go-stream-muxer-multistream v0.1.1/go.mod h1:zmGdfkQ1AzOECIAcccoL8L//laqawOsO03zX8Sa+eGw= -github.com/libp2p/go-stream-muxer-multistream v0.2.0 h1:714bRJ4Zy9mdhyTLJ+ZKiROmAFwUHpeRidG+q7LTQOg= github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc= github.com/libp2p/go-stream-muxer-multistream v0.3.0 h1:TqnSHPJEIqDEO7h1wZZ0p3DXdvDSiLHQidKKUGZtiOY= github.com/libp2p/go-stream-muxer-multistream v0.3.0/go.mod h1:yDh8abSIzmZtqtOt64gFJUXEryejzNb0lisTt+fAMJA= @@ -768,7 +829,6 @@ github.com/libp2p/go-ws-transport v0.0.5/go.mod h1:Qbl4BxPfXXhhd/o0wcrgoaItHqA9t github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw7yT74kj3raBFuo= github.com/libp2p/go-ws-transport v0.1.2/go.mod h1:dsh2Ld8F+XNmzpkaAijmg5Is+e9l6/1tK/6VFOdN69Y= github.com/libp2p/go-ws-transport v0.2.0/go.mod h1:9BHJz/4Q5A9ludYWKoGCFC5gUElzlHoKzu0yY9p/klM= -github.com/libp2p/go-ws-transport v0.3.0 h1:mjo6pL5aVR9rCjl9wNq3DupbaQlyR61pzoOT2MdtxaA= github.com/libp2p/go-ws-transport v0.3.0/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk= github.com/libp2p/go-ws-transport v0.3.1 h1:ZX5rWB8nhRRJVaPO6tmkGI/Xx8XNboYX20PW5hXIscw= github.com/libp2p/go-ws-transport v0.3.1/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk= @@ -777,12 +837,15 @@ github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZ github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.3.0/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.3.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= -github.com/libp2p/go-yamux v1.3.5 h1:ibuz4naPAully0pN6J/kmUARiqLpnDQIzI/8GCOrljg= github.com/libp2p/go-yamux v1.3.5/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.3.6 h1:O5qcBXRcfqecvQ/My9NqDNHB3/5t58yuJYqthcKhhgE= github.com/libp2p/go-yamux v1.3.6/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lucas-clemente/quic-go v0.11.2 h1:Mop0ac3zALaBR3wGs6j8OYe/tcFvFsxTUFMkE/7yUOI= github.com/lucas-clemente/quic-go v0.11.2/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw= +github.com/lufia/iostat v1.1.0/go.mod h1:rEPNA0xXgjHQjuI5Cy05sLlS2oRcSlWHRLrvh/AQ+Pg= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA= @@ -796,15 +859,23 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-xmlrpc v0.0.3/go.mod h1:mqc2dz7tP5x5BKlCahN/n+hs7OSZKJkS9JsHNBRlrxA= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mdlayher/genetlink v1.0.0/go.mod h1:0rJ0h4itni50A86M2kHcgS85ttZazNt7a8H2a2cw0Gc= +github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= +github.com/mdlayher/netlink v0.0.0-20190828143259-340058475d09/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M= +github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M= +github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY= +github.com/mdlayher/wifi v0.0.0-20190303161829-b1436901ddee/go.mod h1:Evt/EIne46u9PtQbeTx2NTcqURpr5K4SvKtGmBuDPN8= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -816,8 +887,14 @@ github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -836,7 +913,6 @@ github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lg github.com/multiformats/go-multiaddr v0.1.0/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= -github.com/multiformats/go-multiaddr v0.2.1 h1:SgG/cw5vqyB5QQe5FPe2TqggU9WtrA9X4nZw7LlVqOI= github.com/multiformats/go-multiaddr v0.2.1/go.mod h1:s/Apk6IyxfvMjDafnhJgJ3/46z7tZ04iMk5wP4QMGGE= github.com/multiformats/go-multiaddr v0.2.2 h1:XZLDTszBIJe6m0zF6ITBrEcZR73OPUhCBBS9rYAuUzI= github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u0xW5UouOmQQrn6a3Y= @@ -857,7 +933,6 @@ github.com/multiformats/go-multiaddr-net v0.1.3/go.mod h1:ilNnaM9HbmVFqsb/qcNysj github.com/multiformats/go-multiaddr-net v0.1.4/go.mod h1:ilNnaM9HbmVFqsb/qcNysjCu4PVONlrBZpHIrw/qQuA= github.com/multiformats/go-multiaddr-net v0.1.5 h1:QoRKvu0xHN1FCFJcMQLbG/yQE2z441L5urvG3+qyz7g= github.com/multiformats/go-multiaddr-net v0.1.5/go.mod h1:ilNnaM9HbmVFqsb/qcNysjCu4PVONlrBZpHIrw/qQuA= -github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.2 h1:2pAgScmS1g9XjH7EtAfNhTuyrWYEWcxy0G5Wo85hWDA= github.com/multiformats/go-multibase v0.0.2/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= @@ -879,12 +954,19 @@ github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c h1:5bFTChQxSKNwy8ALwOebjekYExl9HTT9urdawqC95tA= github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c/go.mod h1:7qN3Y0BvzRUf4LofcoJplQL10lsFDb4PYlePTVwrP28= -github.com/nikkolasg/slog v0.0.0-20170921200349-3c8d441d7a1e h1:07zdEcJ4Fble5uWsqKpjW19699kQWRLXP+RZh1a6ZRg= -github.com/nikkolasg/slog v0.0.0-20170921200349-3c8d441d7a1e/go.mod h1:79GLCU4P87rYvYYACbNwVyc1WmRvkwQbYnybpCmRXzg= -github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -895,16 +977,32 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02 h1:0R5mDLI66Qw13qN80TRz85zthQ2nf2+uDyiV23w6c3Q= +github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02/go.mod h1:JNdpVEzCpXBgIiv4ds+TzhN1hrtxq6ClLrTlT9OQRSc= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9 h1:QsgXACQhd9QJhEmRumbsMQQvBtmdS0mafoVEBplWXEg= +github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= @@ -912,34 +1010,42 @@ github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14/go.mod h1:uIp+gprXx github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a h1:hjZfReYVLbqFkAtr2us7vdy04YWz3LVAirzP7reh8+M= github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/node_exporter v1.0.0-rc.0.0.20200428091818-01054558c289/go.mod h1:FGbBv5OPKjch+jNUJmEQpMZytIdyW0NdBtWFcfSKusc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= +github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.6.0 h1:G9tHG9lebljV9mfp9SNPDL36nCDxmo3zTlAf1YgvzmI= @@ -948,11 +1054,19 @@ github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNue github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sercand/kuberesolver v2.1.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJPl/ZshwPsX/n4Y7u0CW9E7aQIQ= +github.com/sercand/kuberesolver v2.4.0+incompatible h1:WE2OlRf6wjLxHwNkkFLQGaZcVLEXjMjBPjjEU5vksH8= +github.com/sercand/kuberesolver v2.4.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJPl/ZshwPsX/n4Y7u0CW9E7aQIQ= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/siebenmann/go-kstat v0.0.0-20160321171754-d34789b79745/go.mod h1:G81aIFAMS9ECrwBYR9YxhlPjWgrItd+Kje78O6+uqm8= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -960,11 +1074,13 @@ github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHei github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= -github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/soundcloud/go-runit v0.0.0-20150630195641-06ad41a06c4a/go.mod h1:LeFCbQYJ3KJlPs/FvPz2dy1tkpxyeNESVyCNNzRXFR0= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= @@ -973,13 +1089,17 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= @@ -990,21 +1110,31 @@ github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-client-go v2.23.1+incompatible h1:uArBYHQR0HqLFFAypI7RsWTzPSj/bDpmZZuQjMLSg1A= +github.com/uber/jaeger-client-go v2.23.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-lib v1.5.1-0.20181102163054-1fc5c315e03c/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= +github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/weaveworks/common v0.0.0-20200512154658-384f10054ec5 h1:EYxr08r8x6r/5fLEAMMkida1BVgxVXE4LfZv/XV+znU= +github.com/weaveworks/common v0.0.0-20200512154658-384f10054ec5/go.mod h1:c98fKi5B9u8OsKGiWHLRKus6ToQ1Tubeow44ECO1uxY= +github.com/weaveworks/promrus v1.2.0 h1:jOLf6pe6/vss4qGHjXmGz4oDJQA+AOCqEL3FvvZGz7M= +github.com/weaveworks/promrus v1.2.0/go.mod h1:SaE82+OJ91yqjrE1rsvBWVzNZKcHYFtMUyS1+Ogs/KA= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba h1:X4n8JG2e2biEZZXdBKt9HX7DN3bYGFUqljqqy0DqgnY= github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba/go.mod h1:CHQnYnQUEPydYCwuy8lmTHfGmdw9TKrhWV0xLx8l0oM= @@ -1024,7 +1154,6 @@ github.com/whyrusleeping/go-ctrlnet v0.0.0-20180313164037-f564fbbdaa95/go.mod h1 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= -github.com/whyrusleeping/go-logging v0.0.1 h1:fwpzlmT0kRC/Fmd0MdmGgJG/CXIZ6gFq46FQZjprUcc= github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8= github.com/whyrusleeping/go-smux-multiplex v3.0.16+incompatible/go.mod h1:34LEDbeKFZInPUrAG+bjuJmUXONGdEFW7XL0SpTY1y4= @@ -1043,6 +1172,7 @@ github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSv github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= github.com/whyrusleeping/yamux v1.1.5/go.mod h1:E8LnQQ8HKx5KD29HZFUwM1PxCOdPRzGwur1mcYhXcD8= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= @@ -1054,14 +1184,18 @@ go.dedis.ch/protobuf v1.0.5/go.mod h1:eIV4wicvi6JK0q/QnfIEGeSFNG0ZeB24kzut5+HaRL go.dedis.ch/protobuf v1.0.7/go.mod h1:pv5ysfkDX/EawiPqcW3ikOxsL5t+BqnV6xHSmE79KI4= go.dedis.ch/protobuf v1.0.11 h1:FTYVIEzY/bfl37lu3pR4lIj+F9Vp1jE8oh91VmxKgLo= go.dedis.ch/protobuf v1.0.11/go.mod h1:97QR256dnkimeNdfmURz0wAMNVbd1VmLXhG1CrTYrJ4= -go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= +go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1082,16 +1216,15 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEa go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -go4.org v0.0.0-20190218023631-ce4c26f7be8e h1:m9LfARr2VIOW0vsV19kEKp/sWQvZnGobA8JHui/XJoY= go4.org v0.0.0-20190218023631-ce4c26f7be8e/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20190313082347-94abd6928b1d h1:JkRdGP3zvTtTbabWSAC6n67ka30y7gOzWAah4XYJSfw= go4.org v0.0.0-20190313082347-94abd6928b1d/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1104,6 +1237,7 @@ golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1111,8 +1245,6 @@ golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200427165652-729f1e841bcc h1:ZGI/fILM2+ueot/UixBSoj9188jCAxVHEZEGhqq67I4= -golang.org/x/crypto v0.0.0-20200427165652-729f1e841bcc/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1122,6 +1254,7 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -1133,8 +1266,10 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1142,13 +1277,17 @@ golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200519113804-d87ec0cfa476 h1:E7ct1C6/33eOdrGZKMoyntcEvs2dwZnDe30crG5vpYU= golang.org/x/net v0.0.0-20200519113804-d87ec0cfa476/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -1162,9 +1301,12 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1174,7 +1316,9 @@ golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1183,33 +1327,37 @@ golang.org/x/sys v0.0.0-20190524152521-dbbf3f1254d4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190902133755-9109b7679e13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191025090151-53bf42e6b339/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200427175716-29b57079015a h1:08u6b1caTT9MQY4wSbmsd4Ulm6DmgNYnbImBuZjGJow= -golang.org/x/sys v0.0.0-20200427175716-29b57079015a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f h1:mOhmO9WsBaJCNmaZHPtHs9wOcdqdKCjF6OPJlmDM3KI= golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1231,8 +1379,10 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200108195415-316d2f248479/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200216192241-b320d3a0f5a2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200318150045-ba25ddc85566 h1:OXjomkWHhzUx4+HldlJ2TsMxJdWgEo5CTtspD1wdhdk= golang.org/x/tools v0.0.0-20200318150045-ba25ddc85566/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1243,38 +1393,50 @@ google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMt google.golang.org/api v0.3.2 h1:iTp+3yyl/KOtxa/d1/JUE0GGSoR6FuW5udver22iwpw= google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c h1:hrpEMCZ2O7DR5gC1n2AJGVhrwiEjOi35+jxtIuZpTMo= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20200406120821-33397c535dc2 h1:KlOjjpQjL4dqscfbhtQvAnRMm5PaRTchHHczffkUiq0= -google.golang.org/genproto v0.0.0-20200406120821-33397c535dc2/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= -google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 h1:Ggy3mWN4l3PUFPfSG0YB3n5fVYggzysUmiUQ89SnX6Y= gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8/go.mod h1:cKXr3E0k4aosgycml1b5z33BVV6hai1Kh7uDgFOkbcs= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1294,3 +1456,5 @@ howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqp howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= From 218b2b82c1cf0dfd7c690f755002d4c641968b36 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 27 May 2020 14:09:04 +0200 Subject: [PATCH 003/160] Update drand chain info Signed-off-by: Jakub Sztandera --- Makefile | 2 +- build/params_shared.go | 2 +- chain/beacon/drand/drand_test.go | 6 ++---- go.sum | 7 ++++++- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 1af851082..a189ccca2 100644 --- a/Makefile +++ b/Makefile @@ -119,7 +119,7 @@ benchmarks: @curl -X POST 'http://benchmark.kittyhawk.wtf/benchmark' -d '@bench.json' -u "${benchmark_http_cred}" .PHONY: benchmarks -pond: build +pond: 2k go build -o pond ./lotuspond (cd lotuspond/front && npm i && CI=false npm run build) .PHONY: pond diff --git a/build/params_shared.go b/build/params_shared.go index 7de5a11d0..06d708444 100644 --- a/build/params_shared.go +++ b/build/params_shared.go @@ -120,4 +120,4 @@ const VerifSigCacheSize = 32000 const BlockMessageLimit = 512 const BlockGasLimit = 100_000_000 -var DrandChain = `{"public_key":"kiouk4KP+DNFuuUz9RcmaaJsAtx21r9ZyAiS4SqxRVwikhGIbzW7Vq9tW+qYECTf","period":25,"genesis_time":1590445175,"hash":"E4oySqZUD5PQ2tACqolFSxvsK26UhoLN5r1NtA9LfJs="}` +var DrandChain = `{"public_key":"922a2e93828ff83345bae533f5172669a26c02dc76d6bf59c80892e12ab1455c229211886f35bb56af6d5bea981024df","period":25,"genesis_time":1590445175,"hash":"138a324aa6540f93d0dad002aa89454b1bec2b6e948682cde6bd4db40f4b7c9b"}` diff --git a/chain/beacon/drand/drand_test.go b/chain/beacon/drand/drand_test.go index b8b089732..85b80cbe6 100644 --- a/chain/beacon/drand/drand_test.go +++ b/chain/beacon/drand/drand_test.go @@ -1,8 +1,7 @@ package drand import ( - "encoding/json" - "fmt" + "os" "testing" dchain "github.com/drand/drand/chain" @@ -18,7 +17,6 @@ func TestPrintGroupInfo(t *testing.T) { }) chain, err := cg.FetchChainInfo(nil) assert.NoError(t, err) - cbytes, err := json.Marshal(chain.ToProto()) + err = chain.ToJSON(os.Stdout) assert.NoError(t, err) - fmt.Printf("%s\n", cbytes) } diff --git a/go.sum b/go.sum index 155a60127..7134301be 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,7 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= @@ -29,6 +30,7 @@ github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -205,7 +207,6 @@ github.com/filecoin-project/sector-storage v0.0.0-20200522011946-a59ca7536a95/go github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= -github.com/filecoin-project/specs-actors v0.4.1-0.20200509020627-3c96f54f3d7d/go.mod h1:UW3ft23q6VS8wQoNqLWjENsu9gu1uh6lxOd+H8cwhT8= github.com/filecoin-project/specs-actors v0.5.2/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= github.com/filecoin-project/specs-actors v0.5.4-0.20200521014528-0df536f7e461 h1:yyAoJ9dNboljDWj0uBdJEbdaTak/YhkjYUQt0GzlY0A= github.com/filecoin-project/specs-actors v0.5.4-0.20200521014528-0df536f7e461/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= @@ -534,6 +535,7 @@ github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZl github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= @@ -963,6 +965,7 @@ github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c h1:5bFTChQxSKNwy8ALwOebjekYExl9HTT9urdawqC95tA= github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c/go.mod h1:7qN3Y0BvzRUf4LofcoJplQL10lsFDb4PYlePTVwrP28= +github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= @@ -1124,7 +1127,9 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= From aa682a377d0c26a631cd362ba473193add046e63 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 27 May 2020 15:16:37 +0200 Subject: [PATCH 004/160] Partially fix pond spawn Signed-off-by: Jakub Sztandera --- lotuspond/spawn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lotuspond/spawn.go b/lotuspond/spawn.go index 4d8b8b9fd..5b0f7f704 100644 --- a/lotuspond/spawn.go +++ b/lotuspond/spawn.go @@ -171,7 +171,7 @@ func (api *api) SpawnStorage(fullNodeRepo string) (nodeInfo, error) { initArgs := []string{"init", "--nosync"} if fullNodeRepo == api.running[1].meta.Repo { presealPrefix := filepath.Join(fullNodeRepo, "preseal") - initArgs = []string{"init", "--actor=t01000", "--genesis-miner", "--pre-sealed-sectors=" + presealPrefix, "--pre-sealed-metadata=" + filepath.Join(presealPrefix, "pre-seal-t0101.json")} + initArgs = []string{"init", "--actor=t01000", "--genesis-miner", "--pre-sealed-sectors=" + presealPrefix, "--pre-sealed-metadata=" + filepath.Join(presealPrefix, "pre-seal-t01000.json")} } id := atomic.AddInt32(&api.cmds, 1) From 3f9cde28fc95fb8c367ec6e07e376588237953a9 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 27 May 2020 16:16:29 +0200 Subject: [PATCH 005/160] Update go-log, reduce drand entry log level, add TAG env for dev scripts Signed-off-by: Jakub Sztandera --- chain/beacon/drand/drand.go | 2 +- go.mod | 2 +- go.sum | 2 ++ scripts/dev/gen-daemon | 9 +++++---- scripts/dev/sminer-init | 5 ++++- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/chain/beacon/drand/drand.go b/chain/beacon/drand/drand.go index 39c185ee7..f5d6b2b41 100644 --- a/chain/beacon/drand/drand.go +++ b/chain/beacon/drand/drand.go @@ -106,7 +106,7 @@ func (db *DrandBeacon) Entry(ctx context.Context, round uint64) <-chan beacon.Re } go func() { - log.Warnw("fetching randomness", "round", round) + log.Infow("fetching randomness", "round", round) resp, err := db.client.Get(ctx, round) var br beacon.Response diff --git a/go.mod b/go.mod index 3a4912c92..b8fa35330 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( github.com/ipfs/go-ipld-cbor v0.0.5-0.20200428170625-a0bd04d3cbdf github.com/ipfs/go-ipld-format v0.2.0 github.com/ipfs/go-log v1.0.4 - github.com/ipfs/go-log/v2 v2.1.0 + github.com/ipfs/go-log/v2 v2.1.1 github.com/ipfs/go-merkledag v0.3.1 github.com/ipfs/go-path v0.0.7 github.com/ipfs/go-unixfs v0.2.4 diff --git a/go.sum b/go.sum index 7134301be..61e846c3d 100644 --- a/go.sum +++ b/go.sum @@ -480,6 +480,8 @@ github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscw github.com/ipfs/go-log/v2 v2.0.8/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= github.com/ipfs/go-log/v2 v2.1.0 h1:7l5yjP7aLl5xr7RmqEvPZy9Skq7PP2fIP0Pa7lU/UII= github.com/ipfs/go-log/v2 v2.1.0/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= +github.com/ipfs/go-log/v2 v2.1.1 h1:G4TtqN+V9y9HY9TA6BwbCVyyBZ2B9MbCjR2MtGx8FR0= +github.com/ipfs/go-log/v2 v2.1.1/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/ipfs/go-merkledag v0.0.3/go.mod h1:Oc5kIXLHokkE1hWGMBHw+oxehkAaTOqtEb7Zbh6BhLA= github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto= github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= diff --git a/scripts/dev/gen-daemon b/scripts/dev/gen-daemon index ba896b5d6..62a07595d 100755 --- a/scripts/dev/gen-daemon +++ b/scripts/dev/gen-daemon @@ -3,8 +3,9 @@ set -o xtrace export TRUST_PARAMS=1 +tag=${TAG:-debug} -go run -tags=debug ./cmd/lotus-seed pre-seal --sector-size 2048 --num-sectors 2 -go run -tags=debug ./cmd/lotus-seed genesis new localnet.json -go run -tags=debug ./cmd/lotus-seed genesis add-miner localnet.json ~/.genesis-sectors/pre-seal-t01000.json -go run -tags=debug ./cmd/lotus daemon --lotus-make-genesis=devel.gen --genesis-template=localnet.json --bootstrap=false +go run -tags=$tag ./cmd/lotus-seed pre-seal --sector-size 2048 --num-sectors 2 +go run -tags=$tag ./cmd/lotus-seed genesis new localnet.json +go run -tags=$tag ./cmd/lotus-seed genesis add-miner localnet.json ~/.genesis-sectors/pre-seal-t01000.json +go run -tags=$tag ./cmd/lotus daemon --lotus-make-genesis=devel.gen --genesis-template=localnet.json --bootstrap=false diff --git a/scripts/dev/sminer-init b/scripts/dev/sminer-init index ce38d5ba2..2f4a3f7af 100755 --- a/scripts/dev/sminer-init +++ b/scripts/dev/sminer-init @@ -4,4 +4,7 @@ set -o xtrace export TRUST_PARAMS=1 -go run -tags=debug ./cmd/lotus-storage-miner init --actor=t0101 --genesis-miner --pre-sealed-sectors=~/.genesis-sectors --pre-sealed-metadata=~/.genesis-sectors/pre-seal-t0101.json +tag=${TAG:-debug} + +go run -tags=$tag ./cmd/lotus wallet import ~/.genesis-sectors/pre-seal-t01000.key +go run -tags=$tag ./cmd/lotus-storage-miner init --actor=t01000 --genesis-miner --pre-sealed-sectors=~/.genesis-sectors --pre-sealed-metadata=~/.genesis-sectors/pre-seal-t01000.json From 4604b60fb12a56b7ccddda713a5a0a2e28f89529 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 27 May 2020 20:24:26 +0200 Subject: [PATCH 006/160] Remove drand on miner side, MinerGetBaseInfo provides the same info Signed-off-by: Jakub Sztandera --- chain/beacon/drand/drand.go | 5 +++-- cmd/lotus-storage-miner/init.go | 13 +------------ miner/miner.go | 13 +++---------- miner/testminer.go | 6 ++---- node/modules/storageminer.go | 4 ++-- 5 files changed, 11 insertions(+), 30 deletions(-) diff --git a/chain/beacon/drand/drand.go b/chain/beacon/drand/drand.go index f5d6b2b41..d8df81f03 100644 --- a/chain/beacon/drand/drand.go +++ b/chain/beacon/drand/drand.go @@ -106,7 +106,8 @@ func (db *DrandBeacon) Entry(ctx context.Context, round uint64) <-chan beacon.Re } go func() { - log.Infow("fetching randomness", "round", round) + start := time.Now() + log.Infow("start fetching randomness", "round", round) resp, err := db.client.Get(ctx, round) var br beacon.Response @@ -116,7 +117,7 @@ func (db *DrandBeacon) Entry(ctx context.Context, round uint64) <-chan beacon.Re br.Entry.Round = resp.Round() br.Entry.Data = resp.Signature() } - + log.Infow("done fetching randomness", "round", round, "took", time.Since(start)) out <- br close(out) }() diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index f834a09a5..c28e9548c 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -37,7 +37,6 @@ import ( lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/beacon/drand" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/genesis" @@ -446,17 +445,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, return err } - gen, err := api.ChainGetGenesis(ctx) - if err != nil { - return err - } - - beacon, err := drand.NewDrandBeacon(gen.Blocks()[0].Timestamp, build.BlockDelay) - if err != nil { - return err - } - - m := miner.NewMiner(api, epp, beacon, a) + m := miner.NewMiner(api, epp, a) { if err := m.Start(ctx); err != nil { return xerrors.Errorf("failed to start up genesis miner: %w", err) diff --git a/miner/miner.go b/miner/miner.go index cef3518aa..31264ec50 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -15,7 +15,6 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -31,7 +30,7 @@ var log = logging.Logger("miner") // returns a callback reporting whether we mined a blocks in this round type waitFunc func(ctx context.Context, baseTime uint64) (func(bool), error) -func NewMiner(api api.FullNode, epp gen.WinningPoStProver, beacon beacon.RandomBeacon, addr address.Address) *Miner { +func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address) *Miner { arc, err := lru.NewARC(10000) if err != nil { panic(err) @@ -40,7 +39,6 @@ func NewMiner(api api.FullNode, epp gen.WinningPoStProver, beacon beacon.RandomB return &Miner{ api: api, epp: epp, - beacon: beacon, address: addr, waitFunc: func(ctx context.Context, baseTime uint64) (func(bool), error) { // Wait around for half the block time in case other parents come in @@ -56,8 +54,7 @@ func NewMiner(api api.FullNode, epp gen.WinningPoStProver, beacon beacon.RandomB type Miner struct { api api.FullNode - epp gen.WinningPoStProver - beacon beacon.RandomBeacon + epp gen.WinningPoStProver lk sync.Mutex address address.Address @@ -267,12 +264,8 @@ func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*types.BlockMsg, beaconPrev := mbi.PrevBeaconEntry - bvals, err := beacon.BeaconEntriesForBlock(ctx, m.beacon, round, beaconPrev) - if err != nil { - return nil, xerrors.Errorf("get beacon entries failed: %w", err) - } - tDrand := time.Now() + bvals := mbi.BeaconEntries hasPower, err := m.hasPower(ctx, m.address, base.TipSet) if err != nil { diff --git a/miner/testminer.go b/miner/testminer.go index 3eab836c5..1576f2779 100644 --- a/miner/testminer.go +++ b/miner/testminer.go @@ -5,20 +5,18 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/gen" lru "github.com/hashicorp/golang-lru" ) -func NewTestMiner(nextCh <-chan func(bool), addr address.Address) func(api.FullNode, gen.WinningPoStProver, beacon.RandomBeacon) *Miner { - return func(api api.FullNode, epp gen.WinningPoStProver, b beacon.RandomBeacon) *Miner { +func NewTestMiner(nextCh <-chan func(bool), addr address.Address) func(api.FullNode, gen.WinningPoStProver) *Miner { + return func(api api.FullNode, epp gen.WinningPoStProver) *Miner { arc, err := lru.NewARC(10000) if err != nil { panic(err) } m := &Miner{ - beacon: b, api: api, waitFunc: chanWaiter(nextCh), epp: epp, diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index f54a2672e..ee3fc9a1a 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -263,13 +263,13 @@ func StagingGraphsync(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.Stagi return gs } -func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api lapi.FullNode, epp gen.WinningPoStProver, beacon beacon.RandomBeacon) (*miner.Miner, error) { +func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api lapi.FullNode, epp gen.WinningPoStProver) (*miner.Miner, error) { minerAddr, err := minerAddrFromDS(ds) if err != nil { return nil, err } - m := miner.NewMiner(api, epp, beacon, minerAddr) + m := miner.NewMiner(api, epp, minerAddr) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { From d6615b6286fe803a0bc6938e800e141cdb66c791 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 27 May 2020 22:53:20 +0200 Subject: [PATCH 007/160] Cleanup many lint warnings Signed-off-by: Jakub Sztandera --- .golangci.yml | 12 +- api/api_worker.go | 2 +- api/apistruct/permissioned.go | 2 +- api/apistruct/struct.go | 2 +- api/client/client.go | 2 +- api/docgen/docgen.go | 1 + api/test/deals.go | 2 +- api/types.go | 1 + build/parameters.go | 2 +- build/version.go | 1 + chain/actors/aerrors/wrap.go | 1 + chain/blocksync/blocksync.go | 4 +- chain/blocksync/blocksync_client.go | 6 +- chain/events/events.go | 2 +- chain/gen/genesis/miners.go | 3 + chain/state/statetree_test.go | 5 +- chain/stmgr/stmgr.go | 4 +- chain/store/store.go | 10 +- chain/sync_test.go | 4 +- chain/types/blockheader_test.go | 7 +- chain/vectors/gen/main.go | 9 +- chain/vm/runtime.go | 11 +- chain/vm/syscalls.go | 2 +- chain/vm/validation_test.go | 2 + chain/vm/vm.go | 4 +- chain/wallet/wallet.go | 4 +- cli/chain.go | 7 +- cli/multisig.go | 5 +- cli/params.go | 2 +- cli/paych.go | 2 +- cmd/chain-noise/main.go | 3 +- cmd/lotus-bench/import.go | 6 +- cmd/lotus-bench/main.go | 19 +- cmd/lotus-chainwatch/main.go | 2 +- cmd/lotus-chainwatch/sync.go | 2 + cmd/lotus-fountain/main.go | 6 +- cmd/lotus-health/main.go | 2 +- cmd/lotus-seal-worker/main.go | 8 +- cmd/lotus-seed/main.go | 1 - cmd/lotus-seed/seed/seed.go | 4 +- cmd/lotus-shed/import-car.go | 2 +- cmd/lotus-shed/params.go | 2 +- cmd/lotus-storage-miner/init.go | 4 +- cmd/lotus/daemon.go | 4 +- cmd/lotus/rpc.go | 4 +- extern/rleplus/internal/bitvector.go | 154 ---------------- extern/rleplus/internal/bitvector_test.go | 136 -------------- extern/rleplus/rleplus.go | 204 --------------------- extern/rleplus/rleplus_test.go | 213 ---------------------- genesis/types.go | 2 +- lib/addrutil/parse.go | 2 +- lib/increadtimeout/incrt.go | 2 +- lib/lotuslog/levels.go | 1 + markets/utils/converters.go | 2 +- node/config/storage.go | 3 +- node/hello/hello.go | 10 +- node/impl/client/client.go | 4 +- node/impl/full/chain.go | 2 +- node/impl/remoteworker.go | 2 +- node/modules/core.go | 2 +- node/modules/lp2p/addrs.go | 2 +- node/modules/storageminer.go | 14 +- node/node_test.go | 6 +- node/repo/fsrepo.go | 3 +- paychmgr/store.go | 4 +- storage/miner.go | 5 +- 66 files changed, 156 insertions(+), 807 deletions(-) delete mode 100644 extern/rleplus/internal/bitvector.go delete mode 100644 extern/rleplus/internal/bitvector_test.go delete mode 100644 extern/rleplus/rleplus.go delete mode 100644 extern/rleplus/rleplus_test.go diff --git a/.golangci.yml b/.golangci.yml index 396a1c2ac..c8805a7a3 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -22,13 +22,23 @@ issues: - "func name will be used as test\\.Test.* by other packages, and that stutters; consider calling this" - "Potential file inclusion via variable" - "should have( a package)? comment" + - "Error return value of `logging.SetLogLevel` is not checked" exclude-use-default: false exclude-rules: + - path: lotuspond + linters: + - errcheck + - path: node/modules/lp2p linters: - golint - - path: ".*_test.go" + + - path: build/params_.*\.go + linters: + - golint + + - path: .*_test.go linters: - gosec diff --git a/api/api_worker.go b/api/api_worker.go index dbad20651..d5fea5d6c 100644 --- a/api/api_worker.go +++ b/api/api_worker.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lotus/build" ) -type WorkerApi interface { +type WorkerAPI interface { Version(context.Context) (build.Version, error) // TODO: Info() (name, ...) ? diff --git a/api/apistruct/permissioned.go b/api/apistruct/permissioned.go index 8ffc49bf9..c93662733 100644 --- a/api/apistruct/permissioned.go +++ b/api/apistruct/permissioned.go @@ -31,7 +31,7 @@ func PermissionedFullAPI(a api.FullNode) api.FullNode { return &out } -func PermissionedWorkerAPI(a api.WorkerApi) api.WorkerApi { +func PermissionedWorkerAPI(a api.WorkerAPI) api.WorkerAPI { var out WorkerStruct auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal) return &out diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 368a2b145..8a3237736 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -865,4 +865,4 @@ func (w *WorkerStruct) Closing(ctx context.Context) (<-chan struct{}, error) { var _ api.Common = &CommonStruct{} var _ api.FullNode = &FullNodeStruct{} var _ api.StorageMiner = &StorageMinerStruct{} -var _ api.WorkerApi = &WorkerStruct{} +var _ api.WorkerAPI = &WorkerStruct{} diff --git a/api/client/client.go b/api/client/client.go index d3eceb24a..20bad2048 100644 --- a/api/client/client.go +++ b/api/client/client.go @@ -48,7 +48,7 @@ func NewStorageMinerRPC(addr string, requestHeader http.Header) (api.StorageMine return &res, closer, err } -func NewWorkerRPC(addr string, requestHeader http.Header) (api.WorkerApi, jsonrpc.ClientCloser, error) { +func NewWorkerRPC(addr string, requestHeader http.Header) (api.WorkerAPI, jsonrpc.ClientCloser, error) { var res apistruct.WorkerStruct closer, err := jsonrpc.NewMergeClient(addr, "Filecoin", []interface{}{ diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index 0ad3b02bb..f2d4d6ce4 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -320,6 +320,7 @@ func main() { }) for _, g := range groupslice { + g := g fmt.Printf("## %s\n", g.GroupName) fmt.Printf("%s\n\n", g.Header) diff --git a/api/test/deals.go b/api/test/deals.go index 2414d3fe2..98edce67a 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -193,7 +193,7 @@ func testRetrieval(t *testing.T, ctx context.Context, err error, client *impl.Fu if err != nil { t.Fatal(err) } - defer os.RemoveAll(rpath) + defer os.RemoveAll(rpath) //nolint:errcheck caddr, err := client.WalletDefaultAddress(ctx) if err != nil { diff --git a/api/types.go b/api/types.go index 732f1f328..26d0695e1 100644 --- a/api/types.go +++ b/api/types.go @@ -7,6 +7,7 @@ import ( ) // TODO: check if this exists anywhere else + type MultiaddrSlice []ma.Multiaddr func (m *MultiaddrSlice) UnmarshalJSON(raw []byte) (err error) { diff --git a/build/parameters.go b/build/parameters.go index b7fac93d1..7d34a7831 100644 --- a/build/parameters.go +++ b/build/parameters.go @@ -2,6 +2,6 @@ package build import rice "github.com/GeertJohan/go.rice" -func ParametersJson() []byte { +func ParametersJSON() []byte { return rice.MustFindBox("proof-params").MustBytes("parameters.json") } diff --git a/build/version.go b/build/version.go index c5d96e9af..e0ddebf36 100644 --- a/build/version.go +++ b/build/version.go @@ -33,6 +33,7 @@ func (ve Version) EqMajorMinor(v2 Version) bool { // APIVersion is a semver version of the rpc api exposed var APIVersion Version = newVer(0, 3, 0) +//nolint:varcheck const ( majorMask = 0xff0000 minorMask = 0xffff00 diff --git a/chain/actors/aerrors/wrap.go b/chain/actors/aerrors/wrap.go index 2e7444101..338659966 100644 --- a/chain/actors/aerrors/wrap.go +++ b/chain/actors/aerrors/wrap.go @@ -50,6 +50,7 @@ func Newf(retCode exitcode.ExitCode, format string, args ...interface{}) ActorEr } // todo: bit hacky + func NewfSkip(skip int, retCode exitcode.ExitCode, format string, args ...interface{}) ActorError { if retCode == 0 { return &actorError{ diff --git a/chain/blocksync/blocksync.go b/chain/blocksync/blocksync.go index ccb7a5498..daca9ce20 100644 --- a/chain/blocksync/blocksync.go +++ b/chain/blocksync/blocksync.go @@ -91,7 +91,7 @@ func (bss *BlockSyncService) HandleStream(s inet.Stream) { ctx, span := trace.StartSpan(context.Background(), "blocksync.HandleStream") defer span.End() - defer s.Close() + defer s.Close() //nolint:errcheck var req BlockSyncRequest if err := cborutil.ReadCborRPC(bufio.NewReader(s), &req); err != nil { @@ -107,7 +107,7 @@ func (bss *BlockSyncService) HandleStream(s inet.Stream) { } writeDeadline := 60 * time.Second - s.SetDeadline(time.Now().Add(writeDeadline)) + _ = s.SetDeadline(time.Now().Add(writeDeadline)) if err := cborutil.WriteCborRPC(s, resp); err != nil { log.Warnw("failed to write back response for handle stream", "err", err, "peer", s.Conn().RemotePeer()) return diff --git a/chain/blocksync/blocksync_client.go b/chain/blocksync/blocksync_client.go index 0790cb128..f6453efde 100644 --- a/chain/blocksync/blocksync_client.go +++ b/chain/blocksync/blocksync_client.go @@ -283,14 +283,14 @@ func (bs *BlockSync) fetchBlocksBlockSync(ctx context.Context, p peer.ID, req *B bs.RemovePeer(p) return nil, xerrors.Errorf("failed to open stream to peer: %w", err) } - s.SetWriteDeadline(time.Now().Add(5 * time.Second)) + _ = s.SetWriteDeadline(time.Now().Add(5 * time.Second)) if err := cborutil.WriteCborRPC(s, req); err != nil { - s.SetWriteDeadline(time.Time{}) + _ = s.SetWriteDeadline(time.Time{}) bs.syncPeers.logFailure(p, time.Since(start)) return nil, err } - s.SetWriteDeadline(time.Time{}) + _ = s.SetWriteDeadline(time.Time{}) var res BlockSyncResponse r := incrt.New(s, 50<<10, 5*time.Second) diff --git a/chain/events/events.go b/chain/events/events.go index 02321235b..c6d54d536 100644 --- a/chain/events/events.go +++ b/chain/events/events.go @@ -67,7 +67,7 @@ func NewEvents(ctx context.Context, api eventApi) *Events { heightEvents: heightEvents{ tsc: tsc, ctx: ctx, - gcConfidence: abi.ChainEpoch(gcConfidence), + gcConfidence: gcConfidence, heightTriggers: map[uint64]*heightHandler{}, htTriggerHeights: map[abi.ChainEpoch][]uint64{}, diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index 51a128d54..fa71865b8 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -58,6 +58,8 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid for i, m := range miners { // Create miner through power actor + i := i + m := m spt, err := ffiwrapper.SealProofTypeFromSectorSize(m.SectorSize) if err != nil { @@ -154,6 +156,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid // Commit sectors for pi, preseal := range m.Sectors { + preseal := preseal // TODO: Maybe check seal (Can just be snark inputs, doesn't go into the genesis file) // check deals, get dealWeight diff --git a/chain/state/statetree_test.go b/chain/state/statetree_test.go index 4cdc87d19..3c832f7cc 100644 --- a/chain/state/statetree_test.go +++ b/chain/state/statetree_test.go @@ -255,12 +255,15 @@ func TestStateTreeConsistency(t *testing.T) { } for i, a := range addrs { - st.SetActor(a, &types.Actor{ + err := st.SetActor(a, &types.Actor{ Code: randomCid, Head: randomCid, Balance: types.NewInt(uint64(10000 + i)), Nonce: uint64(1000 - i), }) + if err != nil { + t.Fatalf("while setting actor: %+v", err) + } } root, err := st.Flush(context.TODO()) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index e291fe623..5a1eb88ea 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -382,8 +382,8 @@ func (sm *StateManager) LoadActorState(ctx context.Context, a address.Address, o cst := cbor.NewCborStore(sm.cs.Blockstore()) if err := cst.Get(ctx, act.Head, out); err != nil { var r cbg.Deferred - cst.Get(ctx, act.Head, &r) - fmt.Printf("badhead %x\n", r.Raw) + _ = cst.Get(ctx, act.Head, &r) + log.Errorw("bad actor head", "error", err, "raw", r.Raw, "address", a) return nil, err } diff --git a/chain/store/store.go b/chain/store/store.go index 1d0668032..e8c04b11b 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -895,11 +895,17 @@ func DrawRandomness(rbase []byte, pers crypto.DomainSeparationTag, round abi.Cha return nil, xerrors.Errorf("deriving randomness: %w", err) } VRFDigest := blake2b.Sum256(rbase) - h.Write(VRFDigest[:]) + _, err := h.Write(VRFDigest[:]) + if err != nil { + return nil, xerrors.Errorf("hashing VRFDigest: %w", err) + } if err := binary.Write(h, binary.BigEndian, round); err != nil { return nil, xerrors.Errorf("deriving randomness: %w", err) } - h.Write(entropy) + _, err = h.Write(entropy) + if err != nil { + return nil, xerrors.Errorf("hashing entropy: %w", err) + } return h.Sum(nil), nil } diff --git a/chain/sync_test.go b/chain/sync_test.go index 9c7d47bc6..e1c846d4e 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -417,7 +417,7 @@ func TestSyncBadTimestamp(t *testing.T) { a1 := tu.mineOnBlock(base, 0, nil, false, true) tu.g.Timestamper = nil - tu.g.ResyncBankerNonce(a1.TipSet()) + require.NoError(t, tu.g.ResyncBankerNonce(a1.TipSet())) fmt.Println("After mine bad block!") tu.printHeads() @@ -479,7 +479,7 @@ func TestSyncFork(t *testing.T) { a := tu.mineOnBlock(a1, p1, []int{0}, true, false) a = tu.mineOnBlock(a, p1, []int{0}, true, false) - tu.g.ResyncBankerNonce(a1.TipSet()) + require.NoError(t, tu.g.ResyncBankerNonce(a1.TipSet())) // chain B will now be heaviest b := tu.mineOnBlock(base, p2, []int{1}, true, false) b = tu.mineOnBlock(b, p2, []int{1}, true, false) diff --git a/chain/types/blockheader_test.go b/chain/types/blockheader_test.go index e18e4028a..387fe86c2 100644 --- a/chain/types/blockheader_test.go +++ b/chain/types/blockheader_test.go @@ -4,13 +4,14 @@ import ( "bytes" "encoding/hex" "fmt" - "github.com/filecoin-project/specs-actors/actors/abi" "reflect" "testing" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/specs-actors/actors/crypto" cid "github.com/ipfs/go-cid" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/crypto" ) func testBlockHeader(t testing.TB) *BlockHeader { diff --git a/chain/vectors/gen/main.go b/chain/vectors/gen/main.go index cf8ebd859..2ebcb9a60 100644 --- a/chain/vectors/gen/main.go +++ b/chain/vectors/gen/main.go @@ -7,6 +7,7 @@ import ( "os" "github.com/filecoin-project/go-address" + "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/types" @@ -173,14 +174,18 @@ func WriteJsonToFile(fname string, obj interface{}) error { if err != nil { return err } - defer fi.Close() + defer fi.Close() //nolint:errcheck out, err := json.MarshalIndent(obj, "", " ") if err != nil { return err } - fi.Write(out) + _, err = fi.Write(out) + if err != nil { + return xerrors.Errorf("writing json: %w", err) + } + return nil } diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index ebaf48f59..403a92fa7 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -5,6 +5,7 @@ import ( "context" "encoding/binary" "fmt" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/abi/big" @@ -182,7 +183,7 @@ func (rs *Runtime) ValidateImmediateCallerAcceptAny() { func (rs *Runtime) CurrentBalance() abi.TokenAmount { b, err := rs.GetBalance(rs.Message().Receiver()) if err != nil { - rs.Abortf(exitcode.ExitCode(err.RetCode()), "get current balance: %v", err) + rs.Abortf(err.RetCode(), "get current balance: %v", err) } return b } @@ -354,13 +355,13 @@ func (rs *Runtime) Send(to address.Address, method abi.MethodNum, m vmr.CBORMars params = buf.Bytes() } - ret, err := rs.internalSend(rs.Message().Receiver(), to, method, types.BigInt(value), params) + ret, err := rs.internalSend(rs.Message().Receiver(), to, method, value, params) if err != nil { if err.IsFatal() { panic(err) } log.Warnf("vmctx send failed: to: %s, method: %d: ret: %d, err: %s", to, method, ret, err) - return nil, exitcode.ExitCode(err.RetCode()) + return nil, err.RetCode() } return &dumbWrapperType{ret}, 0 } @@ -399,7 +400,7 @@ func (rt *Runtime) internalSend(from, to address.Address, method abi.MethodNum, } mr := types.MessageReceipt{ - ExitCode: exitcode.ExitCode(aerrors.RetCode(errSend)), + ExitCode: aerrors.RetCode(errSend), Return: ret, GasUsed: 0, } @@ -431,6 +432,7 @@ type shimStateHandle struct { func (ssh *shimStateHandle) Create(obj vmr.CBORMarshaler) { c := ssh.rs.Put(obj) + // TODO: handle error below ssh.rs.stateCommit(EmptyObjectCid, c) } @@ -460,6 +462,7 @@ func (ssh *shimStateHandle) Transaction(obj vmr.CBORer, f func() interface{}) in c := ssh.rs.Put(obj) + // TODO: handle error below ssh.rs.stateCommit(baseState, c) return out diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index c3fe4375f..8039d2b2a 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -225,7 +225,7 @@ func (ss *syscallShim) VerifySeal(info abi.SealVerifyInfo) error { } ticket := []byte(info.Randomness) - proof := []byte(info.Proof) + proof := info.Proof seed := []byte(info.InteractiveRandomness) log.Debugf("Verif r:%x; d:%x; m:%s; t:%x; s:%x; N:%d; p:%x", info.SealedCID, info.UnsealedCID, miner, ticket, seed, info.SectorID.Number, proof) diff --git a/chain/vm/validation_test.go b/chain/vm/validation_test.go index c84cb4adc..880b33401 100644 --- a/chain/vm/validation_test.go +++ b/chain/vm/validation_test.go @@ -41,6 +41,7 @@ func init() { func TestChainValidationMessageSuite(t *testing.T) { f := factory.NewFactories() for _, testCase := range suites.MessageTestCases() { + testCase := testCase if TestSuiteSkipper.Skip(testCase) { continue } @@ -53,6 +54,7 @@ func TestChainValidationMessageSuite(t *testing.T) { func TestChainValidationTipSetSuite(t *testing.T) { f := factory.NewFactories() for _, testCase := range suites.TipSetTestCases() { + testCase := testCase if TestSuiteSkipper.Skip(testCase) { continue } diff --git a/chain/vm/vm.go b/chain/vm/vm.go index dd96f3928..49294e9cf 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -246,7 +246,7 @@ func (vm *VM) ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*Ap ret, actorErr, rt := vm.send(ctx, msg, nil, 0) return &ApplyRet{ MessageReceipt: types.MessageReceipt{ - ExitCode: exitcode.ExitCode(aerrors.RetCode(actorErr)), + ExitCode: aerrors.RetCode(actorErr), Return: ret, GasUsed: 0, }, @@ -415,7 +415,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, return &ApplyRet{ MessageReceipt: types.MessageReceipt{ - ExitCode: exitcode.ExitCode(errcode), + ExitCode: errcode, Return: ret, GasUsed: gasUsed, }, diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 54587e1b5..7ad37a205 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -12,8 +12,8 @@ import ( "github.com/filecoin-project/go-address" - _ "github.com/filecoin-project/lotus/lib/sigs/bls" - _ "github.com/filecoin-project/lotus/lib/sigs/secp" + _ "github.com/filecoin-project/lotus/lib/sigs/bls" // enable bls signatures + _ "github.com/filecoin-project/lotus/lib/sigs/secp" // enable secp signatures "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sigs" diff --git a/cli/chain.go b/cli/chain.go index 033c7b234..9e245c875 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -804,7 +804,12 @@ var chainExportCmd = &cli.Command{ if err != nil { return err } - defer fi.Close() + defer func() { + err := fi.Close() + if err != nil { + fmt.Printf("error closing output file: %+v", err) + } + }() ts, err := LoadTipSet(ctx, cctx, api) if err != nil { diff --git a/cli/multisig.go b/cli/multisig.go index ff2ff2fce..9acf1af08 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -206,7 +206,10 @@ var msigInspectCmd = &cli.Command{ tx := pending[txid] fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%d\t%x\n", txid, state(tx), tx.To, types.FIL(tx.Value), tx.Method, tx.Params) } - w.Flush() + if err := w.Flush(); err != nil { + return xerrors.Errorf("flushing ouput: %+v", err) + } + } return nil diff --git a/cli/params.go b/cli/params.go index 518add665..f6acbaad7 100644 --- a/cli/params.go +++ b/cli/params.go @@ -24,7 +24,7 @@ var fetchParamCmd = &cli.Command{ return err } sectorSize := uint64(sectorSizeInt) - err = paramfetch.GetParams(build.ParametersJson(), sectorSize) + err = paramfetch.GetParams(build.ParametersJSON(), sectorSize) if err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/cli/paych.go b/cli/paych.go index e150d6769..5fe5fac0c 100644 --- a/cli/paych.go +++ b/cli/paych.go @@ -370,7 +370,7 @@ var paychVoucherSubmitCmd = &cli.Command{ return fmt.Errorf("message execution failed (exit code %d)", mwait.Receipt.ExitCode) } - fmt.Println("channel updated succesfully") + fmt.Println("channel updated successfully") return nil }, diff --git a/cmd/chain-noise/main.go b/cmd/chain-noise/main.go index dc64b70ed..e152ccf75 100644 --- a/cmd/chain-noise/main.go +++ b/cmd/chain-noise/main.go @@ -53,8 +53,7 @@ var runCmd = &cli.Command{ defer closer() ctx := lcli.ReqContext(cctx) - sendSmallFundsTxs(ctx, api, addr, 5) - return nil + return sendSmallFundsTxs(ctx, api, addr, 5) }, } diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 6306cd18c..f33cc1d9d 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -55,7 +55,7 @@ var importBenchCmd = &cli.Command{ if err != nil { return err } - defer cfi.Close() + defer cfi.Close() //nolint:errcheck // read only file tdir, err := ioutil.TempDir("", "lotus-import-bench") if err != nil { @@ -80,7 +80,7 @@ var importBenchCmd = &cli.Command{ if err != nil { return err } - defer prof.Close() + defer prof.Close() //nolint:errcheck if err := pprof.StartCPUProfile(prof); err != nil { return err @@ -146,7 +146,7 @@ var importBenchCmd = &cli.Command{ if err != nil { return err } - defer ibj.Close() + defer ibj.Close() //nolint:errcheck if err := json.NewEncoder(ibj).Encode(out); err != nil { return err diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 23471500d..b8740f2f3 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -141,7 +141,10 @@ var sealBenchCmd = &cli.Command{ }, Action: func(c *cli.Context) error { if c.Bool("no-gpu") { - os.Setenv("BELLMAN_NO_GPU", "1") + err := os.Setenv("BELLMAN_NO_GPU", "1") + if err != nil { + return xerrors.Errorf("setting no-gpu flag: %w", err) + } } robench := c.String("benchmark-existing-sectorbuilder") @@ -154,7 +157,10 @@ var sealBenchCmd = &cli.Command{ return err } - os.MkdirAll(sdir, 0775) + err = os.MkdirAll(sdir, 0775) + if err != nil { + return xerrors.Errorf("creating sectorbuilder dir: %w", err) + } tsdir, err := ioutil.TempDir(sdir, "bench") if err != nil { @@ -209,7 +215,7 @@ var sealBenchCmd = &cli.Command{ // Only fetch parameters if actually needed if !c.Bool("skip-commit2") { - if err := paramfetch.GetParams(build.ParametersJson(), uint64(sectorSize)); err != nil { + if err := paramfetch.GetParams(build.ParametersJSON(), uint64(sectorSize)); err != nil { return xerrors.Errorf("getting params: %w", err) } } @@ -599,7 +605,10 @@ var proveCmd = &cli.Command{ }, Action: func(c *cli.Context) error { if c.Bool("no-gpu") { - os.Setenv("BELLMAN_NO_GPU", "1") + err := os.Setenv("BELLMAN_NO_GPU", "1") + if err != nil { + return xerrors.Errorf("setting no-gpu flag: %w", err) + } } if !c.Args().Present() { @@ -616,7 +625,7 @@ var proveCmd = &cli.Command{ return xerrors.Errorf("unmarshalling input file: %w", err) } - if err := paramfetch.GetParams(build.ParametersJson(), c2in.SectorSize); err != nil { + if err := paramfetch.GetParams(build.ParametersJSON(), c2in.SectorSize); err != nil { return xerrors.Errorf("getting params: %w", err) } diff --git a/cmd/lotus-chainwatch/main.go b/cmd/lotus-chainwatch/main.go index 71722486d..a3114d8d0 100644 --- a/cmd/lotus-chainwatch/main.go +++ b/cmd/lotus-chainwatch/main.go @@ -86,7 +86,7 @@ var runCmd = &cli.Command{ if err != nil { return err } - defer st.close() + defer st.close() //nolint:errcheck runSyncer(ctx, api, st, maxBatch) diff --git a/cmd/lotus-chainwatch/sync.go b/cmd/lotus-chainwatch/sync.go index 8fa903097..a62136ca8 100644 --- a/cmd/lotus-chainwatch/sync.go +++ b/cmd/lotus-chainwatch/sync.go @@ -198,6 +198,8 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS } for a, act := range changes { + act := act + addr, err := address.NewFromString(a) if err != nil { log.Error(err) diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go index b2275b9ca..8fc1804d7 100644 --- a/cmd/lotus-fountain/main.go +++ b/cmd/lotus-fountain/main.go @@ -157,7 +157,7 @@ func (h *handler) send(w http.ResponseWriter, r *http.Request) { to, err := address.NewFromString(r.FormValue("address")) if err != nil { w.WriteHeader(400) - w.Write([]byte(err.Error())) + _, _ = w.Write([]byte(err.Error())) return } @@ -204,11 +204,11 @@ func (h *handler) send(w http.ResponseWriter, r *http.Request) { }) if err != nil { w.WriteHeader(400) - w.Write([]byte(err.Error())) + _, _ = w.Write([]byte(err.Error())) return } - w.Write([]byte(smsg.Cid().String())) + _, _ = w.Write([]byte(smsg.Cid().String())) } func (h *handler) mkminer(w http.ResponseWriter, r *http.Request) { diff --git a/cmd/lotus-health/main.go b/cmd/lotus-health/main.go index 99b5b0129..6f5ce8e90 100644 --- a/cmd/lotus-health/main.go +++ b/cmd/lotus-health/main.go @@ -79,7 +79,7 @@ var watchHeadCmd = &cli.Command{ &cli.IntFlag{ Name: "api-retries", Value: 8, - Usage: "number of API retry attemps", + Usage: "number of API retry attempts", }, }, Action: func(c *cli.Context) error { diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index a2e14a1e6..b43b7267f 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -26,7 +26,7 @@ import ( lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/lotuslog" "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/sector-storage" + sectorstorage "github.com/filecoin-project/sector-storage" "github.com/filecoin-project/sector-storage/sealtasks" "github.com/filecoin-project/sector-storage/stores" ) @@ -107,7 +107,9 @@ var runCmd = &cli.Command{ }, Action: func(cctx *cli.Context) error { if !cctx.Bool("enable-gpu-proving") { - os.Setenv("BELLMAN_NO_GPU", "true") + if err := os.Setenv("BELLMAN_NO_GPU", "true"); err != nil { + return xerrors.Errorf("could not set no-gpu env: %+v", err) + } } if cctx.String("address") == "" { @@ -146,7 +148,7 @@ var runCmd = &cli.Command{ } if cctx.Bool("commit") { - if err := paramfetch.GetParams(build.ParametersJson(), uint64(ssize)); err != nil { + if err := paramfetch.GetParams(build.ParametersJSON(), uint64(ssize)); err != nil { return xerrors.Errorf("get params: %w", err) } } diff --git a/cmd/lotus-seed/main.go b/cmd/lotus-seed/main.go index 4eaeca291..7b80a0479 100644 --- a/cmd/lotus-seed/main.go +++ b/cmd/lotus-seed/main.go @@ -145,7 +145,6 @@ var aggregateManifestsCmd = &cli.Command{ if err != nil { return err } - defer fi.Close() var val map[string]genesis.Miner if err := json.NewDecoder(fi).Decode(&val); err != nil { return err diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index 6e5271f33..0f6539563 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "github.com/minio/blake2b-simd" "io/ioutil" "os" "path/filepath" @@ -15,6 +14,7 @@ import ( logging "github.com/ipfs/go-log/v2" ic "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" + "github.com/minio/blake2b-simd" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -48,7 +48,7 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum SealProofType: spt, } - if err := os.MkdirAll(sbroot, 0775); err != nil { + if err := os.MkdirAll(sbroot, 0775); err != nil { //golint:gosec return nil, nil, err } diff --git a/cmd/lotus-shed/import-car.go b/cmd/lotus-shed/import-car.go index 8933f8890..f2f517a7f 100644 --- a/cmd/lotus-shed/import-car.go +++ b/cmd/lotus-shed/import-car.go @@ -34,7 +34,7 @@ var importCarCmd = &cli.Command{ if err != nil { return err } - defer lr.Close() + defer lr.Close() //nolint:errcheck cf := cctx.Args().Get(0) f, err := os.OpenFile(cf, os.O_RDONLY, 0664) diff --git a/cmd/lotus-shed/params.go b/cmd/lotus-shed/params.go index ff00f2b4f..210711b91 100644 --- a/cmd/lotus-shed/params.go +++ b/cmd/lotus-shed/params.go @@ -24,7 +24,7 @@ var fetchParamCmd = &cli.Command{ return err } sectorSize := uint64(sectorSizeInt) - err = paramfetch.GetParams(build.ParametersJson(), sectorSize) + err = paramfetch.GetParams(build.ParametersJSON(), sectorSize) if err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index f834a09a5..7a6ac8257 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -127,7 +127,7 @@ var initCmd = &cli.Command{ } log.Info("Checking proof parameters") - if err := paramfetch.GetParams(build.ParametersJson(), uint64(ssize)); err != nil { + if err := paramfetch.GetParams(build.ParametersJSON(), uint64(ssize)); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } @@ -389,7 +389,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, if err != nil { return err } - defer lr.Close() + defer lr.Close() //notlint:errcheck log.Info("Initializing libp2p identity") diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 91af88a4e..9e4e7d1a3 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -134,7 +134,7 @@ var DaemonCmd = &cli.Command{ return xerrors.Errorf("repo init error: %w", err) } - if err := paramfetch.GetParams(build.ParametersJson(), 0); err != nil { + if err := paramfetch.GetParams(build.ParametersJSON(), 0); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } @@ -269,7 +269,7 @@ func ImportChain(r repo.Repo, fname string) error { if err != nil { return err } - defer lr.Close() + defer lr.Close() //nolint:errcheck ds, err := lr.Datastore("/blocks") if err != nil { diff --git a/cmd/lotus/rpc.go b/cmd/lotus/rpc.go index 7b1fc7286..d053e2a2e 100644 --- a/cmd/lotus/rpc.go +++ b/cmd/lotus/rpc.go @@ -85,14 +85,14 @@ func handleImport(a *impl.FullNodeAPI) func(w http.ResponseWriter, r *http.Reque } if !auth.HasPerm(r.Context(), nil, apistruct.PermWrite) { w.WriteHeader(401) - json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing write permission"}) + _ = json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing write permission"}) return } c, err := a.ClientImportLocal(r.Context(), r.Body) if err != nil { w.WriteHeader(500) - json.NewEncoder(w).Encode(struct{ Error string }{err.Error()}) + _ = json.NewEncoder(w).Encode(struct{ Error string }{err.Error()}) return } w.WriteHeader(200) diff --git a/extern/rleplus/internal/bitvector.go b/extern/rleplus/internal/bitvector.go deleted file mode 100644 index 65bae074e..000000000 --- a/extern/rleplus/internal/bitvector.go +++ /dev/null @@ -1,154 +0,0 @@ -package bitvector - -import ( - "errors" - "log" -) - -var ( - // ErrOutOfRange - the index passed is out of range for the BitVector - ErrOutOfRange = errors.New("index out of range") -) - -// BitNumbering indicates the ordering of bits, either -// least-significant bit in position 0, or most-significant bit -// in position 0. -// -// It it used in 3 ways with BitVector: -// 1. Ordering of bits within the Buf []byte structure -// 2. What order to add bits when using Extend() -// 3. What order to read bits when using Take() -// -// https://en.wikipedia.org/wiki/Bit_numbering -type BitNumbering int - -const ( - // LSB0 - bit ordering starts with the low-order bit - LSB0 BitNumbering = iota - - // MSB0 - bit ordering starts with the high-order bit - MSB0 -) - -// BitVector is used to manipulate ordered collections of bits -type BitVector struct { - Buf []byte - - // BytePacking is the bit ordering within bytes - BytePacking BitNumbering - - // Len is the logical number of bits in the vector. - // The last byte in Buf may have undefined bits if Len is not a multiple of 8 - Len uint -} - -// NewBitVector constructs a new BitVector from a slice of bytes. -// -// The bytePacking parameter is required to know how to interpret the bit ordering within the bytes. -func NewBitVector(buf []byte, bytePacking BitNumbering) *BitVector { - return &BitVector{ - BytePacking: bytePacking, - Buf: buf, - Len: uint(len(buf) * 8), - } -} - -// Push adds a single bit to the BitVector. -// -// Although it takes a byte, only the low-order bit is used, so just use 0 or 1. -func (v *BitVector) Push(val byte) { - if v.Len%8 == 0 { - v.Buf = append(v.Buf, 0) - } - lastIdx := v.Len / 8 - - switch v.BytePacking { - case LSB0: - v.Buf[lastIdx] |= (val & 1) << (v.Len % 8) - default: - v.Buf[lastIdx] |= (val & 1) << (7 - (v.Len % 8)) - } - - v.Len++ -} - -// Get returns a single bit as a byte -- either 0 or 1 -func (v *BitVector) Get(idx uint) (byte, error) { - if idx >= v.Len { - return 0, ErrOutOfRange - } - blockIdx := idx / 8 - - switch v.BytePacking { - case LSB0: - return v.Buf[blockIdx] >> (idx % 8) & 1, nil - default: - return v.Buf[blockIdx] >> (7 - idx%8) & 1, nil - } -} - -// Extend adds up to 8 bits to the receiver -// -// Given a byte b == 0b11010101 -// v.Extend(b, 4, LSB0) would add < 1, 0, 1, 0 > -// v.Extend(b, 4, MSB0) would add < 1, 1, 0, 1 > -// -// Panics if count is out of range -func (v *BitVector) Extend(val byte, count uint, order BitNumbering) { - if count > 8 { - log.Panicf("invalid count") - } - - for i := uint(0); i < count; i++ { - switch order { - case LSB0: - v.Push((val >> i) & 1) - default: - v.Push((val >> (7 - i)) & 1) - } - } -} - -// Take reads up to 8 bits at the given index. -// -// Given a BitVector < 1, 1, 0, 1, 0, 1, 0, 1 > -// v.Take(0, 4, LSB0) would return 0b00001011 -// v.Take(0, 4, MSB0) would return 0b11010000 -// -// Panics if count is out of range -func (v *BitVector) Take(index uint, count uint, order BitNumbering) (out byte) { - if count > 8 { - log.Panicf("invalid count") - } - - for i := uint(0); i < count; i++ { - val, _ := v.Get(index + i) - - switch order { - case LSB0: - out |= val << i - default: - out |= val << (7 - i) - } - } - return -} - -// Iterator returns a function, which when invoked, returns the number -// of bits requested, and increments an internal cursor. -// -// When the end of the BitVector is reached, it returns zeroes indefinitely -// -// Panics if count is out of range -func (v *BitVector) Iterator(order BitNumbering) func(uint) byte { - cursor := uint(0) - return func(count uint) (out byte) { - if count > 8 { - log.Panicf("invalid count") - } - - out = v.Take(cursor, count, order) - cursor += count - return - } -} diff --git a/extern/rleplus/internal/bitvector_test.go b/extern/rleplus/internal/bitvector_test.go deleted file mode 100644 index 2895db3a3..000000000 --- a/extern/rleplus/internal/bitvector_test.go +++ /dev/null @@ -1,136 +0,0 @@ -package bitvector_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - bitvector "github.com/filecoin-project/lotus/extern/rleplus/internal" -) - -func TestBitVector(t *testing.T) { - t.Run("zero value", func(t *testing.T) { - var v bitvector.BitVector - - assert.Equal(t, bitvector.LSB0, v.BytePacking) - }) - - t.Run("Push", func(t *testing.T) { - // MSB0 bit numbering - v := bitvector.BitVector{BytePacking: bitvector.MSB0} - v.Push(1) - v.Push(0) - v.Push(1) - v.Push(1) - - assert.Equal(t, byte(176), v.Buf[0]) - - // LSB0 bit numbering - v = bitvector.BitVector{BytePacking: bitvector.LSB0} - v.Push(1) - v.Push(0) - v.Push(1) - v.Push(1) - - assert.Equal(t, byte(13), v.Buf[0]) - }) - - t.Run("Get", func(t *testing.T) { - bits := []byte{1, 0, 1, 1, 0, 0, 1, 0} - - for _, numbering := range []bitvector.BitNumbering{bitvector.MSB0, bitvector.LSB0} { - v := bitvector.BitVector{BytePacking: numbering} - - for _, bit := range bits { - v.Push(bit) - } - - for idx, expected := range bits { - actual, _ := v.Get(uint(idx)) - assert.Equal(t, expected, actual) - } - } - }) - - t.Run("Extend", func(t *testing.T) { - val := byte(171) // 0b10101011 - - var v bitvector.BitVector - - // MSB0 bit numbering - v = bitvector.BitVector{} - v.Extend(val, 4, bitvector.MSB0) - assertBitVector(t, []byte{1, 0, 1, 0}, v) - v.Extend(val, 5, bitvector.MSB0) - assertBitVector(t, []byte{1, 0, 1, 0, 1, 0, 1, 0, 1}, v) - - // LSB0 bit numbering - v = bitvector.BitVector{} - v.Extend(val, 4, bitvector.LSB0) - assertBitVector(t, []byte{1, 1, 0, 1}, v) - v.Extend(val, 5, bitvector.LSB0) - assertBitVector(t, []byte{1, 1, 0, 1, 1, 1, 0, 1, 0}, v) - }) - - t.Run("invalid counts to Take/Extend/Iterator cause panics", func(t *testing.T) { - v := bitvector.BitVector{BytePacking: bitvector.LSB0} - - assert.Panics(t, func() { v.Extend(0xff, 9, bitvector.LSB0) }) - - assert.Panics(t, func() { v.Take(0, 9, bitvector.LSB0) }) - - next := v.Iterator(bitvector.LSB0) - assert.Panics(t, func() { next(9) }) - }) - - t.Run("Take", func(t *testing.T) { - var v bitvector.BitVector - - bits := []byte{1, 0, 1, 0, 1, 0, 1, 1} - for _, bit := range bits { - v.Push(bit) - } - - assert.Equal(t, byte(176), v.Take(4, 4, bitvector.MSB0)) - assert.Equal(t, byte(13), v.Take(4, 4, bitvector.LSB0)) - }) - - t.Run("Iterator", func(t *testing.T) { - var buf []byte - - // make a bitvector of 256 sample bits - for i := 0; i < 32; i++ { - buf = append(buf, 128+32) - } - - v := bitvector.NewBitVector(buf, bitvector.LSB0) - - next := v.Iterator(bitvector.LSB0) - - // compare to Get() - for i := uint(0); i < v.Len; i++ { - expected, _ := v.Get(i) - assert.Equal(t, expected, next(1)) - } - - // out of range should return zero - assert.Equal(t, byte(0), next(1)) - assert.Equal(t, byte(0), next(8)) - - // compare to Take() - next = v.Iterator(bitvector.LSB0) - assert.Equal(t, next(5), v.Take(0, 5, bitvector.LSB0)) - assert.Equal(t, next(8), v.Take(5, 8, bitvector.LSB0)) - }) -} - -// Note: When using this helper assertion, expectedBits should *only* be 0s and 1s. -func assertBitVector(t *testing.T, expectedBits []byte, actual bitvector.BitVector) { - assert.Equal(t, uint(len(expectedBits)), actual.Len) - - for idx, bit := range expectedBits { - actualBit, err := actual.Get(uint(idx)) - assert.NoError(t, err) - assert.Equal(t, bit, actualBit) - } -} diff --git a/extern/rleplus/rleplus.go b/extern/rleplus/rleplus.go deleted file mode 100644 index 5d09c2e18..000000000 --- a/extern/rleplus/rleplus.go +++ /dev/null @@ -1,204 +0,0 @@ -package rleplus - -import ( - "encoding/binary" - "errors" - "fmt" - "sort" - - bitvector "github.com/filecoin-project/lotus/extern/rleplus/internal" -) - -// Version is the 2 lowest bits of this constant -const Version = 0 - -var ( - // ErrRunLengthTooLarge - data implies a run-length which isn't supported - ErrRunLengthTooLarge = fmt.Errorf("run length too large for RLE+ version %d", Version) - - // ErrDecode - invalid encoding for this version - ErrDecode = fmt.Errorf("invalid encoding for RLE+ version %d", Version) - - // ErrWrongVersion - wrong version of RLE+ - ErrWrongVersion = errors.New("invalid RLE+ version") -) - -// Encode returns the RLE+ representation of the provided integers. -// Also returned is the number of bits required by this encoding, -// which is not necessarily on a byte boundary. -// -// The RLE+ spec is here: https://github.com/filecoin-project/specs/blob/master/data-structures.md#rle-bitset-encoding -// and is described by the BNF Grammar: -// -// ::=
-//
::= -// ::= "00" -// ::= | "" -// ::= | | -// ::= "1" -// ::= "01" -// ::= "00" -// ::= "0" | "1" -// -// Filecoin specific: -// The encoding is returned as a []byte, each byte packed starting with the low-order bit (LSB0) -func Encode(ints []uint64) ([]byte, uint, error) { - v := bitvector.BitVector{BytePacking: bitvector.LSB0} - firstBit, runs := RunLengths(ints) - - // Add version header - v.Extend(Version, 2, bitvector.LSB0) - - v.Push(firstBit) - - for _, run := range runs { - switch { - case run == 1: - v.Push(1) - case run < 16: - v.Push(0) - v.Push(1) - v.Extend(byte(run), 4, bitvector.LSB0) - case run >= 16: - v.Push(0) - v.Push(0) - // 10 bytes needed to encode MaxUint64 - buf := make([]byte, 10) - numBytes := binary.PutUvarint(buf, run) - for i := 0; i < numBytes; i++ { - v.Extend(buf[i], 8, bitvector.LSB0) - } - default: - return nil, 0, ErrRunLengthTooLarge - } - } - - return v.Buf, v.Len, nil -} - -// Decode returns integers represented by the given RLE+ encoding -// -// The length of the encoding is not specified. It is inferred by -// reading zeroes from the (possibly depleted) BitVector, by virtue -// of the behavior of BitVector.Take() returning 0 when the end of -// the BitVector has been reached. This has the downside of not -// being able to detect corrupt encodings. -// -// The passed []byte should be packed in LSB0 bit numbering -func Decode(buf []byte) (ints []uint64, err error) { - if len(buf) == 0 { - return - } - - v := bitvector.NewBitVector(buf, bitvector.LSB0) - take := v.Iterator(bitvector.LSB0) - - // Read version and check - // Version check - ver := take(2) - if ver != Version { - return nil, ErrWrongVersion - } - - curIdx := uint64(0) - curBit := take(1) - var runLength int - done := false - - for done == false { - y := take(1) - switch y { - case 1: - runLength = 1 - case 0: - val := take(1) - - if val == 1 { - // short block - runLength = int(take(4)) - } else { - // long block - var buf []byte - for { - b := take(8) - buf = append(buf, b) - - if b&0x80 == 0 { - break - } - - // 10 bytes is required to store math.MaxUint64 in a uvarint - if len(buf) > 10 { - return nil, ErrDecode - } - } - x, _ := binary.Uvarint(buf) - - if x == 0 { - done = true - } - runLength = int(x) - } - } - - if curBit == 1 { - for j := 0; j < runLength; j++ { - ints = append(ints, curIdx+uint64(j)) - } - } - curIdx += uint64(runLength) - curBit = 1 - curBit - } - - return -} - -// RunLengths transforms integers into its bit-set-run-length representation. -// -// A set of unsigned integers { 0, 2, 4, 5, 6 } can be thought of as -// indices into a bitset { 1, 0, 1, 0, 1, 1, 1 } where bitset[index] == 1. -// -// The bit set run lengths of this set would then be { 1, 1, 1, 1, 3 }, -// representing lengths of runs alternating between 1 and 0, starting -// with a first bit of 1. -// -// Duplicated numbers are ignored. -// -// This is a helper function for Encode() -func RunLengths(ints []uint64) (firstBit byte, runs []uint64) { - if len(ints) == 0 { - return - } - - // Sort our incoming numbers - sort.Slice(ints, func(i, j int) bool { return ints[i] < ints[j] }) - - prev := ints[0] - - // Initialize our return value - if prev == 0 { - firstBit = 1 - } - - if firstBit == 0 { - // first run of zeroes - runs = append(runs, prev) - } - runs = append(runs, 1) - - for _, cur := range ints[1:] { - delta := cur - prev - switch { - case delta == 1: - runs[len(runs)-1]++ - case delta > 1: - // add run of zeroes if there is a gap - runs = append(runs, delta-1) - runs = append(runs, 1) - default: - // repeated number? - } - prev = cur - } - return -} diff --git a/extern/rleplus/rleplus_test.go b/extern/rleplus/rleplus_test.go deleted file mode 100644 index ecb2259e6..000000000 --- a/extern/rleplus/rleplus_test.go +++ /dev/null @@ -1,213 +0,0 @@ -package rleplus_test - -import ( - "fmt" - "math" - "sort" - "testing" - - "github.com/filecoin-project/lotus/extern/rleplus" - bitvector "github.com/filecoin-project/lotus/extern/rleplus/internal" - "gotest.tools/assert" -) - -func TestRleplus(t *testing.T) { - - t.Run("Encode", func(t *testing.T) { - // Encode an intset - ints := []uint64{ - // run of 1 - 0, - // gap of 1 - // run of 1 - 2, - // gap of 1 - // run of 3 - 4, 5, 6, - // gap of 4 - // run of 17 - 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, - } - - expectedBits := []byte{ - 0, 0, // version - 1, // first bit - 1, // run of 1 - 1, // gap of 1 - 1, // run of 1 - 1, // gap of 1 - 0, 1, 1, 1, 0, 0, // run of 3 - 0, 1, 0, 0, 1, 0, // gap of 4 - - // run of 17 < 0 0 (varint) > - 0, 0, - 1, 0, 0, 0, 1, 0, 0, 0, - } - - v := bitvector.BitVector{} - for _, bit := range expectedBits { - v.Push(bit) - } - actualBytes, _, err := rleplus.Encode(ints) - assert.NilError(t, err) - - assert.Equal(t, len(v.Buf), len(actualBytes)) - for idx, expected := range v.Buf { - assert.Equal( - t, - fmt.Sprintf("%08b", expected), - fmt.Sprintf("%08b", actualBytes[idx]), - ) - } - }) - - t.Run("Encode allows all runs sizes possible uint64", func(t *testing.T) { - // create a run of math.MaxUint64 - ints := []uint64{math.MaxUint64} - - // There would be 64 bits(1) for the UvarInt, totally 9 bytes. - expected := []byte{0xe0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f, 0x20} - encodeBytes, _, err := rleplus.Encode(ints) - assert.NilError(t, err) - for idx, v := range encodeBytes { - assert.Equal( - t, - fmt.Sprintf("%8b", v), - fmt.Sprintf("%8b", expected[idx]), - ) - } - }) - - t.Run("Encode for some big numbers", func(t *testing.T) { - // create a run of math.MaxUint64 - ints := make([]uint64, 1024) - - // ints {2^63 .. 2^63+1023} - for i := uint64(0); i < 1024; i++ { - ints[i] = uint64(1)<<63 + i - } - - expected := []byte{0x00, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x30, 0x00, 0x40, 0x04} - encodeBytes, _, err := rleplus.Encode(ints) - assert.NilError(t, err) - for idx, v := range encodeBytes { - // fmt.Println(v, expected[idx]) - assert.Equal( - t, - fmt.Sprintf("%8b", v), - fmt.Sprintf("%8b", expected[idx]), - ) - } - }) - - t.Run("Decode", func(t *testing.T) { - testCases := [][]uint64{ - {}, - {1}, - {0}, - {0, 1, 2, 3}, - { - // run of 1 - 0, - // gap of 1 - // run of 1 - 2, - // gap of 1 - // run of 3 - 4, 5, 6, - // gap of 4 - // run of 17 - 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, - }, - } - - for _, tc := range testCases { - encoded, _, err := rleplus.Encode(tc) - assert.NilError(t, err) - - result, err := rleplus.Decode(encoded) - assert.NilError(t, err) - - sort.Slice(tc, func(i, j int) bool { return tc[i] < tc[j] }) - sort.Slice(result, func(i, j int) bool { return result[i] < result[j] }) - - assert.Equal(t, len(tc), len(result)) - - for idx, expected := range tc { - assert.Equal(t, expected, result[idx]) - } - } - }) - - t.Run("Decode version check", func(t *testing.T) { - _, err := rleplus.Decode([]byte{0xff}) - assert.Error(t, err, "invalid RLE+ version") - }) - - t.Run("Decode returns an error with a bad encoding", func(t *testing.T) { - // create an encoding with a buffer with a run which is too long - _, err := rleplus.Decode([]byte{0xe0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}) - assert.Error(t, err, "invalid encoding for RLE+ version 0") - }) - - t.Run("outputs same as reference implementation", func(t *testing.T) { - // Encoding bitvec![LittleEndian; 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] - // in the Rust reference implementation gives an encoding of [223, 145, 136, 0] (without version field) - // The bit vector is equivalent to the integer set { 0, 2, 4, 5, 6, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 } - - // This is the above reference output with a version header "00" manually added - referenceEncoding := []byte{124, 71, 34, 2} - - expectedNumbers := []uint64{0, 2, 4, 5, 6, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27} - - encoded, _, err := rleplus.Encode(expectedNumbers) - assert.NilError(t, err) - - // Our encoded bytes are the same as the ref bytes - assert.Equal(t, len(referenceEncoding), len(encoded)) - for idx, expected := range referenceEncoding { - assert.Equal(t, expected, encoded[idx]) - } - - decoded, err := rleplus.Decode(referenceEncoding) - assert.NilError(t, err) - - // Our decoded integers are the same as expected - sort.Slice(decoded, func(i, j int) bool { return decoded[i] < decoded[j] }) - assert.Equal(t, len(expectedNumbers), len(decoded)) - for idx, expected := range expectedNumbers { - assert.Equal(t, expected, decoded[idx]) - } - }) - - t.Run("RunLengths", func(t *testing.T) { - testCases := []struct { - ints []uint64 - first byte - runs []uint64 - }{ - // empty - {}, - - // leading with ones - {[]uint64{0}, 1, []uint64{1}}, - {[]uint64{0, 1}, 1, []uint64{2}}, - {[]uint64{0, 0xffffffff, 0xffffffff + 1}, 1, []uint64{1, 0xffffffff - 1, 2}}, - - // leading with zeroes - {[]uint64{1}, 0, []uint64{1, 1}}, - {[]uint64{2}, 0, []uint64{2, 1}}, - {[]uint64{10, 11, 13, 20}, 0, []uint64{10, 2, 1, 1, 6, 1}}, - {[]uint64{10, 11, 11, 13, 20, 10, 11, 13, 20}, 0, []uint64{10, 2, 1, 1, 6, 1}}, - } - - for _, testCase := range testCases { - first, runs := rleplus.RunLengths(testCase.ints) - assert.Equal(t, testCase.first, first) - assert.Equal(t, len(testCase.runs), len(runs)) - for idx, runLength := range testCase.runs { - assert.Equal(t, runLength, runs[idx]) - } - } - }) -} diff --git a/genesis/types.go b/genesis/types.go index ce690d3d0..4b95d3311 100644 --- a/genesis/types.go +++ b/genesis/types.go @@ -28,7 +28,7 @@ type PreSeal struct { type Miner struct { Owner address.Address Worker address.Address - PeerId peer.ID + PeerId peer.ID //nolint:golint MarketBalance abi.TokenAmount PowerBalance abi.TokenAmount diff --git a/lib/addrutil/parse.go b/lib/addrutil/parse.go index fad389e33..f9ee04c3f 100644 --- a/lib/addrutil/parse.go +++ b/lib/addrutil/parse.go @@ -11,7 +11,7 @@ import ( madns "github.com/multiformats/go-multiaddr-dns" ) -// parseAddresses is a function that takes in a slice of string peer addresses +// ParseAddresses is a function that takes in a slice of string peer addresses // (multiaddr + peerid) and returns a slice of properly constructed peers func ParseAddresses(ctx context.Context, addrs []string) ([]peer.AddrInfo, error) { // resolve addresses diff --git a/lib/increadtimeout/incrt.go b/lib/increadtimeout/incrt.go index 95e6c0e42..0b9c65d4d 100644 --- a/lib/increadtimeout/incrt.go +++ b/lib/increadtimeout/incrt.go @@ -57,7 +57,7 @@ func (crt *incrt) Read(buf []byte) (int, error) { n, err := crt.rd.Read(buf) - crt.rd.SetReadDeadline(time.Time{}) + _ = crt.rd.SetReadDeadline(time.Time{}) if err == nil { dur := now().Sub(start) crt.wait -= dur diff --git a/lib/lotuslog/levels.go b/lib/lotuslog/levels.go index faf8fb137..c72d5a9c7 100644 --- a/lib/lotuslog/levels.go +++ b/lib/lotuslog/levels.go @@ -6,6 +6,7 @@ import ( logging "github.com/ipfs/go-log/v2" ) +//nolint:gosec func SetupLogLevels() { if _, set := os.LookupEnv("GOLOG_LOG_LEVEL"); !set { logging.SetLogLevel("*", "INFO") diff --git a/markets/utils/converters.go b/markets/utils/converters.go index 91a1ff8fc..d6316839b 100644 --- a/markets/utils/converters.go +++ b/markets/utils/converters.go @@ -5,7 +5,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/abi/big" "github.com/filecoin-project/specs-actors/actors/builtin/market" - peer "github.com/libp2p/go-libp2p-peer" + peer "github.com/libp2p/go-libp2p-core/peer" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/storagemarket" diff --git a/node/config/storage.go b/node/config/storage.go index 2d07565cc..2c603df03 100644 --- a/node/config/storage.go +++ b/node/config/storage.go @@ -2,12 +2,13 @@ package config import ( "encoding/json" - "github.com/filecoin-project/sector-storage/stores" "io" "io/ioutil" "os" "golang.org/x/xerrors" + + "github.com/filecoin-project/sector-storage/stores" ) func StorageFromFile(path string, def *stores.StorageConfig) (*stores.StorageConfig, error) { diff --git a/node/hello/hello.go b/node/hello/hello.go index f4e8d421b..a2a5fbd4e 100644 --- a/node/hello/hello.go +++ b/node/hello/hello.go @@ -64,7 +64,7 @@ func (hs *Service) HandleStream(s inet.Stream) { var hmsg HelloMessage if err := cborutil.ReadCborRPC(s, &hmsg); err != nil { log.Infow("failed to read hello message, disconnecting", "error", err) - s.Conn().Close() + _ = s.Conn().Close() return } arrived := time.Now() @@ -76,11 +76,11 @@ func (hs *Service) HandleStream(s inet.Stream) { if hmsg.GenesisHash != hs.syncer.Genesis.Cids()[0] { log.Warnf("other peer has different genesis! (%s)", hmsg.GenesisHash) - s.Conn().Close() + _ = s.Conn().Close() return } go func() { - defer s.Close() + defer s.Close() //nolint:errcheck sent := time.Now() msg := &LatencyMessage{ @@ -152,10 +152,10 @@ func (hs *Service) SayHello(ctx context.Context, pid peer.ID) error { } go func() { - defer s.Close() + defer s.Close() //nolint:errcheck lmsg := &LatencyMessage{} - s.SetReadDeadline(time.Now().Add(10 * time.Second)) + _ = s.SetReadDeadline(time.Now().Add(10 * time.Second)) err := cborutil.ReadCborRPC(s, lmsg) if err != nil { log.Infow("reading latency message", "error", err) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 320579ce1..01526fec2 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -427,7 +427,7 @@ func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath stri return err } - defer bufferedDS.Remove(ctx, c) + defer bufferedDS.Remove(ctx, c) //nolint:errcheck ssb := builder.NewSelectorSpecBuilder(basicnode.Style.Any) // entire DAG selector @@ -435,7 +435,7 @@ func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath stri ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() f, err := os.Create(outputPath) - defer f.Close() + defer f.Close() //nolint:errcheck if err != nil { return err } diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index 8b00cfb34..f6553fa54 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -425,7 +425,7 @@ func (a *ChainAPI) ChainExport(ctx context.Context, tsk types.TipSetKey) (<-chan r, w := io.Pipe() out := make(chan []byte) go func() { - defer w.Close() + defer w.Close() //nolint:errcheck // it is a pipe if err := a.Chain.Export(ctx, ts, w); err != nil { log.Errorf("chain export call failed: %s", err) return diff --git a/node/impl/remoteworker.go b/node/impl/remoteworker.go index e93ef7d5c..b7cee83ef 100644 --- a/node/impl/remoteworker.go +++ b/node/impl/remoteworker.go @@ -17,7 +17,7 @@ import ( ) type remoteWorker struct { - api.WorkerApi + api.WorkerAPI closer jsonrpc.ClientCloser } diff --git a/node/modules/core.go b/node/modules/core.go index 426705b34..6521b60df 100644 --- a/node/modules/core.go +++ b/node/modules/core.go @@ -34,7 +34,7 @@ func RecordValidator(ps peerstore.Peerstore) record.Validator { } } -const JWTSecretName = "auth-jwt-private" +const JWTSecretName = "auth-jwt-private" //nolint:gosec type jwtPayload struct { Allow []auth.Permission diff --git a/node/modules/lp2p/addrs.go b/node/modules/lp2p/addrs.go index 95971449d..da0de9ad9 100644 --- a/node/modules/lp2p/addrs.go +++ b/node/modules/lp2p/addrs.go @@ -18,7 +18,7 @@ func AddrFilters(filters []string) func() (opts Libp2pOpts, err error) { if err != nil { return opts, fmt.Errorf("incorrectly formatted address filter in config: %s", s) } - opts.Opts = append(opts.Opts, libp2p.FilterAddresses(f)) + opts.Opts = append(opts.Opts, libp2p.FilterAddresses(f)) //golint:staticcheck } return opts, nil } diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index f54a2672e..32002c7c4 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -72,7 +72,7 @@ func GetParams(sbc *ffiwrapper.Config) error { return err } - if err := paramfetch.GetParams(build.ParametersJson(), uint64(ssize)); err != nil { + if err := paramfetch.GetParams(build.ParametersJSON(), uint64(ssize)); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } @@ -167,12 +167,10 @@ func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h func HandleRetrieval(host host.Host, lc fx.Lifecycle, m retrievalmarket.RetrievalProvider) { lc.Append(fx.Hook{ OnStart: func(context.Context) error { - m.Start() - return nil + return m.Start() }, OnStop: func(context.Context) error { - m.Stop() - return nil + return m.Stop() }, }) } @@ -182,12 +180,10 @@ func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h sto lc.Append(fx.Hook{ OnStart: func(context.Context) error { - h.Start(ctx) - return nil + return h.Start(ctx) }, OnStop: func(context.Context) error { - h.Stop() - return nil + return h.Stop() }, }) } diff --git a/node/node_test.go b/node/node_test.go index 0b0af0bcf..a31b4162b 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -84,9 +84,11 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a nic := storedcounter.New(ds, datastore.NewKey(modules.StorageCounterDSPrefix)) for i := 0; i < nGenesisPreseals; i++ { - nic.Next() + _, err := nic.Next() + require.NoError(t, err) } - nic.Next() + _, err = nic.Next() + require.NoError(t, err) err = lr.Close() require.NoError(t, err) diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 322adbdef..b03a6bf37 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -3,7 +3,6 @@ package repo import ( "encoding/json" "fmt" - "github.com/filecoin-project/sector-storage/stores" "io" "io/ioutil" "os" @@ -21,6 +20,8 @@ import ( "github.com/multiformats/go-multiaddr" "golang.org/x/xerrors" + "github.com/filecoin-project/sector-storage/stores" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/config" ) diff --git a/paychmgr/store.go b/paychmgr/store.go index 3a425c4fc..66a514feb 100644 --- a/paychmgr/store.go +++ b/paychmgr/store.go @@ -115,7 +115,7 @@ func (ps *Store) ListChannels() ([]address.Address, error) { if err != nil { return nil, err } - defer res.Close() + defer res.Close() //nolint:errcheck var out []address.Address for { @@ -144,7 +144,7 @@ func (ps *Store) findChan(filter func(*ChannelInfo) bool) (address.Address, erro if err != nil { return address.Undef, err } - defer res.Close() + defer res.Close() //nolint:errcheck var ci ChannelInfo diff --git a/storage/miner.go b/storage/miner.go index 779f981e9..57b162622 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -107,14 +107,13 @@ func (m *Miner) Run(ctx context.Context) error { pcp := sealing.NewBasicPreCommitPolicy(adaptedAPI, 10000000, md.PeriodStart%miner.WPoStProvingPeriod) m.sealing = sealing.New(adaptedAPI, NewEventsAdapter(evts), m.maddr, m.ds, m.sealer, m.sc, m.verif, &pcp) - go m.sealing.Run(ctx) + go m.sealing.Run(ctx) //nolint:errcheck // logged intside the function return nil } func (m *Miner) Stop(ctx context.Context) error { - defer m.sealing.Stop(ctx) - return nil + return m.sealing.Stop(ctx) } func (m *Miner) runPreflightChecks(ctx context.Context) error { From f26a336afb77ffa75775b38de86b4497f96e32bd Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 28 May 2020 00:45:17 +0200 Subject: [PATCH 008/160] Use git.base_revision Signed-off-by: Jakub Sztandera --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ddcbd5481..be4ab11ce 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -331,7 +331,7 @@ workflows: ci: jobs: - lint-changes: - args: "--new-from-rev origin/master" + args: "--new-from-rev << pipeline.git.base_revision >>" - test: codecov-upload: true - mod-tidy-check From 7232e5481c9edb65a87fdffab420455b3a94ed1d Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 28 May 2020 00:49:52 +0200 Subject: [PATCH 009/160] Use origin/next for linting Signed-off-by: Jakub Sztandera --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index be4ab11ce..312aa42ca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -331,7 +331,7 @@ workflows: ci: jobs: - lint-changes: - args: "--new-from-rev << pipeline.git.base_revision >>" + args: "--new-from-rev origin/next" - test: codecov-upload: true - mod-tidy-check From b372881e91b9059fa317a335371991bfa524e619 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 28 May 2020 01:15:19 +0200 Subject: [PATCH 010/160] Fix more lint warnings Signed-off-by: Jakub Sztandera --- cli/multisig.go | 2 +- cmd/lotus-bench/main.go | 2 +- cmd/lotus-seed/seed/seed.go | 2 +- cmd/lotus-storage-miner/init.go | 2 +- lib/lotuslog/levels.go | 19 +++++++++---------- node/impl/client/client.go | 3 +-- node/modules/lp2p/addrs.go | 2 +- 7 files changed, 15 insertions(+), 17 deletions(-) diff --git a/cli/multisig.go b/cli/multisig.go index 9acf1af08..37106dff5 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -207,7 +207,7 @@ var msigInspectCmd = &cli.Command{ fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%d\t%x\n", txid, state(tx), tx.To, types.FIL(tx.Value), tx.Method, tx.Params) } if err := w.Flush(); err != nil { - return xerrors.Errorf("flushing ouput: %+v", err) + return xerrors.Errorf("flushing output: %+v", err) } } diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index b8740f2f3..cbb59309d 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -157,7 +157,7 @@ var sealBenchCmd = &cli.Command{ return err } - err = os.MkdirAll(sdir, 0775) + err = os.MkdirAll(sdir, 0775) //nolint:gosec if err != nil { return xerrors.Errorf("creating sectorbuilder dir: %w", err) } diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index 0f6539563..06cb011b4 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -48,7 +48,7 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum SealProofType: spt, } - if err := os.MkdirAll(sbroot, 0775); err != nil { //golint:gosec + if err := os.MkdirAll(sbroot, 0775); err != nil { //nolint:gosec return nil, nil, err } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 7a6ac8257..0dcc9563b 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -389,7 +389,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, if err != nil { return err } - defer lr.Close() //notlint:errcheck + defer lr.Close() //nolint:errcheck log.Info("Initializing libp2p identity") diff --git a/lib/lotuslog/levels.go b/lib/lotuslog/levels.go index c72d5a9c7..ae3959568 100644 --- a/lib/lotuslog/levels.go +++ b/lib/lotuslog/levels.go @@ -6,17 +6,16 @@ import ( logging "github.com/ipfs/go-log/v2" ) -//nolint:gosec func SetupLogLevels() { if _, set := os.LookupEnv("GOLOG_LOG_LEVEL"); !set { - logging.SetLogLevel("*", "INFO") - logging.SetLogLevel("dht", "ERROR") - logging.SetLogLevel("swarm2", "WARN") - logging.SetLogLevel("bitswap", "WARN") - //logging.SetLogLevel("pubsub", "WARN") - logging.SetLogLevel("connmgr", "WARN") - logging.SetLogLevel("advmgr", "DEBUG") - logging.SetLogLevel("stores", "DEBUG") - logging.SetLogLevel("nat", "INFO") + _ = logging.SetLogLevel("*", "INFO") + _ = logging.SetLogLevel("dht", "ERROR") + _ = logging.SetLogLevel("swarm2", "WARN") + _ = logging.SetLogLevel("bitswap", "WARN") + //_ = logging.SetLogLevel("pubsub", "WARN") + _ = logging.SetLogLevel("connmgr", "WARN") + _ = logging.SetLogLevel("advmgr", "DEBUG") + _ = logging.SetLogLevel("stores", "DEBUG") + _ = logging.SetLogLevel("nat", "INFO") } } diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 01526fec2..7d6d5369b 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -435,7 +435,6 @@ func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath stri ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() f, err := os.Create(outputPath) - defer f.Close() //nolint:errcheck if err != nil { return err } @@ -445,7 +444,7 @@ func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath stri return err } - return nil + return f.Close() } func (a *API) clientImport(ref api.FileRef, bufferedDS *ipld.BufferedDAG) (cid.Cid, error) { diff --git a/node/modules/lp2p/addrs.go b/node/modules/lp2p/addrs.go index da0de9ad9..afb8ce910 100644 --- a/node/modules/lp2p/addrs.go +++ b/node/modules/lp2p/addrs.go @@ -18,7 +18,7 @@ func AddrFilters(filters []string) func() (opts Libp2pOpts, err error) { if err != nil { return opts, fmt.Errorf("incorrectly formatted address filter in config: %s", s) } - opts.Opts = append(opts.Opts, libp2p.FilterAddresses(f)) //golint:staticcheck + opts.Opts = append(opts.Opts, libp2p.FilterAddresses(f)) //nolint:staticcheck } return opts, nil } From 8639409f740855ae92454c8e77afa76ed55db00d Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 28 May 2020 01:16:01 +0200 Subject: [PATCH 011/160] go mod tidy , Signed-off-by: Jakub Sztandera --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index b8fa35330..63a3a07dd 100644 --- a/go.mod +++ b/go.mod @@ -113,7 +113,6 @@ require ( golang.org/x/time v0.0.0-20191024005414-555d28b269f0 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 - gotest.tools v2.2.0+incompatible launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect ) From de3edc2ec7657cd8c1227f5239ecdf115c11fcaf Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Fri, 22 May 2020 11:51:18 -0300 Subject: [PATCH 012/160] remote ipfs support & automatic env Signed-off-by: Ignacio Hagopian --- go.mod | 1 + go.sum | 2 ++ lib/ipfsbstore/ipfsbstore.go | 13 +++++++++++++ node/builder.go | 6 ++++++ node/config/def.go | 3 ++- node/config/load.go | 7 +++++++ node/modules/ipfsclient.go | 19 +++++++++++++++++++ node/repo/fsrepo.go | 5 +++-- 8 files changed, 53 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b8fa35330..aae4a9bbb 100644 --- a/go.mod +++ b/go.mod @@ -69,6 +69,7 @@ require ( github.com/ipfs/interface-go-ipfs-core v0.2.3 github.com/ipld/go-car v0.1.1-0.20200430185908-8ff2e52a4c88 github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e + github.com/kelseyhightower/envconfig v1.4.0 github.com/lib/pq v1.2.0 github.com/libp2p/go-eventbus v0.1.0 github.com/libp2p/go-libp2p v0.9.2 diff --git a/go.sum b/go.sum index 61e846c3d..dc9608cc9 100644 --- a/go.sum +++ b/go.sum @@ -560,6 +560,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/kabukky/httpscerts v0.0.0-20150320125433-617593d7dcb3 h1:Iy7Ifq2ysilWU4QlCx/97OoI4xT1IV7i8byT/EyIT/M= github.com/kabukky/httpscerts v0.0.0-20150320125433-617593d7dcb3/go.mod h1:BYpt4ufZiIGv2nXn4gMxnfKV306n3mWXgNu/d2TqdTU= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= diff --git a/lib/ipfsbstore/ipfsbstore.go b/lib/ipfsbstore/ipfsbstore.go index a79407f5a..33110e295 100644 --- a/lib/ipfsbstore/ipfsbstore.go +++ b/lib/ipfsbstore/ipfsbstore.go @@ -7,6 +7,7 @@ import ( "golang.org/x/xerrors" + "github.com/multiformats/go-multiaddr" "github.com/multiformats/go-multihash" blocks "github.com/ipfs/go-block-format" @@ -35,6 +36,18 @@ func NewIpfsBstore(ctx context.Context) (*IpfsBstore, error) { }, nil } +func NewRemoteIpfsBstore(ctx context.Context, maddr multiaddr.Multiaddr) (*IpfsBstore, error) { + api, err := httpapi.NewApi(maddr) + if err != nil { + return nil, xerrors.Errorf("getting remote ipfs api: %w", err) + } + + return &IpfsBstore{ + ctx: ctx, + api: api, + }, nil +} + func (i *IpfsBstore) DeleteBlock(cid cid.Cid) error { return xerrors.Errorf("not supported") } diff --git a/node/builder.go b/node/builder.go index 0154f1494..5fe1e4346 100644 --- a/node/builder.go +++ b/node/builder.go @@ -378,11 +378,17 @@ func ConfigFullNode(c interface{}) Option { return Error(xerrors.Errorf("invalid config from repo, got: %T", c)) } + remoteIpfsMaddrNotEmpty := func(s *Settings) bool { + return len(cfg.Client.RemoteIpfsMAddr) > 0 + } + return Options( ConfigCommon(&cfg.Common), If(cfg.Client.UseIpfs, Override(new(dtypes.ClientBlockstore), modules.IpfsClientBlockstore), ), + ApplyIf(remoteIpfsMaddrNotEmpty, + Override(new(dtypes.ClientBlockstore), modules.IpfsRemoteClientBlockstore(cfg.Client.RemoteIpfsMAddr))), If(cfg.Metrics.HeadNotifs, Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)), diff --git a/node/config/def.go b/node/config/def.go index 72fdde4ff..eea38965b 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -62,7 +62,8 @@ type Metrics struct { } type Client struct { - UseIpfs bool + UseIpfs bool + RemoteIpfsMAddr string } func defCommon() Common { diff --git a/node/config/load.go b/node/config/load.go index 352d17197..1590efedb 100644 --- a/node/config/load.go +++ b/node/config/load.go @@ -2,10 +2,12 @@ package config import ( "bytes" + "fmt" "io" "os" "github.com/BurntSushi/toml" + "github.com/kelseyhightower/envconfig" "golang.org/x/xerrors" ) @@ -32,6 +34,11 @@ func FromReader(reader io.Reader, def interface{}) (interface{}, error) { return nil, err } + err = envconfig.Process("LOTUS", cfg) + if err != nil { + return nil, fmt.Errorf("processing env vars overrides: %s", err) + } + return cfg, nil } diff --git a/node/modules/ipfsclient.go b/node/modules/ipfsclient.go index 8898e4cd1..373f8531f 100644 --- a/node/modules/ipfsclient.go +++ b/node/modules/ipfsclient.go @@ -6,6 +6,7 @@ import ( "github.com/ipfs/go-filestore" blockstore "github.com/ipfs/go-ipfs-blockstore" + "github.com/multiformats/go-multiaddr" "github.com/filecoin-project/lotus/lib/bufbstore" "github.com/filecoin-project/lotus/lib/ipfsbstore" @@ -24,3 +25,21 @@ func IpfsClientBlockstore(mctx helpers.MetricsCtx, lc fx.Lifecycle, fstore dtype blockstore.NewIdStore((*filestore.Filestore)(fstore)), ), nil } + +func IpfsRemoteClientBlockstore(ipfsMaddr string) func(helpers.MetricsCtx, fx.Lifecycle, dtypes.ClientFilestore) (dtypes.ClientBlockstore, error) { + return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, fstore dtypes.ClientFilestore) (dtypes.ClientBlockstore, error) { + ma, err := multiaddr.NewMultiaddr(ipfsMaddr) + if err != nil { + return nil, xerrors.Errorf("parsing ipfs multiaddr: %w", err) + } + ipfsbs, err := ipfsbstore.NewRemoteIpfsBstore(helpers.LifecycleCtx(mctx, lc), ma) + if err != nil { + return nil, xerrors.Errorf("constructing ipfs blockstore: %w", err) + } + + return bufbstore.NewTieredBstore( + ipfsbs, + blockstore.NewIdStore((*filestore.Filestore)(fstore)), + ), nil + } +} diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 322adbdef..830de3b24 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -3,7 +3,6 @@ package repo import ( "encoding/json" "fmt" - "github.com/filecoin-project/sector-storage/stores" "io" "io/ioutil" "os" @@ -11,6 +10,8 @@ import ( "strings" "sync" + "github.com/filecoin-project/sector-storage/stores" + "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" badger "github.com/ipfs/go-ds-badger2" @@ -276,7 +277,7 @@ func (fsr *fsLockedRepo) Datastore(ns string) (datastore.Batching, error) { return namespace.Wrap(fsr.ds, datastore.NewKey(ns)), nil } -func (fsr *fsLockedRepo) Config() (interface{}, error) { +func (fsr *fsLockedRepo) Config() (df interface{}, err error) { if err := fsr.stillValid(); err != nil { return nil, err } From 0af7cd14483bbc6481fc642c99dd211aaddb0611 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Tue, 26 May 2020 09:50:35 -0300 Subject: [PATCH 013/160] allow using ipfs for retrieval Signed-off-by: Ignacio Hagopian --- node/builder.go | 11 +++-------- node/config/def.go | 5 +++-- node/modules/ipfsclient.go | 36 +++++++++++++++++------------------- 3 files changed, 23 insertions(+), 29 deletions(-) diff --git a/node/builder.go b/node/builder.go index 5fe1e4346..d054be7a5 100644 --- a/node/builder.go +++ b/node/builder.go @@ -378,18 +378,13 @@ func ConfigFullNode(c interface{}) Option { return Error(xerrors.Errorf("invalid config from repo, got: %T", c)) } - remoteIpfsMaddrNotEmpty := func(s *Settings) bool { - return len(cfg.Client.RemoteIpfsMAddr) > 0 - } - + ipfsMaddr := cfg.Client.IpfsMAddr + useForRetrieval := cfg.Client.IpfsUseForRetrieval return Options( ConfigCommon(&cfg.Common), If(cfg.Client.UseIpfs, - Override(new(dtypes.ClientBlockstore), modules.IpfsClientBlockstore), + Override(new(dtypes.ClientBlockstore), modules.IpfsClientBlockstore(ipfsMaddr, useForRetrieval)), ), - ApplyIf(remoteIpfsMaddrNotEmpty, - Override(new(dtypes.ClientBlockstore), modules.IpfsRemoteClientBlockstore(cfg.Client.RemoteIpfsMAddr))), - If(cfg.Metrics.HeadNotifs, Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)), ), diff --git a/node/config/def.go b/node/config/def.go index eea38965b..d5c64d40d 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -62,8 +62,9 @@ type Metrics struct { } type Client struct { - UseIpfs bool - RemoteIpfsMAddr string + UseIpfs bool + IpfsMAddr string + IpfsUseForRetrieval bool } func defCommon() Common { diff --git a/node/modules/ipfsclient.go b/node/modules/ipfsclient.go index 373f8531f..c3710de1c 100644 --- a/node/modules/ipfsclient.go +++ b/node/modules/ipfsclient.go @@ -14,32 +14,30 @@ import ( "github.com/filecoin-project/lotus/node/modules/helpers" ) -func IpfsClientBlockstore(mctx helpers.MetricsCtx, lc fx.Lifecycle, fstore dtypes.ClientFilestore) (dtypes.ClientBlockstore, error) { - ipfsbs, err := ipfsbstore.NewIpfsBstore(helpers.LifecycleCtx(mctx, lc)) - if err != nil { - return nil, xerrors.Errorf("constructing ipfs blockstore: %w", err) - } - - return bufbstore.NewTieredBstore( - ipfsbs, - blockstore.NewIdStore((*filestore.Filestore)(fstore)), - ), nil -} - -func IpfsRemoteClientBlockstore(ipfsMaddr string) func(helpers.MetricsCtx, fx.Lifecycle, dtypes.ClientFilestore) (dtypes.ClientBlockstore, error) { +// IpfsClientBlockstore returns a ClientBlockstore implementation backed by an IPFS node. +// If ipfsMaddr is empty, a local IPFS node is assumed considering IPFS_PATH configuration. +// If ipfsMaddr is not empty, it will connect to the remote IPFS node with the provided multiaddress. +// The flag useForRetrieval indicates if the IPFS node will also be used for storing retrieving deals. +func IpfsClientBlockstore(ipfsMaddr string, useForRetrieval bool) func(helpers.MetricsCtx, fx.Lifecycle, dtypes.ClientFilestore) (dtypes.ClientBlockstore, error) { return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, fstore dtypes.ClientFilestore) (dtypes.ClientBlockstore, error) { ma, err := multiaddr.NewMultiaddr(ipfsMaddr) if err != nil { return nil, xerrors.Errorf("parsing ipfs multiaddr: %w", err) } - ipfsbs, err := ipfsbstore.NewRemoteIpfsBstore(helpers.LifecycleCtx(mctx, lc), ma) + var ipfsbs *ipfsbstore.IpfsBstore + if ipfsMaddr != "" { + ipfsbs, err = ipfsbstore.NewRemoteIpfsBstore(helpers.LifecycleCtx(mctx, lc), ma) + } else { + ipfsbs, err = ipfsbstore.NewIpfsBstore(helpers.LifecycleCtx(mctx, lc)) + } if err != nil { return nil, xerrors.Errorf("constructing ipfs blockstore: %w", err) } - - return bufbstore.NewTieredBstore( - ipfsbs, - blockstore.NewIdStore((*filestore.Filestore)(fstore)), - ), nil + var ws blockstore.Blockstore + ws = ipfsbs + if !useForRetrieval { + ws = blockstore.NewIdStore((*filestore.Filestore)(fstore)) + } + return bufbstore.NewTieredBstore(ipfsbs, ws), nil } } From 10753f14d0ea156da12ed9a748b34ebd8749e7e3 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Tue, 26 May 2020 12:36:21 -0300 Subject: [PATCH 014/160] retrieve: make fileref optional Signed-off-by: Ignacio Hagopian --- api/api_full.go | 2 +- api/apistruct/struct.go | 4 ++-- api/test/deals.go | 2 +- cli/client.go | 2 +- node/impl/client/client.go | 7 ++++++- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 9026bf2d5..c6d442e2b 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -117,7 +117,7 @@ type FullNode interface { ClientListDeals(ctx context.Context) ([]DealInfo, error) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) - ClientRetrieve(ctx context.Context, order RetrievalOrder, ref FileRef) error + ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *FileRef) error ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) ClientCalcCommP(ctx context.Context, inpath string, miner address.Address) (*CommPRet, error) ClientGenCar(ctx context.Context, ref FileRef, outpath string) error diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 368a2b145..f0be44070 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -110,7 +110,7 @@ type FullNodeStruct struct { ClientStartDeal func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) `perm:"admin"` ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error) `perm:"read"` ClientListDeals func(ctx context.Context) ([]api.DealInfo, error) `perm:"write"` - ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, ref api.FileRef) error `perm:"admin"` + ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error `perm:"admin"` ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) `perm:"read"` ClientCalcCommP func(ctx context.Context, inpath string, miner address.Address) (*api.CommPRet, error) `perm:"read"` ClientGenCar func(ctx context.Context, ref api.FileRef, outpath string) error `perm:"write"` @@ -321,7 +321,7 @@ func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]api.DealInfo, e return c.Internal.ClientListDeals(ctx) } -func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref api.FileRef) error { +func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error { return c.Internal.ClientRetrieve(ctx, order, ref) } diff --git a/api/test/deals.go b/api/test/deals.go index 2414d3fe2..625be4583 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -200,7 +200,7 @@ func testRetrieval(t *testing.T, ctx context.Context, err error, client *impl.Fu t.Fatal(err) } - ref := api.FileRef{ + ref := &api.FileRef{ Path: filepath.Join(rpath, "ret"), IsCAR: carExport, } diff --git a/cli/client.go b/cli/client.go index 7f7e5e6c4..3bfc5de70 100644 --- a/cli/client.go +++ b/cli/client.go @@ -384,7 +384,7 @@ var clientRetrieveCmd = &cli.Command{ return nil } - ref := lapi.FileRef{ + ref := &lapi.FileRef{ Path: cctx.Args().Get(1), IsCAR: cctx.Bool("car"), } diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 320579ce1..ff974e8b2 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -302,7 +302,7 @@ func (a *API) ClientListImports(ctx context.Context) ([]api.Import, error) { } } -func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref api.FileRef) error { +func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error { if order.MinerPeerID == "" { mi, err := a.StateMinerInfo(ctx, order.Miner, types.EmptyTSK) if err != nil { @@ -353,6 +353,11 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref unsubscribe() + // If ref is nil, it only fetches the data into the configured blockstore. + if ref == nil { + return nil + } + if ref.IsCAR { f, err := os.OpenFile(ref.Path, os.O_CREATE|os.O_WRONLY, 0644) if err != nil { From 6e63ea6cce527ce14c9be36f3a0e428472fe7ec8 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Tue, 26 May 2020 15:35:20 -0300 Subject: [PATCH 015/160] revert file Signed-off-by: Ignacio Hagopian --- node/repo/fsrepo.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 830de3b24..322adbdef 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -3,6 +3,7 @@ package repo import ( "encoding/json" "fmt" + "github.com/filecoin-project/sector-storage/stores" "io" "io/ioutil" "os" @@ -10,8 +11,6 @@ import ( "strings" "sync" - "github.com/filecoin-project/sector-storage/stores" - "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" badger "github.com/ipfs/go-ds-badger2" @@ -277,7 +276,7 @@ func (fsr *fsLockedRepo) Datastore(ns string) (datastore.Batching, error) { return namespace.Wrap(fsr.ds, datastore.NewKey(ns)), nil } -func (fsr *fsLockedRepo) Config() (df interface{}, err error) { +func (fsr *fsLockedRepo) Config() (interface{}, error) { if err := fsr.stillValid(); err != nil { return nil, err } From 17c378c18af7755e335f38e96d5fb0f66818f720 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Wed, 27 May 2020 09:50:32 -0300 Subject: [PATCH 016/160] fix Signed-off-by: Ignacio Hagopian --- node/modules/ipfsclient.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/node/modules/ipfsclient.go b/node/modules/ipfsclient.go index c3710de1c..f405cb4c6 100644 --- a/node/modules/ipfsclient.go +++ b/node/modules/ipfsclient.go @@ -20,12 +20,14 @@ import ( // The flag useForRetrieval indicates if the IPFS node will also be used for storing retrieving deals. func IpfsClientBlockstore(ipfsMaddr string, useForRetrieval bool) func(helpers.MetricsCtx, fx.Lifecycle, dtypes.ClientFilestore) (dtypes.ClientBlockstore, error) { return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, fstore dtypes.ClientFilestore) (dtypes.ClientBlockstore, error) { - ma, err := multiaddr.NewMultiaddr(ipfsMaddr) - if err != nil { - return nil, xerrors.Errorf("parsing ipfs multiaddr: %w", err) - } + var err error var ipfsbs *ipfsbstore.IpfsBstore if ipfsMaddr != "" { + var ma multiaddr.Multiaddr + ma, err = multiaddr.NewMultiaddr(ipfsMaddr) + if err != nil { + return nil, xerrors.Errorf("parsing ipfs multiaddr: %w", err) + } ipfsbs, err = ipfsbstore.NewRemoteIpfsBstore(helpers.LifecycleCtx(mctx, lc), ma) } else { ipfsbs, err = ipfsbstore.NewIpfsBstore(helpers.LifecycleCtx(mctx, lc)) From 495554e565c4f74391c100ad6ae93d2be827bb1a Mon Sep 17 00:00:00 2001 From: vyzo Date: Tue, 26 May 2020 20:47:00 +0300 Subject: [PATCH 017/160] gomod: update go-libp2p-pubsub@v0.3.1 --- go.mod | 2 +- go.sum | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index aae4a9bbb..b587919c2 100644 --- a/go.mod +++ b/go.mod @@ -80,7 +80,7 @@ require ( github.com/libp2p/go-libp2p-mplex v0.2.3 github.com/libp2p/go-libp2p-peer v0.2.0 github.com/libp2p/go-libp2p-peerstore v0.2.4 - github.com/libp2p/go-libp2p-pubsub v0.3.0 + github.com/libp2p/go-libp2p-pubsub v0.3.1 github.com/libp2p/go-libp2p-quic-transport v0.1.1 github.com/libp2p/go-libp2p-record v0.1.2 github.com/libp2p/go-libp2p-routing-helpers v0.2.1 diff --git a/go.sum b/go.sum index dc9608cc9..29cd94e09 100644 --- a/go.sum +++ b/go.sum @@ -206,6 +206,7 @@ github.com/filecoin-project/sector-storage v0.0.0-20200522011946-a59ca7536a95 h1 github.com/filecoin-project/sector-storage v0.0.0-20200522011946-a59ca7536a95/go.mod h1:1UpuWCUC7XNfgyufyQQq+/40JSzhfGdgcwauK55C8gk= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= +github.com/filecoin-project/specs-actors v0.2.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= github.com/filecoin-project/specs-actors v0.5.2/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= github.com/filecoin-project/specs-actors v0.5.4-0.20200521014528-0df536f7e461 h1:yyAoJ9dNboljDWj0uBdJEbdaTak/YhkjYUQt0GzlY0A= @@ -726,8 +727,8 @@ github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYc github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= -github.com/libp2p/go-libp2p-pubsub v0.3.0 h1:K5FSYyfcSrJWrGExgdbogCLMqwC3pQaXEVt2CaUy1SA= -github.com/libp2p/go-libp2p-pubsub v0.3.0/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato= +github.com/libp2p/go-libp2p-pubsub v0.3.1 h1:7Hyv2d8BK/x1HGRJTZ8X++VQEP+WqDTSwpUSZGTVLYA= +github.com/libp2p/go-libp2p-pubsub v0.3.1/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato= github.com/libp2p/go-libp2p-quic-transport v0.1.1 h1:MFMJzvsxIEDEVKzO89BnB/FgvMj9WI4GDGUW2ArDPUA= github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU= github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFAymLDLm7lxMgU79Q= From f1ccbe496a74ce18d42f8e10c77cad2978698e8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 27 May 2020 10:13:06 +0200 Subject: [PATCH 018/160] cli: Trim spaces from token file --- node/repo/fsrepo.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 322adbdef..396fbf67e 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -1,9 +1,9 @@ package repo import ( + "bytes" "encoding/json" "fmt" - "github.com/filecoin-project/sector-storage/stores" "io" "io/ioutil" "os" @@ -21,6 +21,8 @@ import ( "github.com/multiformats/go-multiaddr" "golang.org/x/xerrors" + "github.com/filecoin-project/sector-storage/stores" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/config" ) @@ -191,7 +193,12 @@ func (fsr *FsRepo) APIToken() ([]byte, error) { } defer f.Close() //nolint: errcheck // Read only op - return ioutil.ReadAll(f) + tb, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + + return bytes.TrimSpace(tb), nil } // Lock acquires exclusive lock on this repo From be1bbe88b0408ad3dcb24053721a7ca52bf8d2d5 Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 21 May 2020 11:17:34 -0700 Subject: [PATCH 019/160] update chain-val syscalls interface --- chain/validation/applier.go | 8 +++++--- chain/validation/factories.go | 19 +++---------------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/chain/validation/applier.go b/chain/validation/applier.go index 5289f840b..f30c543bb 100644 --- a/chain/validation/applier.go +++ b/chain/validation/applier.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/crypto" "github.com/filecoin-project/specs-actors/actors/puppet" + "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/ipfs/go-cid" vtypes "github.com/filecoin-project/chain-validation/chain/types" @@ -24,12 +25,13 @@ import ( // Applier applies messages to state trees and storage. type Applier struct { stateWrapper *StateWrapper + syscalls runtime.Syscalls } var _ vstate.Applier = &Applier{} -func NewApplier(sw *StateWrapper) *Applier { - return &Applier{sw} +func NewApplier(sw *StateWrapper, syscalls runtime.Syscalls) *Applier { + return &Applier{sw, syscalls} } func (a *Applier) ApplyMessage(epoch abi.ChainEpoch, message *vtypes.Message) (vtypes.ApplyMessageResult, error) { @@ -65,7 +67,7 @@ func (a *Applier) ApplySignedMessage(epoch abi.ChainEpoch, msg *vtypes.SignedMes } func (a *Applier) ApplyTipSetMessages(epoch abi.ChainEpoch, blocks []vtypes.BlockMessagesInfo, rnd vstate.RandomnessSource) (vtypes.ApplyTipSetResult, error) { - cs := store.NewChainStore(a.stateWrapper.bs, a.stateWrapper.ds, vdrivers.NewChainValidationSyscalls()) + cs := store.NewChainStore(a.stateWrapper.bs, a.stateWrapper.ds, a.syscalls) sm := stmgr.NewStateManager(cs) var bms []stmgr.BlockMessages diff --git a/chain/validation/factories.go b/chain/validation/factories.go index 223382538..d3771d87d 100644 --- a/chain/validation/factories.go +++ b/chain/validation/factories.go @@ -1,11 +1,9 @@ package validation import ( - "context" + "github.com/filecoin-project/specs-actors/actors/runtime" vstate "github.com/filecoin-project/chain-validation/state" - "github.com/filecoin-project/specs-actors/actors/abi" - acrypto "github.com/filecoin-project/specs-actors/actors/crypto" ) type Factories struct { @@ -18,26 +16,15 @@ func NewFactories() *Factories { return &Factories{} } -func (f *Factories) NewStateAndApplier() (vstate.VMWrapper, vstate.Applier) { +func (f *Factories) NewStateAndApplier(syscalls runtime.Syscalls) (vstate.VMWrapper, vstate.Applier) { st := NewState() - return st, NewApplier(st) + return st, NewApplier(st, syscalls) } func (f *Factories) NewKeyManager() vstate.KeyManager { return newKeyManager() } -type fakeRandSrc struct { -} - -func (r fakeRandSrc) Randomness(_ context.Context, _ acrypto.DomainSeparationTag, _ abi.ChainEpoch, _ []byte) (abi.Randomness, error) { - return abi.Randomness("sausages"), nil -} - -func (f *Factories) NewRandomnessSource() vstate.RandomnessSource { - return &fakeRandSrc{} -} - func (f *Factories) NewValidationConfig() vstate.ValidationConfig { trackGas := true checkExit := true From 9f8ecb0ac4d499b263caa0580af78b4e8f1e136e Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 26 May 2020 10:32:21 -0700 Subject: [PATCH 020/160] update chain-validation - has new syscalls --- chain/validation/applier.go | 3 +-- go.mod | 4 ++-- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/chain/validation/applier.go b/chain/validation/applier.go index f30c543bb..6c1629104 100644 --- a/chain/validation/applier.go +++ b/chain/validation/applier.go @@ -13,7 +13,6 @@ import ( "github.com/ipfs/go-cid" vtypes "github.com/filecoin-project/chain-validation/chain/types" - vdrivers "github.com/filecoin-project/chain-validation/drivers" vstate "github.com/filecoin-project/chain-validation/state" "github.com/filecoin-project/lotus/chain/stmgr" @@ -136,7 +135,7 @@ func (a *Applier) applyMessage(epoch abi.ChainEpoch, lm types.ChainMsg) (vtypes. ctx := context.TODO() base := a.stateWrapper.Root() - lotusVM, err := vm.NewVM(base, epoch, &vmRand{}, a.stateWrapper.bs, vdrivers.NewChainValidationSyscalls()) + lotusVM, err := vm.NewVM(base, epoch, &vmRand{}, a.stateWrapper.bs, a.syscalls) // need to modify the VM invoker to add the puppet actor chainValInvoker := vm.NewInvoker() chainValInvoker.Register(puppet.PuppetActorCodeID, puppet.Actor{}, puppet.State{}) diff --git a/go.mod b/go.mod index b587919c2..642b3576c 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,8 @@ require ( github.com/drand/drand v0.9.2-0.20200526173017-9cffec0d074e github.com/drand/kyber v1.0.2 github.com/fatih/color v1.8.0 - github.com/filecoin-project/chain-validation v0.0.6-0.20200518190139-483332336e8e - github.com/filecoin-project/filecoin-ffi v0.0.0-20200427223233-a0014b17f124 + github.com/filecoin-project/chain-validation v0.0.6-0.20200526171800-c56c1882dc99 + github.com/filecoin-project/filecoin-ffi v0.26.1-0.20200508175440-05b30afeb00d github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2 github.com/filecoin-project/go-bitfield v0.0.1 diff --git a/go.sum b/go.sum index 29cd94e09..f25e4f97e 100644 --- a/go.sum +++ b/go.sum @@ -164,8 +164,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= -github.com/filecoin-project/chain-validation v0.0.6-0.20200518190139-483332336e8e h1:3x2eL2t3ZkMOHt1b5WS5aVWyJeo5+WjWCT77QdPGSwk= -github.com/filecoin-project/chain-validation v0.0.6-0.20200518190139-483332336e8e/go.mod h1:6B3uenDcH8n+PKqgzUtZmgyCzKy4qpiLwJ5aw7Rj2xQ= +github.com/filecoin-project/chain-validation v0.0.6-0.20200526171800-c56c1882dc99 h1:jVuyHbsCX/w4vZ+3pO0cqzOyqvwfo8tB94wzmcwaD2k= +github.com/filecoin-project/chain-validation v0.0.6-0.20200526171800-c56c1882dc99/go.mod h1:hMqMCTt3z+wZ+GL74uy7X4NptnucZTTrb5SUTt1L/0g= github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef h1:Wi5E+P1QfHP8IF27eUiTx5vYfqQZwfPxzq3oFEq8w8U= From 514339e855a20df8c09e0d6ac0d66705b0374640 Mon Sep 17 00:00:00 2001 From: Mike Greenberg Date: Wed, 27 May 2020 17:12:45 -0400 Subject: [PATCH 021/160] update systemd services; include make directives and docs for use --- Makefile | 12 ++++++++ documentation/en/install-systemd-services.md | 29 ++++++++++++++++++++ scripts/lotus-daemon.service | 4 +++ 3 files changed, 45 insertions(+) create mode 100644 documentation/en/install-systemd-services.md diff --git a/Makefile b/Makefile index a189ccca2..37f07b0b6 100644 --- a/Makefile +++ b/Makefile @@ -103,6 +103,18 @@ install: install -C ./lotus-storage-miner /usr/local/bin/lotus-storage-miner install -C ./lotus-seal-worker /usr/local/bin/lotus-seal-worker +install-services: + install -C -m 0644 ./scripts/lotus-daemon.service /etc/systemd/system/lotus-daemon.service + install -C -m 0644 ./scripts/lotus-miner.service /etc/systemd/system/lotus-miner.service + systemctl daemon-reload + @echo + @echo "lotus and lotus-miner services installed. Don't forget to 'systemctl enable lotus|lotus-miner' for it to be enabled on startup." + +clean-services: + rm -f /etc/systemd/system/lotus-daemon.service + rm -f /etc/systemd/system/lotus-miner.service + systemctl daemon-reload + # TOOLS lotus-seed: $(BUILD_DEPS) diff --git a/documentation/en/install-systemd-services.md b/documentation/en/install-systemd-services.md new file mode 100644 index 000000000..8c83d56a1 --- /dev/null +++ b/documentation/en/install-systemd-services.md @@ -0,0 +1,29 @@ +# Lotus and Miner as a systemd service + +Lotus is capable of running as a systemd service daemon. You can find installable service files for systemd in the [lotus repo](https://github.com/filecoin-project/lotus/tree/master/scripts) as files with `.service` extension. In order to install these service files, you can copy these `.service` files to the default systemd service path. + +## Installing via `make` + +NOTE: Before using lotus and lotus-miner as systemd services, don't forget to `sudo make install` to ensure the binaries are accessible by the root user. + +If your host uses the default systemd service path, it can be installed with `sudo make install-services`: + +```sh +$ sudo make install-services +``` + +## Interacting with service logs + +Logs from the services can be reviewed using `journalctl`. + +### Follow logs from a specific service unit + +```sh +$ sudo journalctl -u lotus-daemon -f +``` + +### View logs in reverse order + +```sh +$ sudo journalctl -u lotus-miner -r +``` diff --git a/scripts/lotus-daemon.service b/scripts/lotus-daemon.service index 44b2058b0..86f0dff27 100644 --- a/scripts/lotus-daemon.service +++ b/scripts/lotus-daemon.service @@ -6,6 +6,10 @@ After=network.target ExecStart=/usr/local/bin/lotus daemon Environment=GOLOG_FILE="/var/log/lotus-daemon" Environment=GOLOG_LOG_FMT="json" +MemoryAccounting=true +MemoryHigh=8G +MemoryMax=10G +LimitNOFILE=8192:10240 [Install] WantedBy=multiuser.target From 8e55c91cd232d71a4ec8c134fbf7a01952c13b4b Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 28 May 2020 08:45:34 -0700 Subject: [PATCH 022/160] check neq zero exit code --- storage/wdpost_run.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index cfee3cde3..63109e5f5 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -4,9 +4,10 @@ import ( "bytes" "context" "errors" - "github.com/filecoin-project/go-bitfield" "time" + "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/builtin" @@ -171,7 +172,7 @@ func (s *WindowPoStScheduler) checkRecoveries(ctx context.Context, deadline uint return xerrors.Errorf("declare faults recovered wait error: %w", err) } - if rec.Receipt.ExitCode == 0 { + if rec.Receipt.ExitCode != 0 { return xerrors.Errorf("declare faults recovered wait non-0 exit code: %d", rec.Receipt.ExitCode) } From 91da81c8bc8e9542f1ce5910847056a89ad9a0f5 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 28 May 2020 13:17:55 -0700 Subject: [PATCH 023/160] mod tidy --- go.sum | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/go.sum b/go.sum index f25e4f97e..fb4286856 100644 --- a/go.sum +++ b/go.sum @@ -206,7 +206,6 @@ github.com/filecoin-project/sector-storage v0.0.0-20200522011946-a59ca7536a95 h1 github.com/filecoin-project/sector-storage v0.0.0-20200522011946-a59ca7536a95/go.mod h1:1UpuWCUC7XNfgyufyQQq+/40JSzhfGdgcwauK55C8gk= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= -github.com/filecoin-project/specs-actors v0.2.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= github.com/filecoin-project/specs-actors v0.5.2/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= github.com/filecoin-project/specs-actors v0.5.4-0.20200521014528-0df536f7e461 h1:yyAoJ9dNboljDWj0uBdJEbdaTak/YhkjYUQt0GzlY0A= @@ -727,6 +726,7 @@ github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYc github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= +github.com/libp2p/go-libp2p-pubsub v0.3.0/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato= github.com/libp2p/go-libp2p-pubsub v0.3.1 h1:7Hyv2d8BK/x1HGRJTZ8X++VQEP+WqDTSwpUSZGTVLYA= github.com/libp2p/go-libp2p-pubsub v0.3.1/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato= github.com/libp2p/go-libp2p-quic-transport v0.1.1 h1:MFMJzvsxIEDEVKzO89BnB/FgvMj9WI4GDGUW2ArDPUA= @@ -1255,6 +1255,7 @@ golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200427165652-729f1e841bcc/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1359,6 +1360,7 @@ golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200427175716-29b57079015a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From e2a554c2e68f4e17f09bab3c536961c37c958fe3 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 27 May 2020 17:06:29 -0700 Subject: [PATCH 024/160] update specs actors to version with batch porep verification --- api/api_full.go | 1 + api/apistruct/struct.go | 5 ++++ chain/stmgr/utils.go | 18 +++++++++++++ chain/vm/gas.go | 6 +++++ chain/vm/syscalls.go | 27 +++++++++++++++++++ cmd/lotus-chainwatch/sync.go | 13 ++++----- go.mod | 6 +++++ go.sum | 22 --------------- .../utils.go => lib/parmap/parmap.go | 10 +++---- node/impl/full/state.go | 8 ++++++ storage/adapter_storage_miner.go | 9 +++++++ storage/miner.go | 1 + 12 files changed, 93 insertions(+), 33 deletions(-) rename cmd/lotus-chainwatch/utils.go => lib/parmap/parmap.go (86%) diff --git a/api/api_full.go b/api/api_full.go index c6d442e2b..a1155c206 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -152,6 +152,7 @@ type FullNode interface { StateMinerInitialPledgeCollateral(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (types.BigInt, error) StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) + StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) StatePledgeCollateral(context.Context, types.TipSetKey) (types.BigInt, error) StateWaitMsg(context.Context, cid.Cid) (*MsgLookup, error) StateSearchMsg(context.Context, cid.Cid) (*MsgLookup, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index f0be44070..c97c1285a 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -127,6 +127,7 @@ type FullNodeStruct struct { StateMinerInitialPledgeCollateral func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (types.BigInt, error) `perm:"read"` StateMinerAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) `perm:"read"` StateSectorPreCommitInfo func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) `perm:"read"` + StateSectorGetInfo func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) `perm:"read"` StateCall func(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) `perm:"read"` StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` @@ -556,6 +557,10 @@ func (c *FullNodeStruct) StateSectorPreCommitInfo(ctx context.Context, maddr add return c.Internal.StateSectorPreCommitInfo(ctx, maddr, n, tsk) } +func (c *FullNodeStruct) StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error) { + return c.Internal.StateSectorGetInfo(ctx, maddr, n, tsk) +} + func (c *FullNodeStruct) StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error) { return c.Internal.StateCall(ctx, msg, tsk) } diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 0ae7abb4e..001882ec6 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -138,6 +138,24 @@ func PreCommitInfo(ctx context.Context, sm *StateManager, maddr address.Address, return *i, nil } +func MinerSectorInfo(ctx context.Context, sm *StateManager, maddr address.Address, sid abi.SectorNumber, ts *types.TipSet) (*miner.SectorOnChainInfo, error) { + var mas miner.State + _, err := sm.LoadActorState(ctx, maddr, &mas, ts) + if err != nil { + return nil, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err) + } + + sectorInfo, ok, err := mas.GetSector(sm.cs.Store(ctx), sid) + if err != nil { + return nil, err + } + if !ok { + return nil, xerrors.New("sector not found") + } + + return sectorInfo, nil +} + func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address, filter *abi.BitField, filterOut bool) ([]*api.ChainSectorInfo, error) { var mas miner.State _, err := sm.LoadActorState(ctx, maddr, &mas, ts) diff --git a/chain/vm/gas.go b/chain/vm/gas.go index a24e54b78..f1cf89845 100644 --- a/chain/vm/gas.go +++ b/chain/vm/gas.go @@ -3,6 +3,7 @@ package vm import ( "fmt" + "github.com/filecoin-project/go-address" addr "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/crypto" @@ -143,3 +144,8 @@ func (ps pricedSyscalls) VerifyConsensusFault(h1 []byte, h2 []byte, extra []byte ps.chargeGas(ps.pl.OnVerifyConsensusFault()) return ps.under.VerifyConsensusFault(h1, h2, extra) } + +func (ps pricedSyscalls) BatchVerifySeals(inp map[address.Address][]abi.SealVerifyInfo) (map[address.Address][]bool, error) { + ps.chargeGas(0) // TODO: this is only called by the cron actor. Should we even charge gas? + return ps.under.BatchVerifySeals(inp) +} diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index c3fe4375f..9f9610f4b 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "sync" "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" @@ -252,3 +253,29 @@ func (ss *syscallShim) VerifySignature(sig crypto.Signature, addr address.Addres return sigs.Verify(&sig, kaddr, input) } + +func (ss *syscallShim) BatchVerifySeals(inp map[address.Address][]abi.SealVerifyInfo) (map[address.Address][]bool, error) { + out := make(map[address.Address][]bool) + + var wg sync.WaitGroup + for addr, seals := range inp { + results := make([]bool, len(seals)) + out[addr] = results + + for i, s := range seals { + wg.Add(1) + go func(ma address.Address, ix int, svi abi.SealVerifyInfo, res []bool) { + defer wg.Done() + if err := ss.VerifySeal(svi); err != nil { + log.Warnw("seal verify in batch failed", "miner", ma, "index", ix, "err", err) + res[ix] = false + } else { + res[ix] = true + } + }(addr, i, s, results) + } + } + wg.Wait() + + return out, nil +} diff --git a/cmd/lotus-chainwatch/sync.go b/cmd/lotus-chainwatch/sync.go index 8fa903097..f009e47d9 100644 --- a/cmd/lotus-chainwatch/sync.go +++ b/cmd/lotus-chainwatch/sync.go @@ -20,6 +20,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + parmap "github.com/filecoin-project/lotus/lib/parmap" ) func runSyncer(ctx context.Context, api api.FullNode, st *storage, maxBatch int) { @@ -138,7 +139,7 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS log.Infof("Syncing %d blocks", len(toSync)) paDone := 0 - par(50, maparr(toSync), func(bh *types.BlockHeader) { + parmap.Par(50, parmap.MapArr(toSync), func(bh *types.BlockHeader) { paDone++ if paDone%100 == 0 { log.Infof("pa: %d %d%%", paDone, (paDone*100)/len(toSync)) @@ -152,7 +153,7 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS return } - par(50, aadrs, func(addr address.Address) { + parmap.Par(50, aadrs, func(addr address.Address) { act, err := api.StateGetActor(ctx, addr, ts.Key()) if err != nil { log.Error(err) @@ -239,7 +240,7 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS addresses[message.From] = address.Undef } - par(50, kmaparr(addresses), func(addr address.Address) { + parmap.Par(50, parmap.KMapArr(addresses), func(addr address.Address) { raddr, err := api.StateLookupID(ctx, addr, types.EmptyTSK) if err != nil { log.Warn(err) @@ -268,7 +269,7 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS } } - par(50, kvmaparr(miners), func(it func() (minerKey, *minerInfo)) { + parmap.Par(50, parmap.KVMapArr(miners), func(it func() (minerKey, *minerInfo)) { k, info := it() pow, err := api.StateMinerPower(ctx, k.addr, types.EmptyTSK) @@ -386,7 +387,7 @@ func fetchMessages(ctx context.Context, api api.FullNode, toSync map[cid.Cid]*ty messages := map[cid.Cid]*types.Message{} inclusions := map[cid.Cid][]cid.Cid{} // block -> msgs - par(50, maparr(toSync), func(header *types.BlockHeader) { + parmap.Par(50, parmap.MapArr(toSync), func(header *types.BlockHeader) { msgs, err := api.ChainGetBlockMessages(ctx, header.Cid()) if err != nil { log.Error(err) @@ -423,7 +424,7 @@ func fetchParentReceipts(ctx context.Context, api api.FullNode, toSync map[cid.C var lk sync.Mutex out := map[mrec]*types.MessageReceipt{} - par(50, maparr(toSync), func(header *types.BlockHeader) { + parmap.Par(50, parmap.MapArr(toSync), func(header *types.BlockHeader) { recs, err := api.ChainGetParentReceipts(ctx, header.Cid()) if err != nil { log.Error(err) diff --git a/go.mod b/go.mod index 642b3576c..6d87e15f7 100644 --- a/go.mod +++ b/go.mod @@ -121,3 +121,9 @@ require ( replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v1.18.0 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi + +replace github.com/filecoin-project/specs-actors => ../specs-actors + +replace github.com/filecoin-project/chain-validation => ../chain-validation + +replace github.com/filecoin-project/storage-fsm => ../storage-fsm diff --git a/go.sum b/go.sum index fb4286856..8925a25f9 100644 --- a/go.sum +++ b/go.sum @@ -10,7 +10,6 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= @@ -30,7 +29,6 @@ github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= -github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -164,17 +162,11 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= -github.com/filecoin-project/chain-validation v0.0.6-0.20200526171800-c56c1882dc99 h1:jVuyHbsCX/w4vZ+3pO0cqzOyqvwfo8tB94wzmcwaD2k= -github.com/filecoin-project/chain-validation v0.0.6-0.20200526171800-c56c1882dc99/go.mod h1:hMqMCTt3z+wZ+GL74uy7X4NptnucZTTrb5SUTt1L/0g= -github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef h1:Wi5E+P1QfHP8IF27eUiTx5vYfqQZwfPxzq3oFEq8w8U= github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef/go.mod h1:SrA+pWVoUivqKOfC+ckVYbx41hWz++HxJcrlmHNnebU= -github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg= github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2 h1:jamfsxfK0Q9yCMHt8MPWx7Aa/O9k2Lve8eSc6FILYGQ= github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg= -github.com/filecoin-project/go-bitfield v0.0.0-20200309034705-8c7ac40bd550/go.mod h1:iodsLxOFZnqKtjj2zkgqzoGNrv6vUqj69AT/J8DKXEw= -github.com/filecoin-project/go-bitfield v0.0.0-20200416002808-b3ee67ec9060/go.mod h1:iodsLxOFZnqKtjj2zkgqzoGNrv6vUqj69AT/J8DKXEw= github.com/filecoin-project/go-bitfield v0.0.1 h1:Xg/JnrqqE77aJVKdbEyR04n9FZQWhwrN+buDgQCVpZU= github.com/filecoin-project/go-bitfield v0.0.1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= @@ -204,16 +196,8 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/ github.com/filecoin-project/sector-storage v0.0.0-20200508203401-a74812ba12f3/go.mod h1:B+xzopr/oWZJz2hBL5Ekb7Obcum5ntmfbaAUlaaho28= github.com/filecoin-project/sector-storage v0.0.0-20200522011946-a59ca7536a95 h1:wNpbsUY11P7+3yBfHOXsH+tAyferf1BVSe5S+HRm+7s= github.com/filecoin-project/sector-storage v0.0.0-20200522011946-a59ca7536a95/go.mod h1:1UpuWCUC7XNfgyufyQQq+/40JSzhfGdgcwauK55C8gk= -github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= -github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= -github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= -github.com/filecoin-project/specs-actors v0.5.2/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= -github.com/filecoin-project/specs-actors v0.5.4-0.20200521014528-0df536f7e461 h1:yyAoJ9dNboljDWj0uBdJEbdaTak/YhkjYUQt0GzlY0A= -github.com/filecoin-project/specs-actors v0.5.4-0.20200521014528-0df536f7e461/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 h1:T3f/zkuvgtgqcXrb0NO3BicuveGOxxUAMPa/Yif2kuE= github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102/go.mod h1:xJ1/xl9+8zZeSSSFmDC3Wr6uusCTxyYPI0VeNVSFmPE= -github.com/filecoin-project/storage-fsm v0.0.0-20200522010518-83fd743db8bc h1:6du9HB53x8frk+Cky9ymGYsRbvXZDE91EsaOJoCb5Ok= -github.com/filecoin-project/storage-fsm v0.0.0-20200522010518-83fd743db8bc/go.mod h1:S0u14Wr55mpe22lElCSKbXrhtWg/jquVTTMhefQ8f4Q= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -478,7 +462,6 @@ github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBW github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= github.com/ipfs/go-log/v2 v2.0.8/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= -github.com/ipfs/go-log/v2 v2.1.0 h1:7l5yjP7aLl5xr7RmqEvPZy9Skq7PP2fIP0Pa7lU/UII= github.com/ipfs/go-log/v2 v2.1.0/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/ipfs/go-log/v2 v2.1.1 h1:G4TtqN+V9y9HY9TA6BwbCVyyBZ2B9MbCjR2MtGx8FR0= github.com/ipfs/go-log/v2 v2.1.1/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= @@ -537,7 +520,6 @@ github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZl github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= @@ -970,7 +952,6 @@ github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c h1:5bFTChQxSKNwy8ALwOebjekYExl9HTT9urdawqC95tA= github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c/go.mod h1:7qN3Y0BvzRUf4LofcoJplQL10lsFDb4PYlePTVwrP28= -github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= @@ -1132,9 +1113,7 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= @@ -1151,7 +1130,6 @@ github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba/go.mod h1:CH github.com/whyrusleeping/cbor-gen v0.0.0-20191212224538-d370462a7e8a/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= -github.com/whyrusleeping/cbor-gen v0.0.0-20200206220010-03c9665e2a66/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20200402171437-3d27c146c105/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20200414195334-429a0b5e922e/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20200501014322-5f9941ef88e0/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= diff --git a/cmd/lotus-chainwatch/utils.go b/lib/parmap/parmap.go similarity index 86% rename from cmd/lotus-chainwatch/utils.go rename to lib/parmap/parmap.go index a5c821683..50613509e 100644 --- a/cmd/lotus-chainwatch/utils.go +++ b/lib/parmap/parmap.go @@ -1,11 +1,11 @@ -package main +package parmap import ( "reflect" "sync" ) -func maparr(in interface{}) interface{} { +func MapArr(in interface{}) interface{} { rin := reflect.ValueOf(in) rout := reflect.MakeSlice(reflect.SliceOf(rin.Type().Elem()), rin.Len(), rin.Len()) var i int @@ -19,7 +19,7 @@ func maparr(in interface{}) interface{} { return rout.Interface() } -func kmaparr(in interface{}) interface{} { +func KMapArr(in interface{}) interface{} { rin := reflect.ValueOf(in) rout := reflect.MakeSlice(reflect.SliceOf(rin.Type().Key()), rin.Len(), rin.Len()) var i int @@ -34,7 +34,7 @@ func kmaparr(in interface{}) interface{} { } // map[k]v => []func() (k, v) -func kvmaparr(in interface{}) interface{} { +func KVMapArr(in interface{}) interface{} { rin := reflect.ValueOf(in) t := reflect.FuncOf([]reflect.Type{}, []reflect.Type{ @@ -59,7 +59,7 @@ func kvmaparr(in interface{}) interface{} { return rout.Interface() } -func par(concurrency int, arr interface{}, f interface{}) { +func Par(concurrency int, arr interface{}, f interface{}) { throttle := make(chan struct{}, concurrency) var wg sync.WaitGroup diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 58633b27a..f160efa49 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -539,6 +539,14 @@ func (a *StateAPI) StateSectorPreCommitInfo(ctx context.Context, maddr address.A return stmgr.PreCommitInfo(ctx, a.StateManager, maddr, n, ts) } +func (a *StateAPI) StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error) { + ts, err := a.Chain.GetTipSetFromKey(tsk) + if err != nil { + return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) + } + return stmgr.MinerSectorInfo(ctx, a.StateManager, maddr, n, ts) +} + func (a *StateAPI) StateListMessages(ctx context.Context, match *types.Message, tsk types.TipSetKey, toheight abi.ChainEpoch) ([]cid.Cid, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { diff --git a/storage/adapter_storage_miner.go b/storage/adapter_storage_miner.go index 391a38b37..09197e71a 100644 --- a/storage/adapter_storage_miner.go +++ b/storage/adapter_storage_miner.go @@ -170,6 +170,15 @@ func (s SealingAPIAdapter) StateSectorPreCommitInfo(ctx context.Context, maddr a return &pci, nil } +func (s SealingAPIAdapter) StateSectorGetInfo(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok sealing.TipSetToken) (*miner.SectorOnChainInfo, error) { + tsk, err := types.TipSetKeyFromBytes(tok) + if err != nil { + return nil, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) + } + + return s.delegate.StateSectorGetInfo(ctx, maddr, sectorNumber, tsk) +} + func (s SealingAPIAdapter) StateMarketStorageDeal(ctx context.Context, dealID abi.DealID, tok sealing.TipSetToken) (market.DealProposal, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { diff --git a/storage/miner.go b/storage/miner.go index 779f981e9..2512a7124 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -50,6 +50,7 @@ type storageMinerApi interface { StateMinerDeadlines(ctx context.Context, maddr address.Address, tok types.TipSetKey) (*miner.Deadlines, error) StateMinerSectors(context.Context, address.Address, *abi.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error) StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) + StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error) StateMinerInitialPledgeCollateral(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (types.BigInt, error) From fa04da29953fae9f754030db87a058980ea9ea7f Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 28 May 2020 01:22:25 -0400 Subject: [PATCH 025/160] Update specs-actors and storage-fsm --- go.mod | 8 ++------ go.sum | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 6d87e15f7..84e1705b3 100644 --- a/go.mod +++ b/go.mod @@ -30,9 +30,9 @@ require ( github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/sector-storage v0.0.0-20200522011946-a59ca7536a95 - github.com/filecoin-project/specs-actors v0.5.4-0.20200521014528-0df536f7e461 + github.com/filecoin-project/specs-actors v0.5.4 github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 - github.com/filecoin-project/storage-fsm v0.0.0-20200522010518-83fd743db8bc + github.com/filecoin-project/storage-fsm v0.0.0-20200528050623-cdada6e88960 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-ole/go-ole v1.2.4 // indirect github.com/google/uuid v1.1.1 @@ -122,8 +122,4 @@ replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi -replace github.com/filecoin-project/specs-actors => ../specs-actors - replace github.com/filecoin-project/chain-validation => ../chain-validation - -replace github.com/filecoin-project/storage-fsm => ../storage-fsm diff --git a/go.sum b/go.sum index 8925a25f9..0ed1bae2a 100644 --- a/go.sum +++ b/go.sum @@ -162,11 +162,15 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= +github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef h1:Wi5E+P1QfHP8IF27eUiTx5vYfqQZwfPxzq3oFEq8w8U= github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef/go.mod h1:SrA+pWVoUivqKOfC+ckVYbx41hWz++HxJcrlmHNnebU= +github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg= github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2 h1:jamfsxfK0Q9yCMHt8MPWx7Aa/O9k2Lve8eSc6FILYGQ= github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg= +github.com/filecoin-project/go-bitfield v0.0.0-20200309034705-8c7ac40bd550/go.mod h1:iodsLxOFZnqKtjj2zkgqzoGNrv6vUqj69AT/J8DKXEw= +github.com/filecoin-project/go-bitfield v0.0.0-20200416002808-b3ee67ec9060/go.mod h1:iodsLxOFZnqKtjj2zkgqzoGNrv6vUqj69AT/J8DKXEw= github.com/filecoin-project/go-bitfield v0.0.1 h1:Xg/JnrqqE77aJVKdbEyR04n9FZQWhwrN+buDgQCVpZU= github.com/filecoin-project/go-bitfield v0.0.1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= @@ -196,8 +200,17 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/ github.com/filecoin-project/sector-storage v0.0.0-20200508203401-a74812ba12f3/go.mod h1:B+xzopr/oWZJz2hBL5Ekb7Obcum5ntmfbaAUlaaho28= github.com/filecoin-project/sector-storage v0.0.0-20200522011946-a59ca7536a95 h1:wNpbsUY11P7+3yBfHOXsH+tAyferf1BVSe5S+HRm+7s= github.com/filecoin-project/sector-storage v0.0.0-20200522011946-a59ca7536a95/go.mod h1:1UpuWCUC7XNfgyufyQQq+/40JSzhfGdgcwauK55C8gk= +github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= +github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= +github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= +github.com/filecoin-project/specs-actors v0.5.2/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= +github.com/filecoin-project/specs-actors v0.5.4-0.20200521014528-0df536f7e461/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= +github.com/filecoin-project/specs-actors v0.5.4 h1:Od00SYb/pTXC84oZGz9h2xGBm8GIHo8GXUaRKg1ClZo= +github.com/filecoin-project/specs-actors v0.5.4/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 h1:T3f/zkuvgtgqcXrb0NO3BicuveGOxxUAMPa/Yif2kuE= github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102/go.mod h1:xJ1/xl9+8zZeSSSFmDC3Wr6uusCTxyYPI0VeNVSFmPE= +github.com/filecoin-project/storage-fsm v0.0.0-20200528050623-cdada6e88960 h1:KXkOn1r3lyxKn2rHUcVox6WYtfIEsgyZdBVoIVJARjY= +github.com/filecoin-project/storage-fsm v0.0.0-20200528050623-cdada6e88960/go.mod h1:S0u14Wr55mpe22lElCSKbXrhtWg/jquVTTMhefQ8f4Q= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -1130,6 +1143,7 @@ github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba/go.mod h1:CH github.com/whyrusleeping/cbor-gen v0.0.0-20191212224538-d370462a7e8a/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= +github.com/whyrusleeping/cbor-gen v0.0.0-20200206220010-03c9665e2a66/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20200402171437-3d27c146c105/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20200414195334-429a0b5e922e/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20200501014322-5f9941ef88e0/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= From fa69adf45ca80e6d043461846c9ef64c51ae145c Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 28 May 2020 15:58:31 -0400 Subject: [PATCH 026/160] update chain-validation --- go.mod | 4 +--- go.sum | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 84e1705b3..0a5d64d52 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/drand/drand v0.9.2-0.20200526173017-9cffec0d074e github.com/drand/kyber v1.0.2 github.com/fatih/color v1.8.0 - github.com/filecoin-project/chain-validation v0.0.6-0.20200526171800-c56c1882dc99 + github.com/filecoin-project/chain-validation v0.0.6-0.20200528194021-3ce40d696ae1 github.com/filecoin-project/filecoin-ffi v0.26.1-0.20200508175440-05b30afeb00d github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2 @@ -121,5 +121,3 @@ require ( replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v1.18.0 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi - -replace github.com/filecoin-project/chain-validation => ../chain-validation diff --git a/go.sum b/go.sum index 0ed1bae2a..7a29003fe 100644 --- a/go.sum +++ b/go.sum @@ -162,6 +162,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= +github.com/filecoin-project/chain-validation v0.0.6-0.20200528194021-3ce40d696ae1 h1:cJPPzINJq4oUbdC4gQyMrEzuSiJ95RKxyXsLn6UgfcI= +github.com/filecoin-project/chain-validation v0.0.6-0.20200528194021-3ce40d696ae1/go.mod h1:aV5C4aX+kZwh5OlTHNCHxQ5vTnpTFKUuKiDTk9f/8Es= github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef h1:Wi5E+P1QfHP8IF27eUiTx5vYfqQZwfPxzq3oFEq8w8U= @@ -203,7 +205,6 @@ github.com/filecoin-project/sector-storage v0.0.0-20200522011946-a59ca7536a95/go github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= -github.com/filecoin-project/specs-actors v0.5.2/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= github.com/filecoin-project/specs-actors v0.5.4-0.20200521014528-0df536f7e461/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= github.com/filecoin-project/specs-actors v0.5.4 h1:Od00SYb/pTXC84oZGz9h2xGBm8GIHo8GXUaRKg1ClZo= github.com/filecoin-project/specs-actors v0.5.4/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= From 5129fed7c89d2f8017a0524d87086925f6c9a9e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 29 May 2020 14:36:56 +0200 Subject: [PATCH 027/160] pond: rerun method-gen --- go.sum | 3 +++ lotuspond/front/src/chain/methods.json | 9 +++++---- lotuspond/spawn.go | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/go.sum b/go.sum index 7a29003fe..87f4f9148 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,8 @@ github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -966,6 +968,7 @@ github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c h1:5bFTChQxSKNwy8ALwOebjekYExl9HTT9urdawqC95tA= github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c/go.mod h1:7qN3Y0BvzRUf4LofcoJplQL10lsFDb4PYlePTVwrP28= +github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= diff --git a/lotuspond/front/src/chain/methods.json b/lotuspond/front/src/chain/methods.json index 5a2de5d7f..0d0658716 100644 --- a/lotuspond/front/src/chain/methods.json +++ b/lotuspond/front/src/chain/methods.json @@ -44,12 +44,11 @@ "Constructor", "AddBalance", "WithdrawBalance", - "HandleExpiredDeals", "PublishStorageDeals", "VerifyDealsOnSectorProveCommit", "OnMinerSectorsTerminate", "ComputeDataCommitment", - "HandleInitTimeoutDeals" + "CronTick" ], "fil/1/storageminer": [ "Send", @@ -68,7 +67,8 @@ "CheckSectorProven", "AddLockedFund", "ReportConsensusFault", - "WithdrawBalance" + "WithdrawBalance", + "ConfirmSectorProofsValid" ], "fil/1/storagepower": [ "Send", @@ -83,7 +83,8 @@ "EnrollCronEvent", "OnEpochTickEnd", "UpdatePledgeTotal", - "OnConsensusFault" + "OnConsensusFault", + "SubmitPoRepForBulkVerify" ], "fil/1/verifiedregistry": [ "Send", diff --git a/lotuspond/spawn.go b/lotuspond/spawn.go index 5b0f7f704..76b3f3aa2 100644 --- a/lotuspond/spawn.go +++ b/lotuspond/spawn.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "fmt" + "github.com/filecoin-project/lotus/chain/types" "io" "io/ioutil" "os" @@ -16,7 +17,6 @@ import ( "github.com/filecoin-project/go-address" genesis2 "github.com/filecoin-project/lotus/chain/gen/genesis" "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/filecoin-project/specs-actors/actors/abi/big" "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" @@ -65,7 +65,7 @@ func (api *api) Spawn() (nodeInfo, error) { template.Miners = append(template.Miners, *genm) template.Accounts = append(template.Accounts, genesis.Actor{ Type: genesis.TAccount, - Balance: big.NewInt(100000000000000), + Balance: types.FromFil(5000000), Meta: (&genesis.AccountMeta{Owner: genm.Owner}).ActorMeta(), }) From 71eb41b9aa3deb42a0b6002e219b669b6ca34839 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 29 May 2020 15:46:05 +0200 Subject: [PATCH 028/160] Add pubsub support to drand beacon Signed-off-by: Jakub Sztandera --- chain/beacon/drand/drand.go | 34 ++++++++++++++++++++++++---------- go.mod | 3 ++- go.sum | 8 ++++++-- node/builder.go | 1 - node/modules/services.go | 13 ++++++++++--- node/modules/storageminer.go | 11 ----------- 6 files changed, 42 insertions(+), 28 deletions(-) diff --git a/chain/beacon/drand/drand.go b/chain/beacon/drand/drand.go index d8df81f03..0de519f1e 100644 --- a/chain/beacon/drand/drand.go +++ b/chain/beacon/drand/drand.go @@ -6,17 +6,22 @@ import ( "sync" "time" + dchain "github.com/drand/drand/chain" + dclient "github.com/drand/drand/client" + gclient "github.com/drand/drand/cmd/relay-gossip/client" + dlog "github.com/drand/drand/log" "github.com/drand/kyber" + kzap "github.com/go-kit/kit/log/zap" + "go.uber.org/zap/zapcore" + "golang.org/x/xerrors" + + logging "github.com/ipfs/go-log" + pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/abi" - "golang.org/x/xerrors" - - logging "github.com/ipfs/go-log" - - dchain "github.com/drand/drand/chain" - dclient "github.com/drand/drand/client" ) var log = logging.Logger("drand") @@ -67,15 +72,24 @@ type DrandBeacon struct { localCache map[uint64]types.BeaconEntry } -func NewDrandBeacon(genesisTs, interval uint64) (*DrandBeacon, error) { +func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub) (*DrandBeacon, error) { if genesisTs == 0 { panic("what are you doing this cant be zero") } - client, err := dclient.New( - dclient.WithHTTPEndpoints(drandServers), + opts := []dclient.Option{ dclient.WithChainInfo(drandChain), + dclient.WithHTTPEndpoints(drandServers), dclient.WithCacheSize(1024), - ) + dclient.WithLogger(dlog.NewKitLoggerFrom(kzap.NewZapSugarLogger( + log.SugaredLogger.Desugar(), zapcore.InfoLevel))), + } + if ps != nil { + opts = append(opts, gclient.WithPubsub(ps)) + } else { + log.Info("drand beacon without pubsub") + } + + client, err := dclient.New(opts...) if err != nil { return nil, xerrors.Errorf("creating drand client") } diff --git a/go.mod b/go.mod index 0a5d64d52..a1f550c27 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/coreos/go-systemd/v22 v22.0.0 github.com/docker/go-units v0.4.0 - github.com/drand/drand v0.9.2-0.20200526173017-9cffec0d074e + github.com/drand/drand v0.9.2-0.20200529123141-6b138aefaef2 github.com/drand/kyber v1.0.2 github.com/fatih/color v1.8.0 github.com/filecoin-project/chain-validation v0.0.6-0.20200528194021-3ce40d696ae1 @@ -34,6 +34,7 @@ require ( github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 github.com/filecoin-project/storage-fsm v0.0.0-20200528050623-cdada6e88960 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 + github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.4 diff --git a/go.sum b/go.sum index 7a29003fe..f249f4296 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,7 @@ contrib.go.opencensus.io/exporter/jaeger v0.1.0/go.mod h1:VYianECmuFPwU37O699Vc1 contrib.go.opencensus.io/exporter/prometheus v0.1.0 h1:SByaIoWwNgMdPSgl5sMqM2KDE5H/ukPWBRo314xiDvg= contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -134,8 +135,8 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/drand/bls12-381 v0.3.2 h1:RImU8Wckmx8XQx1tp1q04OV73J9Tj6mmpQLYDP7V1XE= github.com/drand/bls12-381 v0.3.2/go.mod h1:dtcLgPtYT38L3NO6mPDYH0nbpc5tjPassDqiniuAt4Y= -github.com/drand/drand v0.9.2-0.20200526173017-9cffec0d074e h1:e9JEgW+3GaPAqskYoEtL98HueHmUskO5tsRqmPs1ZxE= -github.com/drand/drand v0.9.2-0.20200526173017-9cffec0d074e/go.mod h1:xRGkp/+RCz/BWvF1KjaH2AGJ81hrGFqA6Uml1k1zNto= +github.com/drand/drand v0.9.2-0.20200529123141-6b138aefaef2 h1:IUxiqYU8BM8CyjHqxOTUbjrq7OvSDm8n1v9igaxtapM= +github.com/drand/drand v0.9.2-0.20200529123141-6b138aefaef2/go.mod h1:UIhZq1vHM1BQbpLa9K2HkxpCtbdsGkCc9MnCdKJxYjk= github.com/drand/kyber v1.0.1-0.20200110225416-8de27ed8c0e2/go.mod h1:UpXoA0Upd1N9l4TvRPHr1qAUBBERj6JQ/mnKI3BPEmw= github.com/drand/kyber v1.0.2 h1:dHjtWJZJdn3zBBZ9pqLsLfcR9ScvDvSqzS1sWA8seao= github.com/drand/kyber v1.0.2/go.mod h1:x6KOpK7avKj0GJ4emhXFP5n7M7W7ChAPmnQh/OL6vRw= @@ -390,12 +391,14 @@ github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjv github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s= github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE= +github.com/ipfs/go-ds-badger v0.2.3 h1:J27YvAcpuA5IvZUbeBxOcQgqnYHUPxoygc6QxxkodZ4= github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk= github.com/ipfs/go-ds-badger2 v0.1.0 h1:784py6lXkwlVF+K6XSuqmdMgy5l8GI6k60ngBokb9Fg= github.com/ipfs/go-ds-badger2 v0.1.0/go.mod h1:pbR1p817OZbdId9EvLOhKBgUVTM3BMCSTan78lDDVaw= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= +github.com/ipfs/go-ds-leveldb v0.4.2 h1:QmQoAJ9WkPMUfBLnu1sBVy0xWWlJPg0m4kRAiJL9iaw= github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-filestore v1.0.0 h1:QR7ekKH+q2AGiWDc7W2Q0qHuYSRZGUJqUn0GsegEPb0= github.com/ipfs/go-filestore v1.0.0/go.mod h1:/XOCuNtIe2f1YPbiXdYvD0BKLA0JR1MgPiFOdcuu9SM= @@ -1111,6 +1114,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= diff --git a/node/builder.go b/node/builder.go index d054be7a5..3b8e0d62f 100644 --- a/node/builder.go +++ b/node/builder.go @@ -292,7 +292,6 @@ func Online() Option { Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), Override(new(*storage.Miner), modules.StorageMiner), Override(new(dtypes.NetworkName), modules.StorageNetworkName), - Override(new(beacon.RandomBeacon), modules.MinerRandomBeacon), Override(new(dtypes.StagingBlockstore), modules.StagingBlockstore), Override(new(dtypes.StagingDAG), modules.StagingDAG), diff --git a/node/modules/services.go b/node/modules/services.go index 893ebbf73..f4a007c2b 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -123,12 +123,19 @@ func RetrievalResolver(l *discovery.Local) retrievalmarket.PeerResolver { return discovery.Multi(l) } -func RandomBeacon(cs *store.ChainStore, _ dtypes.AfterGenesisSet) (beacon.RandomBeacon, error) { - gen, err := cs.GetGenesis() +type RandomBeaconParams struct { + fx.In + + PubSub *pubsub.PubSub `optional:"true"` + Cs *store.ChainStore +} + +func RandomBeacon(p RandomBeaconParams, _ dtypes.AfterGenesisSet) (beacon.RandomBeacon, error) { + gen, err := p.Cs.GetGenesis() if err != nil { return nil, err } //return beacon.NewMockBeacon(build.BlockDelay * time.Second) - return drand.NewDrandBeacon(gen.Timestamp, build.BlockDelay) + return drand.NewDrandBeacon(gen.Timestamp, build.BlockDelay, p.PubSub) } diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index ee3fc9a1a..f50293870 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -43,8 +43,6 @@ import ( lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/beacon" - "github.com/filecoin-project/lotus/chain/beacon/drand" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/markets/retrievaladapter" @@ -364,12 +362,3 @@ func StorageAuth(ctx helpers.MetricsCtx, ca lapi.Common) (sectorstorage.StorageA headers.Add("Authorization", "Bearer "+string(token)) return sectorstorage.StorageAuth(headers), nil } - -func MinerRandomBeacon(api lapi.FullNode) (beacon.RandomBeacon, error) { - gents, err := api.ChainGetGenesis(context.TODO()) - if err != nil { - return nil, err - } - - return drand.NewDrandBeacon(gents.Blocks()[0].Timestamp, build.BlockDelay) -} From 002d021eb741e49a9e1e4b6a131e007598db6360 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 29 May 2020 16:20:09 +0200 Subject: [PATCH 029/160] Enable HashMsgId Signed-off-by: Jakub Sztandera --- node/modules/lp2p/pubsub.go | 1 + 1 file changed, 1 insertion(+) diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go index 9af2be346..35285f5c1 100644 --- a/node/modules/lp2p/pubsub.go +++ b/node/modules/lp2p/pubsub.go @@ -38,6 +38,7 @@ func GossipSub(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, nn dtyp options := []pubsub.Option{ // Gossipsubv1.1 configuration pubsub.WithFloodPublish(true), + pubsub.WithMessageIdFn(HashMsgId), pubsub.WithPeerScore( &pubsub.PeerScoreParams{ AppSpecificScore: func(p peer.ID) float64 { From 30d82ba9629bf262dee5d6947cfce09829edfc1d Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 29 May 2020 17:21:10 +0200 Subject: [PATCH 030/160] Explicity Watch for randomness Signed-off-by: Jakub Sztandera --- chain/beacon/drand/drand.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/chain/beacon/drand/drand.go b/chain/beacon/drand/drand.go index 0de519f1e..8742350ba 100644 --- a/chain/beacon/drand/drand.go +++ b/chain/beacon/drand/drand.go @@ -76,13 +76,16 @@ func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub) (*DrandBeacon if genesisTs == 0 { panic("what are you doing this cant be zero") } + + dlogger := dlog.NewKitLoggerFrom(kzap.NewZapSugarLogger( + log.SugaredLogger.Desugar(), zapcore.InfoLevel)) opts := []dclient.Option{ dclient.WithChainInfo(drandChain), dclient.WithHTTPEndpoints(drandServers), dclient.WithCacheSize(1024), - dclient.WithLogger(dlog.NewKitLoggerFrom(kzap.NewZapSugarLogger( - log.SugaredLogger.Desugar(), zapcore.InfoLevel))), + dclient.WithLogger(dlogger), } + if ps != nil { opts = append(opts, gclient.WithPubsub(ps)) } else { @@ -94,6 +97,14 @@ func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub) (*DrandBeacon return nil, xerrors.Errorf("creating drand client") } + go func() { + // Explicitly Watch until that is fixed in drand + ch := client.Watch(context.Background()) + for range ch { + } + log.Error("dranch Watch bork") + }() + db := &DrandBeacon{ client: client, localCache: make(map[uint64]types.BeaconEntry), From 7dd25c3ac5e1c3e7fb6f20fe41bddd799661664c Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 28 May 2020 13:22:32 -0400 Subject: [PATCH 031/160] Correctly handle transfers with sender = receiver --- chain/vm/vm.go | 22 ++++++++++++++++++---- go.mod | 2 +- go.sum | 8 ++++++-- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 49294e9cf..763708eca 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -607,16 +607,30 @@ func (vm *VM) transfer(from, to address.Address, amt types.BigInt) aerrors.Actor return nil } + fromId, err := vm.cstate.LookupID(from) + if err != nil { + return aerrors.Fatalf("transfer failed when resolving sender address: %s", err) + } + + toId, err := vm.cstate.LookupID(to) + if err != nil { + return aerrors.Fatalf("transfer failed when resolving receiver address: %s", err) + } + + if fromId == toId { + return nil + } + if amt.LessThan(types.NewInt(0)) { return aerrors.Newf(exitcode.SysErrForbidden, "attempted to transfer negative value: %s", amt) } - f, err := vm.cstate.GetActor(from) + f, err := vm.cstate.GetActor(fromId) if err != nil { return aerrors.Fatalf("transfer failed when retrieving sender actor: %s", err) } - t, err := vm.cstate.GetActor(to) + t, err := vm.cstate.GetActor(toId) if err != nil { return aerrors.Fatalf("transfer failed when retrieving receiver actor: %s", err) } @@ -626,11 +640,11 @@ func (vm *VM) transfer(from, to address.Address, amt types.BigInt) aerrors.Actor } depositFunds(t, amt) - if err := vm.cstate.SetActor(from, f); err != nil { + if err := vm.cstate.SetActor(fromId, f); err != nil { return aerrors.Fatalf("transfer failed when setting receiver actor: %s", err) } - if err := vm.cstate.SetActor(to, t); err != nil { + if err := vm.cstate.SetActor(toId, t); err != nil { return aerrors.Fatalf("transfer failed when setting sender actor: %s", err) } diff --git a/go.mod b/go.mod index 192e14563..cb793325f 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/drand/drand v0.9.2-0.20200529123141-6b138aefaef2 github.com/drand/kyber v1.0.2 github.com/fatih/color v1.8.0 - github.com/filecoin-project/chain-validation v0.0.6-0.20200528194021-3ce40d696ae1 + github.com/filecoin-project/chain-validation v0.0.6-0.20200528212128-d4578ae4f874 github.com/filecoin-project/filecoin-ffi v0.26.1-0.20200508175440-05b30afeb00d github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2 diff --git a/go.sum b/go.sum index 86cce5aa9..d5e36ae38 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,7 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= @@ -165,8 +166,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= -github.com/filecoin-project/chain-validation v0.0.6-0.20200528194021-3ce40d696ae1 h1:cJPPzINJq4oUbdC4gQyMrEzuSiJ95RKxyXsLn6UgfcI= -github.com/filecoin-project/chain-validation v0.0.6-0.20200528194021-3ce40d696ae1/go.mod h1:aV5C4aX+kZwh5OlTHNCHxQ5vTnpTFKUuKiDTk9f/8Es= +github.com/filecoin-project/chain-validation v0.0.6-0.20200528212128-d4578ae4f874 h1:NNzyoQjidcvvoARgaM6zo2jYXC0KwNQkaYuDiBFcAD4= +github.com/filecoin-project/chain-validation v0.0.6-0.20200528212128-d4578ae4f874/go.mod h1:aV5C4aX+kZwh5OlTHNCHxQ5vTnpTFKUuKiDTk9f/8Es= github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef h1:Wi5E+P1QfHP8IF27eUiTx5vYfqQZwfPxzq3oFEq8w8U= @@ -539,6 +540,7 @@ github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZl github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= @@ -1134,7 +1136,9 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= From e5c24ff32b54aa0236f7ed53d5fe6f01265d5bc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 26 May 2020 10:20:32 +0200 Subject: [PATCH 032/160] Update sector-storage with ReadPiece impl --- api/api_worker.go | 9 +++++- api/apistruct/struct.go | 48 +++++++++++++++++----------- cmd/lotus-bench/main.go | 7 +--- cmd/lotus-storage-miner/init.go | 2 +- cmd/lotus-storage-miner/storage.go | 2 +- go.mod | 6 ++-- go.sum | 16 +++++++--- markets/retrievaladapter/provider.go | 11 +++++-- node/config/def.go | 1 + node/repo/fsrepo.go | 4 +++ node/repo/interface.go | 1 + node/repo/memrepo.go | 4 +++ 12 files changed, 75 insertions(+), 36 deletions(-) diff --git a/api/api_worker.go b/api/api_worker.go index dbad20651..6270e65fb 100644 --- a/api/api_worker.go +++ b/api/api_worker.go @@ -2,6 +2,9 @@ package api import ( "context" + "io" + + "github.com/ipfs/go-cid" "github.com/filecoin-project/sector-storage/sealtasks" "github.com/filecoin-project/sector-storage/stores" @@ -21,7 +24,11 @@ type WorkerApi interface { Info(context.Context) (storiface.WorkerInfo, error) storage.Sealer - Fetch(context.Context, abi.SectorID, stores.SectorFileType, bool) error + + UnsealPiece(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error + ReadPiece(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) error + + Fetch(context.Context, abi.SectorID, stores.SectorFileType, bool, stores.AcquireMode) error Closing(context.Context) (<-chan struct{}, error) } diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index c97c1285a..8f35044f8 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -2,6 +2,7 @@ package apistruct import ( "context" + "io" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/network" @@ -201,16 +202,16 @@ type StorageMinerStruct struct { WorkerConnect func(context.Context, string) error `perm:"admin"` // TODO: worker perm WorkerStats func(context.Context) (map[uint64]storiface.WorkerStats, error) `perm:"admin"` - StorageList func(context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"` - StorageLocal func(context.Context) (map[stores.ID]string, error) `perm:"admin"` - StorageStat func(context.Context, stores.ID) (stores.FsStat, error) `perm:"admin"` - StorageAttach func(context.Context, stores.StorageInfo, stores.FsStat) error `perm:"admin"` - StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType) error `perm:"admin"` - StorageDropSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType) error `perm:"admin"` - StorageFindSector func(context.Context, abi.SectorID, stores.SectorFileType, bool) ([]stores.StorageInfo, error) `perm:"admin"` - StorageInfo func(context.Context, stores.ID) (stores.StorageInfo, error) `perm:"admin"` - StorageBestAlloc func(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredProof, sealing bool) ([]stores.StorageInfo, error) `perm:"admin"` - StorageReportHealth func(ctx context.Context, id stores.ID, report stores.HealthReport) error `perm:"admin"` + StorageList func(context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"` + StorageLocal func(context.Context) (map[stores.ID]string, error) `perm:"admin"` + StorageStat func(context.Context, stores.ID) (stores.FsStat, error) `perm:"admin"` + StorageAttach func(context.Context, stores.StorageInfo, stores.FsStat) error `perm:"admin"` + StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType, bool) error `perm:"admin"` + StorageDropSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType) error `perm:"admin"` + StorageFindSector func(context.Context, abi.SectorID, stores.SectorFileType, bool) ([]stores.SectorStorageInfo, error) `perm:"admin"` + StorageInfo func(context.Context, stores.ID) (stores.StorageInfo, error) `perm:"admin"` + StorageBestAlloc func(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredProof, sealing stores.PathType) ([]stores.StorageInfo, error) `perm:"admin"` + StorageReportHealth func(ctx context.Context, id stores.ID, report stores.HealthReport) error `perm:"admin"` DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` @@ -235,7 +236,10 @@ type WorkerStruct struct { SealCommit2 func(context.Context, abi.SectorID, storage.Commit1Out) (storage.Proof, error) `perm:"admin"` FinalizeSector func(context.Context, abi.SectorID) error `perm:"admin"` - Fetch func(context.Context, abi.SectorID, stores.SectorFileType, bool) error `perm:"admin"` + UnsealPiece func(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error `perm:"admin"` + ReadPiece func(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) error `perm:"admin"` + + Fetch func(context.Context, abi.SectorID, stores.SectorFileType, bool, stores.AcquireMode) error `perm:"admin"` Closing func(context.Context) (<-chan struct{}, error) `perm:"admin"` } @@ -757,15 +761,15 @@ func (c *StorageMinerStruct) StorageAttach(ctx context.Context, si stores.Storag return c.Internal.StorageAttach(ctx, si, st) } -func (c *StorageMinerStruct) StorageDeclareSector(ctx context.Context, storageId stores.ID, s abi.SectorID, ft stores.SectorFileType) error { - return c.Internal.StorageDeclareSector(ctx, storageId, s, ft) +func (c *StorageMinerStruct) StorageDeclareSector(ctx context.Context, storageId stores.ID, s abi.SectorID, ft stores.SectorFileType, primary bool) error { + return c.Internal.StorageDeclareSector(ctx, storageId, s, ft, primary) } func (c *StorageMinerStruct) StorageDropSector(ctx context.Context, storageId stores.ID, s abi.SectorID, ft stores.SectorFileType) error { return c.Internal.StorageDropSector(ctx, storageId, s, ft) } -func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types stores.SectorFileType, allowFetch bool) ([]stores.StorageInfo, error) { +func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types stores.SectorFileType, allowFetch bool) ([]stores.SectorStorageInfo, error) { return c.Internal.StorageFindSector(ctx, si, types, allowFetch) } @@ -785,8 +789,8 @@ func (c *StorageMinerStruct) StorageInfo(ctx context.Context, id stores.ID) (sto return c.Internal.StorageInfo(ctx, id) } -func (c *StorageMinerStruct) StorageBestAlloc(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredProof, sealing bool) ([]stores.StorageInfo, error) { - return c.Internal.StorageBestAlloc(ctx, allocate, spt, sealing) +func (c *StorageMinerStruct) StorageBestAlloc(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredProof, pt stores.PathType) ([]stores.StorageInfo, error) { + return c.Internal.StorageBestAlloc(ctx, allocate, spt, pt) } func (c *StorageMinerStruct) StorageReportHealth(ctx context.Context, id stores.ID, report stores.HealthReport) error { @@ -859,8 +863,16 @@ func (w *WorkerStruct) FinalizeSector(ctx context.Context, sector abi.SectorID) return w.Internal.FinalizeSector(ctx, sector) } -func (w *WorkerStruct) Fetch(ctx context.Context, id abi.SectorID, fileType stores.SectorFileType, b bool) error { - return w.Internal.Fetch(ctx, id, fileType, b) +func (w *WorkerStruct) UnsealPiece(ctx context.Context, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, c cid.Cid) error { + return w.Internal.UnsealPiece(ctx, id, index, size, randomness, c) +} + +func (w *WorkerStruct) ReadPiece(ctx context.Context, writer io.Writer, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) error { + return w.Internal.ReadPiece(ctx, writer, id, index, size) +} + +func (w *WorkerStruct) Fetch(ctx context.Context, id abi.SectorID, fileType stores.SectorFileType, b bool, am stores.AcquireMode) error { + return w.Internal.Fetch(ctx, id, fileType, b, am) } func (w *WorkerStruct) Closing(ctx context.Context) (<-chan struct{}, error) { diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 23471500d..3173b442d 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -562,15 +562,10 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, mid } } - // TODO: RM unsealed sector first - rc, err := sb.ReadPieceFromSealedSector(context.TODO(), abi.SectorID{Miner: mid, Number: 1}, 0, abi.UnpaddedPieceSize(sectorSize), ticket, cids.Unsealed) + err := sb.UnsealPiece(context.TODO(), abi.SectorID{Miner: mid, Number: 1}, 0, abi.UnpaddedPieceSize(sectorSize), ticket, cids.Unsealed) if err != nil { return nil, nil, err } - - if err := rc.Close(); err != nil { - return nil, nil, err - } } unseal := time.Now() diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index c28e9548c..b43af23c7 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -436,7 +436,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, smgr, err := sectorstorage.New(ctx, lr, stores.NewIndex(), &ffiwrapper.Config{ SealProofType: spt, - }, sectorstorage.SealerConfig{true, true, true}, nil, sa) + }, sectorstorage.SealerConfig{true, true, true, true}, nil, sa) if err != nil { return err } diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index 18dd80dff..40205531a 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -249,7 +249,7 @@ var storageListCmd = &cli.Command{ type storedSector struct { id stores.ID - store stores.StorageInfo + store stores.SectorStorageInfo unsealed, sealed, cache bool } diff --git a/go.mod b/go.mod index 0a5d64d52..607525d8e 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/filecoin-project/filecoin-ffi v0.26.1-0.20200508175440-05b30afeb00d github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2 - github.com/filecoin-project/go-bitfield v0.0.1 + github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.3.0 @@ -29,7 +29,7 @@ require ( github.com/filecoin-project/go-paramfetch v0.0.2-0.20200505180321-973f8949ea8e github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/sector-storage v0.0.0-20200522011946-a59ca7536a95 + github.com/filecoin-project/sector-storage v0.0.0-20200529175241-9df0cdf19326 github.com/filecoin-project/specs-actors v0.5.4 github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 github.com/filecoin-project/storage-fsm v0.0.0-20200528050623-cdada6e88960 @@ -67,7 +67,7 @@ require ( github.com/ipfs/go-path v0.0.7 github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/interface-go-ipfs-core v0.2.3 - github.com/ipld/go-car v0.1.1-0.20200430185908-8ff2e52a4c88 + github.com/ipld/go-car v0.1.1-0.20200526133713-1c7508d55aae github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e github.com/kelseyhightower/envconfig v1.4.0 github.com/lib/pq v1.2.0 diff --git a/go.sum b/go.sum index 87f4f9148..63ab4a537 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,7 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= @@ -118,6 +119,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f h1:BOaYiTvg8p9vBUXpklC22XSK/mifLF7lG9jtmYYi3Tc= github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= +github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e h1:lj77EKYUpYXTd8CD/+QMIf8b6OIOTsfEBSXiAzuEHTU= +github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1:3ZQK6DMPSz/QZ73jlWxBtUhNA8xZx7LzUFSq/OfP8vk= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= @@ -177,6 +180,8 @@ github.com/filecoin-project/go-bitfield v0.0.0-20200309034705-8c7ac40bd550/go.mo github.com/filecoin-project/go-bitfield v0.0.0-20200416002808-b3ee67ec9060/go.mod h1:iodsLxOFZnqKtjj2zkgqzoGNrv6vUqj69AT/J8DKXEw= github.com/filecoin-project/go-bitfield v0.0.1 h1:Xg/JnrqqE77aJVKdbEyR04n9FZQWhwrN+buDgQCVpZU= github.com/filecoin-project/go-bitfield v0.0.1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= +github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e h1:gkG/7G+iKy4He+IiQNeQn+nndFznb/vCoOR8iRQsm60= +github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= @@ -202,8 +207,8 @@ github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZO github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= github.com/filecoin-project/sector-storage v0.0.0-20200508203401-a74812ba12f3/go.mod h1:B+xzopr/oWZJz2hBL5Ekb7Obcum5ntmfbaAUlaaho28= -github.com/filecoin-project/sector-storage v0.0.0-20200522011946-a59ca7536a95 h1:wNpbsUY11P7+3yBfHOXsH+tAyferf1BVSe5S+HRm+7s= -github.com/filecoin-project/sector-storage v0.0.0-20200522011946-a59ca7536a95/go.mod h1:1UpuWCUC7XNfgyufyQQq+/40JSzhfGdgcwauK55C8gk= +github.com/filecoin-project/sector-storage v0.0.0-20200529175241-9df0cdf19326 h1:WIIvOzK9KaWMFXhvivUxeg4cdMzQ/5/MW+bNpYj43wU= +github.com/filecoin-project/sector-storage v0.0.0-20200529175241-9df0cdf19326/go.mod h1:hUQo3z/5B8t8Yl+XgIxBqgNfYtIcCYzKjWWY27dHBCk= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= @@ -510,8 +515,8 @@ github.com/ipfs/iptb v1.4.0/go.mod h1:1rzHpCYtNp87/+hTxG5TfCVn/yMY3dKnLn8tBiMfdm github.com/ipfs/iptb-plugins v0.2.1 h1:au4HWn9/pRPbkxA08pDx2oRAs4cnbgQWgV0teYXuuGA= github.com/ipfs/iptb-plugins v0.2.1/go.mod h1:QXMbtIWZ+jRsW8a4h13qAKU7jcM7qaittO8wOsTP0Rs= github.com/ipld/go-car v0.1.1-0.20200429200904-c222d793c339/go.mod h1:eajxljm6I8o3LitnFeVEmucwZmz7+yLSiKce9yYMefg= -github.com/ipld/go-car v0.1.1-0.20200430185908-8ff2e52a4c88 h1:kjpIMwbJk/OEKM8hexj+HAVZsOgVr4byKML3s++s1b0= -github.com/ipld/go-car v0.1.1-0.20200430185908-8ff2e52a4c88/go.mod h1:2mvxpu4dKRnuH3mj5u6KW/tmRSCcXvy/KYiJ4nC6h4c= +github.com/ipld/go-car v0.1.1-0.20200526133713-1c7508d55aae h1:OV9dxl8iPMCOD8Vi/hvFwRh3JWPXqmkYSVxWr9JnEzM= +github.com/ipld/go-car v0.1.1-0.20200526133713-1c7508d55aae/go.mod h1:2mvxpu4dKRnuH3mj5u6KW/tmRSCcXvy/KYiJ4nC6h4c= github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e h1:ZISbJlM0urTANR9KRfRaqlBmyOj5uUtxs2r4Up9IXsA= github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e/go.mod h1:uVIwe/u0H4VdKv3kaN1ck7uCb6yD9cFLS9/ELyXbsw8= github.com/ipld/go-ipld-prime-proto v0.0.0-20200428191222-c1ffdadc01e1 h1:K1Ysr7kgIlo7YQkPqdkA6H7BVdIugvuAz7OQUTJxLdE= @@ -536,6 +541,7 @@ github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZl github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= @@ -1130,7 +1136,9 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= diff --git a/markets/retrievaladapter/provider.go b/markets/retrievaladapter/provider.go index 1917d8119..6c425ccd8 100644 --- a/markets/retrievaladapter/provider.go +++ b/markets/retrievaladapter/provider.go @@ -2,13 +2,13 @@ package retrievaladapter import ( "context" + "github.com/filecoin-project/sector-storage/storiface" "io" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/shared" sectorstorage "github.com/filecoin-project/sector-storage" - "github.com/filecoin-project/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/builtin/paych" @@ -54,7 +54,14 @@ func (rpn *retrievalProviderNode) UnsealSector(ctx context.Context, sectorID uin Miner: abi.ActorID(mid), Number: abi.SectorNumber(sectorID), } - return rpn.sealer.ReadPieceFromSealedSector(ctx, sid, ffiwrapper.UnpaddedByteIndex(offset), abi.UnpaddedPieceSize(length), si.TicketValue, *si.CommD) + + r, w := io.Pipe() + go func() { + err := rpn.sealer.ReadPiece(ctx, w, sid, storiface.UnpaddedByteIndex(offset), abi.UnpaddedPieceSize(length), si.TicketValue, *si.CommD) + _ = w.CloseWithError(err) + }() + + return r, nil } func (rpn *retrievalProviderNode) SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *paych.SignedVoucher, proof []byte, expectedAmount abi.TokenAmount, tok shared.TipSetToken) (abi.TokenAmount, error) { diff --git a/node/config/def.go b/node/config/def.go index d5c64d40d..9ca97bb47 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -107,6 +107,7 @@ func DefaultStorageMiner() *StorageMiner { AllowPreCommit1: true, AllowPreCommit2: true, AllowCommit: true, + AllowUnseal: true, }, } cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 396fbf67e..a1cc9a6ec 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -319,6 +319,10 @@ func (fsr *fsLockedRepo) SetStorage(c func(*stores.StorageConfig)) error { return config.WriteStorageFile(fsr.join(fsStorageConfig), sc) } +func (fsr *fsLockedRepo) Stat(path string) (stores.FsStat, error) { + return stores.Stat(path) +} + func (fsr *fsLockedRepo) SetAPIEndpoint(ma multiaddr.Multiaddr) error { if err := fsr.stillValid(); err != nil { return err diff --git a/node/repo/interface.go b/node/repo/interface.go index 5ce414479..c87274449 100644 --- a/node/repo/interface.go +++ b/node/repo/interface.go @@ -40,6 +40,7 @@ type LockedRepo interface { GetStorage() (stores.StorageConfig, error) SetStorage(func(*stores.StorageConfig)) error + Stat(path string) (stores.FsStat, error) // SetAPIEndpoint sets the endpoint of the current API // so it can be read by API clients diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index 59dbecda7..17eeb0253 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -61,6 +61,10 @@ func (lmem *lockedMemRepo) SetStorage(c func(*stores.StorageConfig)) error { return nil } +func (lmem *lockedMemRepo) Stat(path string) (stores.FsStat, error) { + return stores.Stat(path) +} + func (lmem *lockedMemRepo) Path() string { lmem.Lock() defer lmem.Unlock() From cd2bf04a6bec4e46cd781f4de12896c8698bfe8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 29 May 2020 21:36:04 +0200 Subject: [PATCH 033/160] wdpost: Better fault handling --- storage/wdpost_run.go | 264 ++++++++++++++++++++++++++++-------------- 1 file changed, 174 insertions(+), 90 deletions(-) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 63109e5f5..08f5def30 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -62,7 +62,56 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *miner.Deadli }() } -func (s *WindowPoStScheduler) checkRecoveries(ctx context.Context, deadline uint64, ts *types.TipSet) error { +func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check *abi.BitField) (*abi.BitField, error) { + spt, err := s.proofType.RegisteredSealProof() + if err != nil { + return nil, xerrors.Errorf("getting seal proof type: %w", err) + } + + mid, err := address.IDFromAddress(s.actor) + if err != nil { + return nil, err + } + + sectors := make(map[abi.SectorID]struct{}) + var tocheck []abi.SectorID + err = check.ForEach(func(snum uint64) error { + s := abi.SectorID{ + Miner: abi.ActorID(mid), + Number: abi.SectorNumber(snum), + } + + tocheck = append(tocheck, s) + sectors[s] = struct{}{} + return nil + }) + if err != nil { + return nil, xerrors.Errorf("iterating over bitfield: %w", err) + } + + bad, err := s.faultTracker.CheckProvable(ctx, spt, tocheck) + if err != nil { + return nil, xerrors.Errorf("checking provable sectors: %w", err) + } + for _, id := range bad { + delete(sectors, id) + } + + log.Warnw("Checked sectors", "checked", len(tocheck), "good", len(sectors)) + + if len(sectors) == 0 { // nothing to recover + return nil, nil + } + + sbf := bitfield.New() + for s := range sectors { + (&sbf).Set(uint64(s.Number)) + } + + return &sbf, nil +} + +func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, deadline uint64, ts *types.TipSet) error { faults, err := s.api.StateMinerFaults(ctx, s.actor, ts.Key()) if err != nil { return xerrors.Errorf("getting on-chain faults: %w", err) @@ -96,53 +145,13 @@ func (s *WindowPoStScheduler) checkRecoveries(ctx context.Context, deadline uint return nil } - spt, err := s.proofType.RegisteredSealProof() + sbf, err := s.checkSectors(ctx, unrecovered) if err != nil { - return xerrors.Errorf("getting seal proof type: %w", err) - } - - mid, err := address.IDFromAddress(s.actor) - if err != nil { - return err - } - - sectors := make(map[abi.SectorID]struct{}) - var tocheck []abi.SectorID - err = unrecovered.ForEach(func(snum uint64) error { - s := abi.SectorID{ - Miner: abi.ActorID(mid), - Number: abi.SectorNumber(snum), - } - - tocheck = append(tocheck, s) - sectors[s] = struct{}{} - return nil - }) - if err != nil { - return xerrors.Errorf("iterating over unrecovered bitfield: %w", err) - } - - bad, err := s.faultTracker.CheckProvable(ctx, spt, tocheck) - if err != nil { - return xerrors.Errorf("checking provable sectors: %w", err) - } - for _, id := range bad { - delete(sectors, id) - } - - log.Warnw("Recoverable sectors", "faulty", len(tocheck), "recoverable", len(sectors)) - - if len(sectors) == 0 { // nothing to recover - return nil - } - - sbf := bitfield.New() - for s := range sectors { - (&sbf).Set(uint64(s.Number)) + return xerrors.Errorf("checking unrecovered sectors: %w", err) } params := &miner.DeclareFaultsRecoveredParams{ - Recoveries: []miner.RecoveryDeclaration{{Deadline: deadline, Sectors: &sbf}}, + Recoveries: []miner.RecoveryDeclaration{{Deadline: deadline, Sectors: sbf}}, } enc, aerr := actors.SerializeParams(params) @@ -179,34 +188,84 @@ func (s *WindowPoStScheduler) checkRecoveries(ctx context.Context, deadline uint return nil } -func (s *WindowPoStScheduler) checkFaults(ctx context.Context, ssi []abi.SectorNumber) ([]abi.SectorNumber, error) { - //faults := s.prover.Scrub(ssi) - log.Warnf("Stub checkFaults") +func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, deadline uint64, deadlineSectors *abi.BitField, ts *types.TipSet) error { + toCheck, err := s.getSectorsToProve(ctx, deadlineSectors, true, ts) + if err != nil { + return xerrors.Errorf("getting next sectors to prove: %w", err) + } - /*declaredFaults := map[abi.SectorNumber]struct{}{} + good, err := s.checkSectors(ctx, deadlineSectors) + if err != nil { + return xerrors.Errorf("checking sectors: %w", err) + } - { - chainFaults, err := s.api.StateMinerFaults(ctx, s.actor, types.EmptyTSK) - if err != nil { - return nil, xerrors.Errorf("checking on-chain faults: %w", err) - } + faulty, err := bitfield.SubtractBitField(toCheck, good) + if err != nil { + return xerrors.Errorf("calculating faulty sector set: %w", err) + } - for _, fault := range chainFaults { - declaredFaults[fault] = struct{}{} - } - }*/ + c, err := faulty.Count() + if err != nil { + return xerrors.Errorf("counting faulty sectors: %w", err) + } - return nil, nil + if c == 0 { + return nil + } + + log.Errorw("DETECTED FAULTY SECTORS, declaring faults", "count", c) + + params := &miner.DeclareFaultsParams{ + Faults: []miner.FaultDeclaration{ + { + Deadline: deadline, + Sectors: faulty, + }, + }, + } + + enc, aerr := actors.SerializeParams(params) + if aerr != nil { + return xerrors.Errorf("could not serialize declare faults parameters: %w", aerr) + } + + msg := &types.Message{ + To: s.actor, + From: s.worker, + Method: builtin.MethodsMiner.DeclareFaults, + Params: enc, + Value: types.NewInt(0), // TODO: Is there a fee? + GasLimit: 10000000, // i dont know help + GasPrice: types.NewInt(2), + } + + sm, err := s.api.MpoolPushMessage(ctx, msg) + if err != nil { + return xerrors.Errorf("pushing message to mpool: %w", err) + } + + log.Warnw("declare faults Message CID", "cid", sm.Cid()) + + rec, err := s.api.StateWaitMsg(context.TODO(), sm.Cid()) + if err != nil { + return xerrors.Errorf("declare faults wait error: %w", err) + } + + if rec.Receipt.ExitCode != 0 { + return xerrors.Errorf("declare faults wait non-0 exit code: %d", rec.Receipt.ExitCode) + } + + return nil } // the input sectors must match with the miner actor -func (s *WindowPoStScheduler) getNeedProveSectors(ctx context.Context, deadlineSectors *abi.BitField, ts *types.TipSet) (*abi.BitField, error) { - faults, err := s.api.StateMinerFaults(ctx, s.actor, ts.Key()) +func (s *WindowPoStScheduler) getSectorsToProve(ctx context.Context, deadlineSectors *abi.BitField, ignoreRecoveries bool, ts *types.TipSet) (*abi.BitField, error) { + stateFaults, err := s.api.StateMinerFaults(ctx, s.actor, ts.Key()) if err != nil { return nil, xerrors.Errorf("getting on-chain faults: %w", err) } - declaredFaults, err := bitfield.IntersectBitField(deadlineSectors, faults) + faults, err := bitfield.IntersectBitField(deadlineSectors, stateFaults) if err != nil { return nil, xerrors.Errorf("failed to intersect proof sectors with faults: %w", err) } @@ -216,17 +275,19 @@ func (s *WindowPoStScheduler) getNeedProveSectors(ctx context.Context, deadlineS return nil, xerrors.Errorf("getting on-chain recoveries: %w", err) } - expectedRecoveries, err := bitfield.IntersectBitField(declaredFaults, recoveries) - if err != nil { - return nil, xerrors.Errorf("failed to intersect recoveries with faults: %w", err) + if !ignoreRecoveries { + expectedRecoveries, err := bitfield.IntersectBitField(faults, recoveries) + if err != nil { + return nil, xerrors.Errorf("failed to intersect recoveries with faults: %w", err) + } + + faults, err = bitfield.SubtractBitField(faults, expectedRecoveries) + if err != nil { + return nil, xerrors.Errorf("failed to subtract recoveries from faults: %w", err) + } } - expectedFaults, err := bitfield.SubtractBitField(declaredFaults, expectedRecoveries) - if err != nil { - return nil, xerrors.Errorf("failed to subtract recoveries from faults: %w", err) - } - - nonFaults, err := bitfield.SubtractBitField(deadlineSectors, expectedFaults) + nonFaults, err := bitfield.SubtractBitField(deadlineSectors, faults) if err != nil { return nil, xerrors.Errorf("failed to diff bitfields: %w", err) } @@ -246,11 +307,26 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo ctx, span := trace.StartSpan(ctx, "storage.runPost") defer span.End() - // check recoveries for the *next* deadline. It's already too late to - // declare them for this deadline - if err := s.checkRecoveries(ctx, (di.Index+1)%miner.WPoStPeriodDeadlines, ts); err != nil { - // TODO: This is potentially quite bad, but not even trying to post when this fails is objectively worse - log.Errorf("checking sector recoveries: %v", err) + deadlines, err := s.api.StateMinerDeadlines(ctx, s.actor, ts.Key()) + if err != nil { + return nil, xerrors.Errorf("getting miner deadlines: %w", err) + } + + { + // check faults / recoveries for the *next* deadline. It's already too + // late to declare them for this deadline + declDeadline := (di.Index + 1) % miner.WPoStPeriodDeadlines + + if err := s.checkNextRecoveries(ctx, declDeadline, ts); err != nil { + // TODO: This is potentially quite bad, but not even trying to post when this fails is objectively worse + log.Errorf("checking sector recoveries: %v", err) + } + + if err := s.checkNextFaults(ctx, declDeadline, deadlines.Due[declDeadline], ts); err != nil { + // TODO: This is also potentially really bad, but we try to post anyways + log.Errorf("checking sector faults: %v", err) + } + } buf := new(bytes.Buffer) @@ -262,11 +338,6 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo return nil, xerrors.Errorf("failed to get chain randomness for windowPost (ts=%d; deadline=%d): %w", ts.Height(), di, err) } - deadlines, err := s.api.StateMinerDeadlines(ctx, s.actor, ts.Key()) - if err != nil { - return nil, err - } - firstPartition, _, err := miner.PartitionsForDeadline(deadlines, s.partitionSectors, di.Index) if err != nil { return nil, xerrors.Errorf("getting partitions for deadline: %w", err) @@ -297,11 +368,29 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo partitions[i] = firstPartition + uint64(i) } - nps, err := s.getNeedProveSectors(ctx, deadlines.Due[di.Index], ts) + nps, err := s.getSectorsToProve(ctx, deadlines.Due[di.Index], false, ts) if err != nil { return nil, xerrors.Errorf("get need prove sectors: %w", err) } + var skipped *abi.BitField + { + good, err := s.checkSectors(ctx, nps) + if err != nil { + return nil, xerrors.Errorf("checking sectors to skip: %w", err) + } + + skipped, err = bitfield.SubtractBitField(nps, good) + if err != nil { + return nil, xerrors.Errorf("nps - good: %w", err) + } + } + + skipCount, err := skipped.Count() + if err != nil { + return nil, xerrors.Errorf("getting skipped sector count: %w", err) + } + ssi, err := s.sortedSectorInfo(ctx, nps, ts) if err != nil { return nil, xerrors.Errorf("getting sorted sector info: %w", err) @@ -315,23 +404,18 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo log.Infow("running windowPost", "chain-random", rand, "deadline", di, - "height", ts.Height()) + "height", ts.Height(), + "skipped", skipCount) var snums []abi.SectorNumber for _, si := range ssi { snums = append(snums, si.SectorNumber) } - faults, err := s.checkFaults(ctx, snums) - if err != nil { - log.Errorf("Failed to declare faults: %+v", err) - } - tsStart := time.Now() log.Infow("generating windowPost", - "sectors", len(ssi), - "faults", len(faults)) + "sectors", len(ssi)) mid, err := address.IDFromAddress(s.actor) if err != nil { @@ -355,7 +439,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo Deadline: di.Index, Partitions: partitions, Proofs: postOut, - Skipped: *abi.NewBitField(), // TODO: Faults here? + Skipped: *skipped, }, nil } From ee29a92021dba9ce38b0be4d225c94205fa6cd15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 1 Jun 2020 14:49:48 +0200 Subject: [PATCH 034/160] Add a test for Window Post --- api/api_storage.go | 3 +- api/test/test.go | 1 + api/test/window_post.go | 168 ++++++++++++++++++++++++++++++++++++++++ node/node_test.go | 48 ++++++++++-- storage/wdpost_run.go | 9 +++ 5 files changed, 222 insertions(+), 7 deletions(-) create mode 100644 api/test/window_post.go diff --git a/api/api_storage.go b/api/api_storage.go index ab75e8058..d6004dcf3 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -11,6 +11,7 @@ import ( "github.com/filecoin-project/sector-storage/stores" "github.com/filecoin-project/sector-storage/storiface" "github.com/filecoin-project/specs-actors/actors/abi" + sealing "github.com/filecoin-project/storage-fsm" "github.com/filecoin-project/lotus/chain/types" ) @@ -119,4 +120,4 @@ func (st *SealSeed) Equals(ost *SealSeed) bool { return bytes.Equal(st.Value, ost.Value) && st.Epoch == ost.Epoch } -type SectorState string +type SectorState = sealing.SectorState diff --git a/api/test/test.go b/api/test/test.go index 43dc7a25e..06f0584fd 100644 --- a/api/test/test.go +++ b/api/test/test.go @@ -20,6 +20,7 @@ type TestStorageNode struct { } var PresealGenesis = -1 +const GenesisPreseals = 2 type StorageMiner struct { Full int diff --git a/api/test/window_post.go b/api/test/window_post.go new file mode 100644 index 000000000..b67b3e498 --- /dev/null +++ b/api/test/window_post.go @@ -0,0 +1,168 @@ +package test + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/specs-actors/actors/abi" + miner2 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + sealing "github.com/filecoin-project/storage-fsm" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/impl" +) + +func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) { + os.Setenv("BELLMAN_NO_GPU", "1") + + ctx := context.Background() + n, sn := b(t, 1, oneMiner) + client := n[0].FullNode.(*impl.FullNodeAPI) + miner := sn[0] + + addrinfo, err := client.NetAddrsListen(ctx) + if err != nil { + t.Fatal(err) + } + + if err := miner.NetConnect(ctx, addrinfo); err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + + mine := true + done := make(chan struct{}) + go func() { + defer close(done) + for mine { + time.Sleep(blocktime) + if err := sn[0].MineOne(ctx, func(bool) {}); err != nil { + t.Error(err) + } + } + }() + + pledgeSectors(t, ctx, miner, nSectors) + + mine = false + <-done +} + +func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n int) { + for i := 0; i < n; i++ { + err := miner.PledgeSector(ctx) + require.NoError(t, err) + } + + for { + s, err := miner.SectorsList(ctx) // Note - the test builder doesn't import genesis sectors into FSM + require.NoError(t, err) + fmt.Printf("Sectors: %d\n", len(s)) + if len(s) >= n { + break + } + + time.Sleep(100 * time.Millisecond) + } + + fmt.Printf("All sectors is fsm\n") + + s, err := miner.SectorsList(ctx) + require.NoError(t, err) + + toCheck := map[abi.SectorNumber]struct{}{} + for _, number := range s { + toCheck[number] = struct{}{} + } + + for len(toCheck) > 0 { + for n := range toCheck { + st, err := miner.SectorsStatus(ctx, n) + require.NoError(t, err) + if st.State == sealing.Proving { + delete(toCheck, n) + } + if strings.Contains(string(st.State), "Fail") { + t.Fatal("sector in a failed state", st.State) + } + } + + time.Sleep(100 * time.Millisecond) + fmt.Printf("WaitSeal: %d\n", len(s)) + } +} + +func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) { + os.Setenv("BELLMAN_NO_GPU", "1") + + ctx := context.Background() + n, sn := b(t, 1, oneMiner) + client := n[0].FullNode.(*impl.FullNodeAPI) + miner := sn[0] + + addrinfo, err := client.NetAddrsListen(ctx) + if err != nil { + t.Fatal(err) + } + + if err := miner.NetConnect(ctx, addrinfo); err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + + mine := true + done := make(chan struct{}) + go func() { + defer close(done) + for mine { + time.Sleep(blocktime) + if err := sn[0].MineOne(ctx, func(bool) {}); err != nil { + t.Error(err) + } + } + }() + + pledgeSectors(t, ctx, miner, nSectors) + + maddr, err := miner.ActorAddress(ctx) + require.NoError(t, err) + + di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + + fmt.Printf("Running one proving periods\n") + + for { + head, err := client.ChainHead(ctx) + require.NoError(t, err) + + if head.Height() > di.PeriodStart + (miner2.WPoStProvingPeriod) + 2 { + break + } + + if head.Height() % 100 == 0 { + fmt.Printf("@%d\n", head.Height()) + } + time.Sleep(blocktime) + } + + p, err := client.StateMinerPower(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + + ssz, err := miner.ActorSectorSize(ctx, maddr) + require.NoError(t, err) + + require.Equal(t, p.MinerPower, p.TotalPower) + require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz) * uint64(nSectors + GenesisPreseals))) + + // TODO: Inject faults here + + mine = false + <-done +} diff --git a/node/node_test.go b/node/node_test.go index a31b4162b..9b543ee7a 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -83,7 +83,7 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a require.NoError(t, err) nic := storedcounter.New(ds, datastore.NewKey(modules.StorageCounterDSPrefix)) - for i := 0; i < nGenesisPreseals; i++ { + for i := 0; i < test.GenesisPreseals; i++ { _, err := nic.Next() require.NoError(t, err) } @@ -188,7 +188,7 @@ func builder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestN if err != nil { t.Fatal(err) } - genm, k, err := seed.PreSeal(maddr, abi.RegisteredProof_StackedDRG2KiBPoSt, 0, nGenesisPreseals, tdir, []byte("make genesis mem random"), nil) + genm, k, err := seed.PreSeal(maddr, abi.RegisteredProof_StackedDRG2KiBPoSt, 0, test.GenesisPreseals, tdir, []byte("make genesis mem random"), nil) if err != nil { t.Fatal(err) } @@ -282,8 +282,6 @@ func builder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestN return fulls, storers } -const nGenesisPreseals = 2 - func mockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { ctx := context.Background() mn := mocknet.New(ctx) @@ -314,7 +312,7 @@ func mockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test preseals := storage[i].Preseal if preseals == test.PresealGenesis { - preseals = nGenesisPreseals + preseals = test.GenesisPreseals } genm, k, err := mockstorage.PreSeal(2048, maddr, preseals) @@ -350,7 +348,7 @@ func mockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test templ := &genesis.Template{ Accounts: genaccs, Miners: genms, - Timestamp: uint64(time.Now().Unix() - 10000), + Timestamp: uint64(time.Now().Unix() - (build.BlockDelay * 20000)), } // END PRESEAL SECTION @@ -493,3 +491,41 @@ func TestDealMining(t *testing.T) { test.TestDealMining(t, mockSbBuilder, 50*time.Millisecond, false) } + +func TestPledgeSectors(t *testing.T) { + logging.SetLogLevel("miner", "ERROR") + logging.SetLogLevel("chainstore", "ERROR") + logging.SetLogLevel("chain", "ERROR") + logging.SetLogLevel("sub", "ERROR") + logging.SetLogLevel("storageminer", "ERROR") + + t.Run("1", func(t *testing.T) { + test.TestPledgeSector(t, mockSbBuilder, 50*time.Millisecond, 1) + }) + + t.Run("100", func(t *testing.T) { + test.TestPledgeSector(t, mockSbBuilder, 50*time.Millisecond, 100) + }) + + t.Run("1000", func(t *testing.T) { + if testing.Short() { // takes ~16s + t.Skip("skipping test in short mode") + } + + test.TestPledgeSector(t, mockSbBuilder, 50*time.Millisecond, 1000) + }) +} + +func TestWindowedPost(t *testing.T) { + if testing.Short() { + t.Skip("this takes a while") + } + + logging.SetLogLevel("miner", "ERROR") + logging.SetLogLevel("chainstore", "ERROR") + logging.SetLogLevel("chain", "ERROR") + logging.SetLogLevel("sub", "ERROR") + logging.SetLogLevel("storageminer", "ERROR") + + test.TestWindowPost(t, mockSbBuilder, 3*time.Millisecond, 10) +} diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 08f5def30..fbdae5199 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -189,6 +189,15 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, deadline } func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, deadline uint64, deadlineSectors *abi.BitField, ts *types.TipSet) error { + dc, err := deadlineSectors.Count() + if err != nil { + return xerrors.Errorf("counting deadline sectors: %w", err) + } + if dc == 0 { + // nothing can become faulty + return nil + } + toCheck, err := s.getSectorsToProve(ctx, deadlineSectors, true, ts) if err != nil { return xerrors.Errorf("getting next sectors to prove: %w", err) From b753cd450330c3a601107eb9c65b65bf579c887e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 1 Jun 2020 14:59:51 +0200 Subject: [PATCH 035/160] Run window post test on circle --- .circleci/config.yml | 13 +++++++++++-- api/api_storage.go | 6 ++---- api/test/window_post.go | 3 ++- node/node_test.go | 7 ++++--- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 312aa42ca..700f1fbc9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -134,6 +134,9 @@ jobs: type: string default: "./..." description: Import paths of packages to be tested. + winpost-test: + type: string + default: "0" test-suite-name: type: string default: unit @@ -171,6 +174,7 @@ jobs: environment: GOTESTSUM_JUNITFILE: /tmp/test-reports/<< parameters.test-suite-name >>/junit.xml GOTESTSUM_FORMAT: << parameters.gotestsum-format >> + LOTUS_TEST_WINDOW_POST: << parameters.winpost-test >> command: | mkdir -p /tmp/test-reports/<< parameters.test-suite-name >> gotestsum -- \ @@ -199,6 +203,8 @@ jobs: test-short: <<: *test + test-window-post: + <<: *test build-macos: description: build darwin lotus binary @@ -332,10 +338,13 @@ workflows: jobs: - lint-changes: args: "--new-from-rev origin/next" - - test: - codecov-upload: true - mod-tidy-check - gofmt + - test: + codecov-upload: true + - test-window-post: + go-test-flags: "-run=TestWindowedPost" + winpost-test: "1" - test-short: go-test-flags: "--timeout 10m --short" filters: diff --git a/api/api_storage.go b/api/api_storage.go index d6004dcf3..7d3c78b48 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -8,12 +8,10 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/sector-storage/stores" "github.com/filecoin-project/sector-storage/storiface" "github.com/filecoin-project/specs-actors/actors/abi" - sealing "github.com/filecoin-project/storage-fsm" - - "github.com/filecoin-project/lotus/chain/types" ) // StorageMiner is a low-level interface to the Filecoin network storage miner node @@ -120,4 +118,4 @@ func (st *SealSeed) Equals(ost *SealSeed) bool { return bytes.Equal(st.Value, ost.Value) && st.Epoch == ost.Epoch } -type SectorState = sealing.SectorState +type SectorState string diff --git a/api/test/window_post.go b/api/test/window_post.go index b67b3e498..b821df2a7 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -3,6 +3,7 @@ package test import ( "context" "fmt" + "github.com/filecoin-project/lotus/api" "os" "strings" "testing" @@ -85,7 +86,7 @@ func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n i for n := range toCheck { st, err := miner.SectorsStatus(ctx, n) require.NoError(t, err) - if st.State == sealing.Proving { + if st.State == api.SectorState(sealing.Proving) { delete(toCheck, n) } if strings.Contains(string(st.State), "Fail") { diff --git a/node/node_test.go b/node/node_test.go index 9b543ee7a..6258fa943 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -6,6 +6,7 @@ import ( "crypto/rand" "io/ioutil" "net/http/httptest" + "os" "testing" "time" @@ -517,8 +518,8 @@ func TestPledgeSectors(t *testing.T) { } func TestWindowedPost(t *testing.T) { - if testing.Short() { - t.Skip("this takes a while") + if os.Getenv("LOTUS_TEST_WINDOW_POST") != "1" { + t.Skip("this takes a few minutes, set LOTUS_TEST_WINDOW_POST=1 to run") } logging.SetLogLevel("miner", "ERROR") @@ -527,5 +528,5 @@ func TestWindowedPost(t *testing.T) { logging.SetLogLevel("sub", "ERROR") logging.SetLogLevel("storageminer", "ERROR") - test.TestWindowPost(t, mockSbBuilder, 3*time.Millisecond, 10) + test.TestWindowPost(t, mockSbBuilder, 5*time.Millisecond, 10) } From c73326e96dc8a032e8b8b76533ab14789846874f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 1 Jun 2020 20:11:55 +0200 Subject: [PATCH 036/160] fsrepo: Use LevelDB for metadata datastore --- api/test/test.go | 1 + api/test/window_post.go | 6 +-- chain/gen/gen.go | 2 +- chain/store/store_test.go | 2 +- cmd/lotus-shed/import-car.go | 2 +- cmd/lotus/daemon.go | 2 +- go.mod | 3 ++ go.sum | 3 ++ node/modules/chain.go | 2 +- node/repo/fsrepo.go | 18 --------- node/repo/fsrepo_ds.go | 74 ++++++++++++++++++++++++++++++++++++ 11 files changed, 89 insertions(+), 26 deletions(-) create mode 100644 node/repo/fsrepo_ds.go diff --git a/api/test/test.go b/api/test/test.go index 06f0584fd..5972ca757 100644 --- a/api/test/test.go +++ b/api/test/test.go @@ -20,6 +20,7 @@ type TestStorageNode struct { } var PresealGenesis = -1 + const GenesisPreseals = 2 type StorageMiner struct { diff --git a/api/test/window_post.go b/api/test/window_post.go index b821df2a7..1239d5424 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -143,11 +143,11 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector head, err := client.ChainHead(ctx) require.NoError(t, err) - if head.Height() > di.PeriodStart + (miner2.WPoStProvingPeriod) + 2 { + if head.Height() > di.PeriodStart+(miner2.WPoStProvingPeriod)+2 { break } - if head.Height() % 100 == 0 { + if head.Height()%100 == 0 { fmt.Printf("@%d\n", head.Height()) } time.Sleep(blocktime) @@ -160,7 +160,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector require.NoError(t, err) require.Equal(t, p.MinerPower, p.TotalPower) - require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz) * uint64(nSectors + GenesisPreseals))) + require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz)*uint64(nSectors+GenesisPreseals))) // TODO: Inject faults here diff --git a/chain/gen/gen.go b/chain/gen/gen.go index b0c4cc044..de2ad9226 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -104,7 +104,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { return nil, xerrors.Errorf("failed to get metadata datastore: %w", err) } - bds, err := lr.Datastore("/blocks") + bds, err := lr.Datastore("/chain") if err != nil { return nil, xerrors.Errorf("failed to get blocks datastore: %w", err) } diff --git a/chain/store/store_test.go b/chain/store/store_test.go index 495b91b61..41b875916 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -55,7 +55,7 @@ func BenchmarkGetRandomness(b *testing.B) { b.Fatal(err) } - bds, err := lr.Datastore("/blocks") + bds, err := lr.Datastore("/chain") if err != nil { b.Fatal(err) } diff --git a/cmd/lotus-shed/import-car.go b/cmd/lotus-shed/import-car.go index f2f517a7f..970341a3a 100644 --- a/cmd/lotus-shed/import-car.go +++ b/cmd/lotus-shed/import-car.go @@ -42,7 +42,7 @@ var importCarCmd = &cli.Command{ return xerrors.Errorf("opening the car file: %w", err) } - ds, err := lr.Datastore("/blocks") + ds, err := lr.Datastore("/chain") if err != nil { return err } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 9e4e7d1a3..2894e0eea 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -271,7 +271,7 @@ func ImportChain(r repo.Repo, fname string) error { } defer lr.Close() //nolint:errcheck - ds, err := lr.Datastore("/blocks") + ds, err := lr.Datastore("/chain") if err != nil { return err } diff --git a/go.mod b/go.mod index d8659f350..e4a4a9993 100644 --- a/go.mod +++ b/go.mod @@ -48,6 +48,8 @@ require ( github.com/ipfs/go-cid v0.0.6-0.20200501230655-7c82f3b81c00 github.com/ipfs/go-datastore v0.4.4 github.com/ipfs/go-ds-badger2 v0.1.0 + github.com/ipfs/go-ds-leveldb v0.4.2 + github.com/ipfs/go-ds-measure v0.1.0 github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.1 github.com/ipfs/go-graphsync v0.0.6-0.20200504202014-9d5f2c26a103 @@ -101,6 +103,7 @@ require ( github.com/opentracing/opentracing-go v1.1.0 github.com/stretchr/objx v0.2.0 // indirect github.com/stretchr/testify v1.5.1 + github.com/syndtr/goleveldb v1.0.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba github.com/whyrusleeping/cbor-gen v0.0.0-20200504204219-64967432584d github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 diff --git a/go.sum b/go.sum index c68604e6d..57da8c1ce 100644 --- a/go.sum +++ b/go.sum @@ -387,6 +387,7 @@ github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAK github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= +github.com/ipfs/go-datastore v0.3.0/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= @@ -407,6 +408,8 @@ github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZ github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ds-leveldb v0.4.2 h1:QmQoAJ9WkPMUfBLnu1sBVy0xWWlJPg0m4kRAiJL9iaw= github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= +github.com/ipfs/go-ds-measure v0.1.0 h1:vE4TyY4aeLeVgnnPBC5QzKIjKrqzha0NCujTfgvVbVQ= +github.com/ipfs/go-ds-measure v0.1.0/go.mod h1:1nDiFrhLlwArTME1Ees2XaBOl49OoCgd2A3f8EchMSY= github.com/ipfs/go-filestore v1.0.0 h1:QR7ekKH+q2AGiWDc7W2Q0qHuYSRZGUJqUn0GsegEPb0= github.com/ipfs/go-filestore v1.0.0/go.mod h1:/XOCuNtIe2f1YPbiXdYvD0BKLA0JR1MgPiFOdcuu9SM= github.com/ipfs/go-fs-lock v0.0.1 h1:XHX8uW4jQBYWHj59XXcjg7BHlHxV9ZOYs6Y43yb7/l0= diff --git a/node/modules/chain.go b/node/modules/chain.go index ca0281acf..229a97cc4 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -61,7 +61,7 @@ func MessagePool(lc fx.Lifecycle, sm *stmgr.StateManager, ps *pubsub.PubSub, ds } func ChainBlockstore(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo) (dtypes.ChainBlockstore, error) { - blocks, err := r.Datastore("/blocks") + blocks, err := r.Datastore("/chain") if err != nil { return nil, err } diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index a1cc9a6ec..75b560d65 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -12,8 +12,6 @@ import ( "sync" "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - badger "github.com/ipfs/go-ds-badger2" fslock "github.com/ipfs/go-fs-lock" logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" @@ -267,22 +265,6 @@ func (fsr *fsLockedRepo) stillValid() error { return nil } -func (fsr *fsLockedRepo) Datastore(ns string) (datastore.Batching, error) { - fsr.dsOnce.Do(func() { - opts := badger.DefaultOptions - opts.Truncate = true - - fsr.ds, fsr.dsErr = badger.NewDatastore(fsr.join(fsDatastore), &opts) - /*if fsr.dsErr == nil { - fsr.ds = datastore.NewLogDatastore(fsr.ds, "fsrepo") - }*/ - }) - if fsr.dsErr != nil { - return nil, fsr.dsErr - } - return namespace.Wrap(fsr.ds, datastore.NewKey(ns)), nil -} - func (fsr *fsLockedRepo) Config() (interface{}, error) { if err := fsr.stillValid(); err != nil { return nil, err diff --git a/node/repo/fsrepo_ds.go b/node/repo/fsrepo_ds.go new file mode 100644 index 000000000..034635c4f --- /dev/null +++ b/node/repo/fsrepo_ds.go @@ -0,0 +1,74 @@ +package repo + +import ( + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/mount" + "github.com/ipfs/go-datastore/namespace" + "golang.org/x/xerrors" + "os" + "path/filepath" + + badger "github.com/ipfs/go-ds-badger2" + levelds "github.com/ipfs/go-ds-leveldb" + "github.com/ipfs/go-ds-measure" + ldbopts "github.com/syndtr/goleveldb/leveldb/opt" +) + +var fsDatastores = map[string]func(path string) (datastore.Batching, error){ + "chain": badgerDs, + "metadata": levelDs, + + // Those need to be fast for large writes... but also need a really good GC :c + "staging": badgerDs, // miner specific + "client": badgerDs, // client specific +} + +func badgerDs(path string) (datastore.Batching, error) { + opts := badger.DefaultOptions + opts.Truncate = true + + return badger.NewDatastore(path, &opts) +} + +func levelDs(path string) (datastore.Batching, error) { + return levelds.NewDatastore(path, &levelds.Options{ + Compression: ldbopts.NoCompression, + }) +} + +func (fsr *fsLockedRepo) openDatastore() (datastore.Batching, error) { + if err := os.MkdirAll(fsr.join(fsDatastore), 0755); err != nil { + return nil, xerrors.Errorf("mkdir %s: %w", fsr.join(fsDatastore), err) + } + + var mounts []mount.Mount + + for p, ctor := range fsDatastores { + prefix := datastore.NewKey(p) + + // TODO: optimization: don't init datastores we don't need + ds, err := ctor(fsr.join(filepath.Join(fsDatastore, p))) + if err != nil { + return nil, xerrors.Errorf("opening datastore %s: %w", prefix, err) + } + + ds = measure.New("fsrepo."+p, ds) + + mounts = append(mounts, mount.Mount{ + Prefix: prefix, + Datastore: ds, + }) + } + + return mount.New(mounts), nil +} + +func (fsr *fsLockedRepo) Datastore(ns string) (datastore.Batching, error) { + fsr.dsOnce.Do(func() { + fsr.ds, fsr.dsErr = fsr.openDatastore() + }) + if fsr.dsErr != nil { + return nil, fsr.dsErr + } + return namespace.Wrap(fsr.ds, datastore.NewKey(ns)), nil +} From 64f4e804467ac01863b1f39b7cdfc639117349b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 1 Jun 2020 20:43:51 +0200 Subject: [PATCH 037/160] Include build type in version --- Makefile | 2 +- build/params_2k.go | 2 ++ build/params_debug.go | 1 + build/version.go | 24 +++++++++++++++++++++++- cli/version.go | 4 +++- cmd/lotus-bench/main.go | 2 +- cmd/lotus-chainwatch/main.go | 2 +- cmd/lotus-fountain/main.go | 2 +- cmd/lotus-health/main.go | 2 +- cmd/lotus-seal-worker/main.go | 2 +- cmd/lotus-seed/main.go | 2 +- cmd/lotus-storage-miner/main.go | 2 +- cmd/lotus/main.go | 2 +- node/impl/common/common.go | 2 +- node/modules/lp2p/host.go | 2 +- 15 files changed, 40 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 7ffead022..99900e310 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ MODULES:= CLEAN:= BINS:= -ldflags=-X=github.com/filecoin-project/lotus/build.CurrentCommit='+git$(subst -,.,$(shell git describe --always --match=NeVeRmAtCh --dirty 2>/dev/null || git rev-parse --short HEAD 2>/dev/null))' +ldflags=-X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.$(subst -,.,$(shell git describe --always --match=NeVeRmAtCh --dirty 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)) ifneq ($(strip $(LDFLAGS)),) ldflags+=-extldflags=$(LDFLAGS) endif diff --git a/build/params_2k.go b/build/params_2k.go index ef1f2ac31..56e1ca2f8 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -16,6 +16,8 @@ func init() { abi.RegisteredProof_StackedDRG2KiBSeal: {}, } verifreg.MinVerifiedDealSize = big.NewInt(256) + + BuildType |= Build2k } // Seconds diff --git a/build/params_debug.go b/build/params_debug.go index e7c6d2e9e..f679c9178 100644 --- a/build/params_debug.go +++ b/build/params_debug.go @@ -4,6 +4,7 @@ package build func init() { InsecurePoStValidation = true + BuildType |= BuildDebug } // NOTE: Also includes settings from params_2k diff --git a/build/version.go b/build/version.go index e0ddebf36..7ee2afe65 100644 --- a/build/version.go +++ b/build/version.go @@ -3,11 +3,33 @@ package build import "fmt" var CurrentCommit string +var BuildType int + +const ( + BuildDefault = 0 + Build2k = 0x1 + BuildDebug = 0x3 +) + +func buildType() string { + switch BuildType { + case BuildDefault: + return "+default" + case BuildDebug: + return "+debug" + case Build2k: + return "+2k" + default: + return "+huh?" + } +} // BuildVersion is the local build version, set by build system const BuildVersion = "0.4.0" -var UserVersion = BuildVersion + CurrentCommit +func UserVersion() string { + return BuildVersion + buildType() + CurrentCommit +} type Version uint32 diff --git a/cli/version.go b/cli/version.go index cf4ad2932..50bc0d259 100644 --- a/cli/version.go +++ b/cli/version.go @@ -23,7 +23,9 @@ var versionCmd = &cli.Command{ if err != nil { return err } - fmt.Println(v) + fmt.Println("Daemon: ", v) + + fmt.Print("Local: ") cli.VersionPrinter(cctx) return nil }, diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 67fc1a43c..4ac52b497 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -78,7 +78,7 @@ func main() { app := &cli.App{ Name: "lotus-bench", Usage: "Benchmark performance of lotus on your hardware", - Version: build.UserVersion, + Version: build.UserVersion(), Commands: []*cli.Command{ proveCmd, sealBenchCmd, diff --git a/cmd/lotus-chainwatch/main.go b/cmd/lotus-chainwatch/main.go index a3114d8d0..4896cb366 100644 --- a/cmd/lotus-chainwatch/main.go +++ b/cmd/lotus-chainwatch/main.go @@ -29,7 +29,7 @@ func main() { app := &cli.App{ Name: "lotus-chainwatch", Usage: "Devnet token distribution utility", - Version: build.UserVersion, + Version: build.UserVersion(), Flags: []cli.Flag{ &cli.StringFlag{ Name: "repo", diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go index 8fc1804d7..8882ba3f1 100644 --- a/cmd/lotus-fountain/main.go +++ b/cmd/lotus-fountain/main.go @@ -46,7 +46,7 @@ func main() { app := &cli.App{ Name: "lotus-fountain", Usage: "Devnet token distribution utility", - Version: build.UserVersion, + Version: build.UserVersion(), Flags: []cli.Flag{ &cli.StringFlag{ Name: "repo", diff --git a/cmd/lotus-health/main.go b/cmd/lotus-health/main.go index 6f5ce8e90..264b3b84e 100644 --- a/cmd/lotus-health/main.go +++ b/cmd/lotus-health/main.go @@ -36,7 +36,7 @@ func main() { app := &cli.App{ Name: "lotus-health", Usage: "Tools for monitoring lotus daemon health", - Version: build.UserVersion, + Version: build.UserVersion(), Commands: local, Flags: []cli.Flag{ &cli.StringFlag{ diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index b43b7267f..e9721e9dd 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -47,7 +47,7 @@ func main() { app := &cli.App{ Name: "lotus-seal-worker", Usage: "Remote storage miner worker", - Version: build.UserVersion, + Version: build.UserVersion(), Flags: []cli.Flag{ &cli.StringFlag{ Name: FlagStorageRepo, diff --git a/cmd/lotus-seed/main.go b/cmd/lotus-seed/main.go index 7b80a0479..47dd850c0 100644 --- a/cmd/lotus-seed/main.go +++ b/cmd/lotus-seed/main.go @@ -39,7 +39,7 @@ func main() { app := &cli.App{ Name: "lotus-seed", Usage: "Seal sectors for genesis miner", - Version: build.UserVersion, + Version: build.UserVersion(), Flags: []cli.Flag{ &cli.StringFlag{ Name: "sector-dir", diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index 1186cbc28..e3e908fd1 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -57,7 +57,7 @@ func main() { app := &cli.App{ Name: "lotus-storage-miner", Usage: "Filecoin decentralized storage network storage miner", - Version: build.UserVersion, + Version: build.UserVersion(), EnableShellCompletion: true, Flags: []cli.Flag{ &cli.StringFlag{ diff --git a/cmd/lotus/main.go b/cmd/lotus/main.go index 33ff56ac3..4bc13d6fe 100644 --- a/cmd/lotus/main.go +++ b/cmd/lotus/main.go @@ -52,7 +52,7 @@ func main() { app := &cli.App{ Name: "lotus", Usage: "Filecoin decentralized storage network client", - Version: build.UserVersion, + Version: build.UserVersion(), EnableShellCompletion: true, Flags: []cli.Flag{ &cli.StringFlag{ diff --git a/node/impl/common/common.go b/node/impl/common/common.go index 5042d2ede..505466a8e 100644 --- a/node/impl/common/common.go +++ b/node/impl/common/common.go @@ -100,7 +100,7 @@ func (a *CommonAPI) ID(context.Context) (peer.ID, error) { func (a *CommonAPI) Version(context.Context) (api.Version, error) { return api.Version{ - Version: build.UserVersion, + Version: build.UserVersion(), APIVersion: build.APIVersion, BlockDelay: build.BlockDelay, diff --git a/node/modules/lp2p/host.go b/node/modules/lp2p/host.go index 2799dc8c3..b0436e9c9 100644 --- a/node/modules/lp2p/host.go +++ b/node/modules/lp2p/host.go @@ -46,7 +46,7 @@ func Host(mctx helpers.MetricsCtx, lc fx.Lifecycle, params P2PHostIn) (RawHost, libp2p.Peerstore(params.Peerstore), libp2p.NoListenAddrs, libp2p.Ping(true), - libp2p.UserAgent("lotus-" + build.UserVersion), + libp2p.UserAgent("lotus-" + build.UserVersion()), } for _, o := range params.Opts { opts = append(opts, o...) From a49dc90386d80bb963e1af24a4f5bd9983385be2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 1 Jun 2020 21:11:37 +0200 Subject: [PATCH 038/160] Fix version test --- api/test/test.go | 8 ++++---- build/version.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/test/test.go b/api/test/test.go index 5972ca757..d28f2e0ee 100644 --- a/api/test/test.go +++ b/api/test/test.go @@ -4,9 +4,11 @@ import ( "context" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - "github.com/stretchr/testify/assert" ) type TestNode struct { @@ -62,9 +64,7 @@ func (ts *testSuite) testVersion(t *testing.T) { if err != nil { t.Fatal(err) } - if v.Version != build.BuildVersion { - t.Error("Version didn't work properly") - } + require.Equal(t, v.Version, build.BuildVersion) } func (ts *testSuite) testID(t *testing.T) { diff --git a/build/version.go b/build/version.go index 7ee2afe65..51ef3e79d 100644 --- a/build/version.go +++ b/build/version.go @@ -14,7 +14,7 @@ const ( func buildType() string { switch BuildType { case BuildDefault: - return "+default" + return "" case BuildDebug: return "+debug" case Build2k: From 5605aae2694a419f32349fffd9566f901f782738 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 2 Jun 2020 16:29:39 +0200 Subject: [PATCH 039/160] Fix even more lint warnings Signed-off-by: Jakub Sztandera --- .golangci.yml | 4 ++ api/docgen/docgen.go | 3 +- api/test/deals.go | 4 +- api/test/mining.go | 2 +- build/version.go | 2 +- chain/blocksync/blocksync_client.go | 22 +++--- chain/events/events.go | 14 ++-- chain/events/events_called.go | 27 +++---- chain/events/events_height.go | 8 +-- chain/events/events_test.go | 2 +- chain/gen/gen.go | 9 +-- chain/messagepool/messagepool_test.go | 30 ++++---- chain/state/statetree.go | 4 +- chain/stmgr/stmgr.go | 6 +- chain/stmgr/utils.go | 6 +- chain/store/store.go | 5 +- chain/sub/incoming.go | 5 +- chain/sync.go | 4 +- chain/types/blockheader.go | 8 +-- chain/types/message.go | 16 ++--- chain/types/signedmessage.go | 16 ++--- chain/types/tipset.go | 4 +- chain/types/tipset_key_test.go | 6 +- chain/vm/invoker.go | 14 ++-- chain/vm/invoker_test.go | 2 +- chain/vm/mkactor.go | 3 +- chain/vm/runtime.go | 100 +++++++++++++------------- chain/vm/syscalls.go | 16 +---- chain/vm/vm.go | 20 +++--- cmd/lotus-chainwatch/blockssub.go | 2 +- cmd/lotus-chainwatch/dot.go | 6 ++ cmd/lotus-chainwatch/main.go | 2 +- cmd/lotus-chainwatch/sync.go | 2 +- cmd/lotus-fountain/main.go | 6 +- lib/parmap/parmap.go | 5 +- lib/sigs/doc.go | 2 +- lib/sigs/sigs.go | 4 +- lotuspond/api.go | 2 +- lotuspond/main.go | 6 +- lotuspond/spawn.go | 7 +- node/impl/full/state.go | 2 +- node/impl/storminer.go | 6 +- node/repo/fsrepo.go | 3 +- tools/stats/metrics.go | 3 +- 44 files changed, 204 insertions(+), 216 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index c8805a7a3..76bbc1949 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -37,6 +37,10 @@ issues: - path: build/params_.*\.go linters: - golint + + - path: api/apistruct/struct.go + linters: + - golint - path: .*_test.go linters: diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index f2d4d6ce4..eb4ab5fd9 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -193,8 +193,7 @@ func (v *Visitor) Visit(node ast.Node) ast.Visitor { const noComment = "There are not yet any comments for this method." -func parseApiASTInfo() (map[string]string, map[string]string) { - +func parseApiASTInfo() (map[string]string, map[string]string) { //nolint:golint fset := token.NewFileSet() pkgs, err := parser.ParseDir(fset, "./api", nil, parser.AllErrors|parser.ParseComments) if err != nil { diff --git a/api/test/deals.go b/api/test/deals.go index c35f0af07..0150a1315 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -35,7 +35,7 @@ func init() { } func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport bool) { - os.Setenv("BELLMAN_NO_GPU", "1") + _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() n, sn := b(t, 1, oneMiner) @@ -72,7 +72,7 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport } func TestDoubleDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) { - os.Setenv("BELLMAN_NO_GPU", "1") + _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() n, sn := b(t, 1, oneMiner) diff --git a/api/test/mining.go b/api/test/mining.go index 374999df1..b19095450 100644 --- a/api/test/mining.go +++ b/api/test/mining.go @@ -82,7 +82,7 @@ func (ts *testSuite) testMiningReal(t *testing.T) { } func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExport bool) { - os.Setenv("BELLMAN_NO_GPU", "1") + _ = os.Setenv("BELLMAN_NO_GPU", "1") // test making a deal with a fresh miner, and see if it starts to mine diff --git a/build/version.go b/build/version.go index 51ef3e79d..d5766ce6e 100644 --- a/build/version.go +++ b/build/version.go @@ -55,7 +55,7 @@ func (ve Version) EqMajorMinor(v2 Version) bool { // APIVersion is a semver version of the rpc api exposed var APIVersion Version = newVer(0, 3, 0) -//nolint:varcheck +//nolint:varcheck,deadcode const ( majorMask = 0xff0000 minorMask = 0xffff00 diff --git a/chain/blocksync/blocksync_client.go b/chain/blocksync/blocksync_client.go index f6453efde..129e8d332 100644 --- a/chain/blocksync/blocksync_client.go +++ b/chain/blocksync/blocksync_client.go @@ -561,26 +561,30 @@ func (bpt *bsPeerTracker) logSuccess(p peer.ID, dur time.Duration) { bpt.lk.Lock() defer bpt.lk.Unlock() - if pi, ok := bpt.peers[p]; !ok { + var pi *peerStats + var ok bool + if pi, ok = bpt.peers[p]; !ok { log.Warnw("log success called on peer not in tracker", "peerid", p.String()) return - } else { - pi.successes++ - - logTime(pi, dur) } + + pi.successes++ + logTime(pi, dur) } func (bpt *bsPeerTracker) logFailure(p peer.ID, dur time.Duration) { bpt.lk.Lock() defer bpt.lk.Unlock() - if pi, ok := bpt.peers[p]; !ok { + + var pi *peerStats + var ok bool + if pi, ok = bpt.peers[p]; !ok { log.Warn("log failure called on peer not in tracker", "peerid", p.String()) return - } else { - pi.failures++ - logTime(pi, dur) } + + pi.failures++ + logTime(pi, dur) } func (bpt *bsPeerTracker) removePeer(p peer.ID) { diff --git a/chain/events/events.go b/chain/events/events.go index c6d54d536..a325b5410 100644 --- a/chain/events/events.go +++ b/chain/events/events.go @@ -19,7 +19,7 @@ import ( var log = logging.Logger("events") -// `curH`-`ts.Height` = `confidence` +// HeightHandler `curH`-`ts.Height` = `confidence` type HeightHandler func(ctx context.Context, ts *types.TipSet, curH abi.ChainEpoch) error type RevertHandler func(ctx context.Context, ts *types.TipSet) error @@ -31,7 +31,7 @@ type heightHandler struct { revert RevertHandler } -type eventApi interface { +type eventAPI interface { ChainNotify(context.Context) (<-chan []*api.HeadChange, error) ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error) ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) @@ -42,7 +42,7 @@ type eventApi interface { } type Events struct { - api eventApi + api eventAPI tsc *tipSetCache lk sync.Mutex @@ -54,7 +54,7 @@ type Events struct { calledEvents } -func NewEvents(ctx context.Context, api eventApi) *Events { +func NewEvents(ctx context.Context, api eventAPI) *Events { gcConfidence := 2 * build.ForkLengthThreshold tsc := newTSCache(gcConfidence, api.ChainGetTipSetByHeight) @@ -82,9 +82,9 @@ func NewEvents(ctx context.Context, api eventApi) *Events { confQueue: map[triggerH]map[msgH][]*queuedEvent{}, revertQueue: map[msgH][]triggerH{}, - triggers: map[triggerId]*callHandler{}, - matchers: map[triggerId][]MatchFunc{}, - timeouts: map[abi.ChainEpoch]map[triggerId]int{}, + triggers: map[triggerID]*callHandler{}, + matchers: map[triggerID][]MatchFunc{}, + timeouts: map[abi.ChainEpoch]map[triggerID]int{}, }, } diff --git a/chain/events/events_called.go b/chain/events/events_called.go index 0ddb9476a..c43130dac 100644 --- a/chain/events/events_called.go +++ b/chain/events/events_called.go @@ -14,7 +14,7 @@ import ( const NoTimeout = math.MaxInt64 -type triggerId = uint64 +type triggerID = uint64 // msgH is the block height at which a message was present / event has happened type msgH = abi.ChainEpoch @@ -23,6 +23,7 @@ type msgH = abi.ChainEpoch // message (msgH+confidence) type triggerH = abi.ChainEpoch +// CalledHandler // `ts` is the tipset, in which the `msg` is included. // `curH`-`ts.Height` = `confidence` type CalledHandler func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (more bool, err error) @@ -48,7 +49,7 @@ type callHandler struct { } type queuedEvent struct { - trigger triggerId + trigger triggerID h abi.ChainEpoch msg *types.Message @@ -57,7 +58,7 @@ type queuedEvent struct { } type calledEvents struct { - cs eventApi + cs eventAPI tsc *tipSetCache ctx context.Context gcConfidence uint64 @@ -66,10 +67,10 @@ type calledEvents struct { lk sync.Mutex - ctr triggerId + ctr triggerID - triggers map[triggerId]*callHandler - matchers map[triggerId][]MatchFunc + triggers map[triggerID]*callHandler + matchers map[triggerID][]MatchFunc // maps block heights to events // [triggerH][msgH][event] @@ -78,8 +79,8 @@ type calledEvents struct { // [msgH][triggerH] revertQueue map[msgH][]triggerH - // [timeoutH+confidence][triggerId]{calls} - timeouts map[abi.ChainEpoch]map[triggerId]int + // [timeoutH+confidence][triggerID]{calls} + timeouts map[abi.ChainEpoch]map[triggerID]int } func (e *calledEvents) headChangeCalled(rev, app []*types.TipSet) error { @@ -157,8 +158,8 @@ func (e *calledEvents) checkNewCalls(ts *types.TipSet) { }) } -func (e *calledEvents) queueForConfidence(triggerId uint64, msg *types.Message, ts *types.TipSet) { - trigger := e.triggers[triggerId] +func (e *calledEvents) queueForConfidence(trigID uint64, msg *types.Message, ts *types.TipSet) { + trigger := e.triggers[trigID] appliedH := ts.Height() @@ -171,7 +172,7 @@ func (e *calledEvents) queueForConfidence(triggerId uint64, msg *types.Message, } byOrigH[appliedH] = append(byOrigH[appliedH], &queuedEvent{ - trigger: triggerId, + trigger: trigID, h: appliedH, msg: msg, }) @@ -231,11 +232,11 @@ func (e *calledEvents) applyTimeouts(ts *types.TipSet) { return // nothing to do } - for triggerId, calls := range triggers { + for triggerID, calls := range triggers { if calls > 0 { continue // don't timeout if the method was called } - trigger := e.triggers[triggerId] + trigger := e.triggers[triggerID] if trigger.disabled { continue } diff --git a/chain/events/events_height.go b/chain/events/events_height.go index 1b89e7bd7..cbf756c20 100644 --- a/chain/events/events_height.go +++ b/chain/events/events_height.go @@ -15,12 +15,12 @@ type heightEvents struct { tsc *tipSetCache gcConfidence abi.ChainEpoch - ctr triggerId + ctr triggerID - heightTriggers map[triggerId]*heightHandler + heightTriggers map[triggerID]*heightHandler - htTriggerHeights map[triggerH][]triggerId - htHeights map[msgH][]triggerId + htTriggerHeights map[triggerH][]triggerID + htHeights map[msgH][]triggerID ctx context.Context } diff --git a/chain/events/events_test.go b/chain/events/events_test.go index aaf3908d0..7533494e9 100644 --- a/chain/events/events_test.go +++ b/chain/events/events_test.go @@ -219,7 +219,7 @@ func (fcs *fakeCS) notifDone() { fcs.sync.Unlock() } -var _ eventApi = &fakeCS{} +var _ eventAPI = &fakeCS{} func TestAt(t *testing.T) { fcs := &fakeCS{ diff --git a/chain/gen/gen.go b/chain/gen/gen.go index de2ad9226..d58bbc28e 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -431,7 +431,7 @@ func (cg *ChainGen) makeBlock(parents *types.TipSet, m address.Address, vrfticke return fblk, err } -// This function is awkward. It's used to deal with messages made when +// ResyncBankerNonce is used for dealing with messages made when // simulating forks func (cg *ChainGen) ResyncBankerNonce(ts *types.TipSet) error { act, err := cg.sm.GetActor(cg.banker, ts) @@ -538,13 +538,6 @@ func (wpp *wppProvider) ComputeProof(context.Context, []abi.SectorInfo, abi.PoSt }}, nil } -type ProofInput struct { - sectors []abi.SectorInfo - hvrf []byte - challengedSectors []uint64 - vrfout []byte -} - func IsRoundWinner(ctx context.Context, ts *types.TipSet, round abi.ChainEpoch, miner address.Address, brand types.BeaconEntry, mbi *api.MiningBaseInfo, a MiningCheckAPI) (*types.ElectionProof, error) { diff --git a/chain/messagepool/messagepool_test.go b/chain/messagepool/messagepool_test.go index 26d35a361..1667cdc5c 100644 --- a/chain/messagepool/messagepool_test.go +++ b/chain/messagepool/messagepool_test.go @@ -16,7 +16,7 @@ import ( "github.com/ipfs/go-datastore" ) -type testMpoolApi struct { +type testMpoolAPI struct { cb func(rev, app []*types.TipSet) error bmsgs map[cid.Cid][]*types.SignedMessage @@ -25,68 +25,68 @@ type testMpoolApi struct { tipsets []*types.TipSet } -func newTestMpoolApi() *testMpoolApi { - return &testMpoolApi{ +func newTestMpoolApi() *testMpoolAPI { + return &testMpoolAPI{ bmsgs: make(map[cid.Cid][]*types.SignedMessage), statenonce: make(map[address.Address]uint64), } } -func (tma *testMpoolApi) applyBlock(t *testing.T, b *types.BlockHeader) { +func (tma *testMpoolAPI) applyBlock(t *testing.T, b *types.BlockHeader) { t.Helper() if err := tma.cb(nil, []*types.TipSet{mock.TipSet(b)}); err != nil { t.Fatal(err) } } -func (tma *testMpoolApi) revertBlock(t *testing.T, b *types.BlockHeader) { +func (tma *testMpoolAPI) revertBlock(t *testing.T, b *types.BlockHeader) { t.Helper() if err := tma.cb([]*types.TipSet{mock.TipSet(b)}, nil); err != nil { t.Fatal(err) } } -func (tma *testMpoolApi) setStateNonce(addr address.Address, v uint64) { +func (tma *testMpoolAPI) setStateNonce(addr address.Address, v uint64) { tma.statenonce[addr] = v } -func (tma *testMpoolApi) setBlockMessages(h *types.BlockHeader, msgs ...*types.SignedMessage) { +func (tma *testMpoolAPI) setBlockMessages(h *types.BlockHeader, msgs ...*types.SignedMessage) { tma.bmsgs[h.Cid()] = msgs tma.tipsets = append(tma.tipsets, mock.TipSet(h)) } -func (tma *testMpoolApi) SubscribeHeadChanges(cb func(rev, app []*types.TipSet) error) *types.TipSet { +func (tma *testMpoolAPI) SubscribeHeadChanges(cb func(rev, app []*types.TipSet) error) *types.TipSet { tma.cb = cb return nil } -func (tma *testMpoolApi) PutMessage(m types.ChainMsg) (cid.Cid, error) { +func (tma *testMpoolAPI) PutMessage(m types.ChainMsg) (cid.Cid, error) { return cid.Undef, nil } -func (tma *testMpoolApi) PubSubPublish(string, []byte) error { +func (tma *testMpoolAPI) PubSubPublish(string, []byte) error { return nil } -func (tma *testMpoolApi) StateGetActor(addr address.Address, ts *types.TipSet) (*types.Actor, error) { +func (tma *testMpoolAPI) StateGetActor(addr address.Address, ts *types.TipSet) (*types.Actor, error) { return &types.Actor{ Nonce: tma.statenonce[addr], Balance: types.NewInt(90000000), }, nil } -func (tma *testMpoolApi) StateAccountKey(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) { +func (tma *testMpoolAPI) StateAccountKey(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) { if addr.Protocol() != address.BLS && addr.Protocol() != address.SECP256K1 { return address.Undef, fmt.Errorf("given address was not a key addr") } return addr, nil } -func (tma *testMpoolApi) MessagesForBlock(h *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error) { +func (tma *testMpoolAPI) MessagesForBlock(h *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error) { return nil, tma.bmsgs[h.Cid()], nil } -func (tma *testMpoolApi) MessagesForTipset(ts *types.TipSet) ([]types.ChainMsg, error) { +func (tma *testMpoolAPI) MessagesForTipset(ts *types.TipSet) ([]types.ChainMsg, error) { if len(ts.Blocks()) != 1 { panic("cant deal with multiblock tipsets in this test") } @@ -108,7 +108,7 @@ func (tma *testMpoolApi) MessagesForTipset(ts *types.TipSet) ([]types.ChainMsg, return out, nil } -func (tma *testMpoolApi) LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error) { +func (tma *testMpoolAPI) LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error) { for _, ts := range tma.tipsets { if types.CidArrsEqual(tsk.Cids(), ts.Cids()) { return ts, nil diff --git a/chain/state/statetree.go b/chain/state/statetree.go index c93b1c83c..024524835 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -21,7 +21,7 @@ import ( var log = logging.Logger("statetree") -// Stores actors state by their ID. +// StateTree stores actors state by their ID. type StateTree struct { root *hamt.Node Store cbor.IpldStore @@ -149,7 +149,7 @@ func (st *StateTree) SetActor(addr address.Address, act *types.Actor) error { return nil } -// `LookupID` gets the ID address of this actor's `addr` stored in the `InitActor`. +// LookupID gets the ID address of this actor's `addr` stored in the `InitActor`. func (st *StateTree) LookupID(addr address.Address) (address.Address, error) { if addr.Protocol() == address.ID { return addr, nil diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 5a1eb88ea..17a9b58c1 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -326,7 +326,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl blkmsgs = append(blkmsgs, bm) } - return sm.ApplyBlocks(ctx, pstate, blkmsgs, abi.ChainEpoch(blks[0].Height), r, cb) + return sm.ApplyBlocks(ctx, pstate, blkmsgs, blks[0].Height, r, cb) } func (sm *StateManager) parentState(ts *types.TipSet) cid.Cid { @@ -405,8 +405,8 @@ func (sm *StateManager) LoadActorStateRaw(ctx context.Context, a address.Address return act, nil } -// Similar to `vm.ResolveToKeyAddr` but does not allow `Actor` type of addresses. Uses the `TipSet` `ts` -// to generate the VM state. +// ResolveToKeyAddr is similar to `vm.ResolveToKeyAddr` but does not allow `Actor` type of addresses. +// Uses the `TipSet` `ts` to generate the VM state. func (sm *StateManager) ResolveToKeyAddress(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) { switch addr.Protocol() { case address.BLS, address.SECP256K1: diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 001882ec6..308c30823 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -286,7 +286,7 @@ func GetMinerRecoveries(ctx context.Context, sm *StateManager, ts *types.TipSet, return mas.Recoveries, nil } -func GetStorageDeal(ctx context.Context, sm *StateManager, dealId abi.DealID, ts *types.TipSet) (*api.MarketDeal, error) { +func GetStorageDeal(ctx context.Context, sm *StateManager, dealID abi.DealID, ts *types.TipSet) (*api.MarketDeal, error) { var state market.State if _, err := sm.LoadActorState(ctx, builtin.StorageMarketActorAddr, &state, ts); err != nil { return nil, err @@ -298,7 +298,7 @@ func GetStorageDeal(ctx context.Context, sm *StateManager, dealId abi.DealID, ts } var dp market.DealProposal - if err := da.Get(ctx, uint64(dealId), &dp); err != nil { + if err := da.Get(ctx, uint64(dealID), &dp); err != nil { return nil, err } @@ -307,7 +307,7 @@ func GetStorageDeal(ctx context.Context, sm *StateManager, dealId abi.DealID, ts return nil, err } - st, found, err := sa.Get(dealId) + st, found, err := sa.Get(dealID) if err != nil { return nil, err } diff --git a/chain/store/store.go b/chain/store/store.go index e8c04b11b..a9e1eb136 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -387,7 +387,7 @@ func (cs *ChainStore) LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error) { return ts, nil } -// returns true if 'a' is an ancestor of 'b' +// IsAncestorOf returns true if 'a' is an ancestor of 'b' func (cs *ChainStore) IsAncestorOf(a, b *types.TipSet) (bool, error) { if b.Height() <= a.Height() { return false, nil @@ -1154,7 +1154,6 @@ func (cr *chainRand) GetRandomness(ctx context.Context, pers crypto.DomainSepara func (cs *ChainStore) GetTipSetFromKey(tsk types.TipSetKey) (*types.TipSet, error) { if tsk.IsEmpty() { return cs.GetHeaviestTipSet(), nil - } else { - return cs.LoadTipSet(tsk) } + return cs.LoadTipSet(tsk) } diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 2ef9c2718..8b5643de7 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -57,8 +57,7 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha return } - //nolint:golint - src := peer.ID(msg.GetFrom()) + src := msg.GetFrom() go func() { log.Infof("New block over pubsub: %s", blk.Cid()) @@ -207,7 +206,7 @@ func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub } } - err = sigs.CheckBlockSignature(blk.Header, ctx, key) + err = sigs.CheckBlockSignature(ctx, blk.Header, key) if err != nil { log.Errorf("block signature verification failed: %s", err) recordFailure("signature_verification_failed") diff --git a/chain/sync.go b/chain/sync.go index 23b790a06..f8a73356f 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -526,7 +526,7 @@ func blockSanityChecks(h *types.BlockHeader) error { return nil } -// Should match up with 'Semantical Validation' in validation.md in the spec +// ValidateBlock should match up with 'Semantical Validation' in validation.md in the spec func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) error { ctx, span := trace.StartSpan(ctx, "validateBlock") defer span.End() @@ -665,7 +665,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err }) blockSigCheck := async.Err(func() error { - if err := sigs.CheckBlockSignature(h, ctx, waddr); err != nil { + if err := sigs.CheckBlockSignature(ctx, h, waddr); err != nil { return xerrors.Errorf("check block signature failed: %w", err) } return nil diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index a5940d9e6..e238b3e5e 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -74,8 +74,8 @@ type BlockHeader struct { validated bool // true if the signature has been validated } -func (b *BlockHeader) ToStorageBlock() (block.Block, error) { - data, err := b.Serialize() +func (blk *BlockHeader) ToStorageBlock() (block.Block, error) { + data, err := blk.Serialize() if err != nil { return nil, err } @@ -89,8 +89,8 @@ func (b *BlockHeader) ToStorageBlock() (block.Block, error) { return block.NewBlockWithCid(data, c) } -func (b *BlockHeader) Cid() cid.Cid { - sb, err := b.ToStorageBlock() +func (blk *BlockHeader) Cid() cid.Cid { + sb, err := blk.ToStorageBlock() if err != nil { panic(err) // Not sure i'm entirely comfortable with this one, needs to be checked } diff --git a/chain/types/message.go b/chain/types/message.go index 0441eacca..68844d63c 100644 --- a/chain/types/message.go +++ b/chain/types/message.go @@ -41,20 +41,16 @@ type Message struct { Params []byte } -func (t *Message) BlockMiner() address.Address { - panic("implement me") +func (m *Message) Caller() address.Address { + return m.From } -func (t *Message) Caller() address.Address { - return t.From +func (m *Message) Receiver() address.Address { + return m.To } -func (t *Message) Receiver() address.Address { - return t.To -} - -func (t *Message) ValueReceived() abi.TokenAmount { - return t.Value +func (m *Message) ValueReceived() abi.TokenAmount { + return m.Value } func DecodeMessage(b []byte) (*Message, error) { diff --git a/chain/types/signedmessage.go b/chain/types/signedmessage.go index 802312ba9..54e82a957 100644 --- a/chain/types/signedmessage.go +++ b/chain/types/signedmessage.go @@ -9,12 +9,12 @@ import ( "github.com/multiformats/go-multihash" ) -func (m *SignedMessage) ToStorageBlock() (block.Block, error) { - if m.Signature.Type == crypto.SigTypeBLS { - return m.Message.ToStorageBlock() +func (sm *SignedMessage) ToStorageBlock() (block.Block, error) { + if sm.Signature.Type == crypto.SigTypeBLS { + return sm.Message.ToStorageBlock() } - data, err := m.Serialize() + data, err := sm.Serialize() if err != nil { return nil, err } @@ -28,12 +28,12 @@ func (m *SignedMessage) ToStorageBlock() (block.Block, error) { return block.NewBlockWithCid(data, c) } -func (m *SignedMessage) Cid() cid.Cid { - if m.Signature.Type == crypto.SigTypeBLS { - return m.Message.Cid() +func (sm *SignedMessage) Cid() cid.Cid { + if sm.Signature.Type == crypto.SigTypeBLS { + return sm.Message.Cid() } - sb, err := m.ToStorageBlock() + sb, err := sm.ToStorageBlock() if err != nil { panic(err) } diff --git a/chain/types/tipset.go b/chain/types/tipset.go index c1934d1d8..09483dc5e 100644 --- a/chain/types/tipset.go +++ b/chain/types/tipset.go @@ -23,8 +23,6 @@ type TipSet struct { height abi.ChainEpoch } -// why didnt i just export the fields? Because the struct has methods with the -// same names already type ExpTipSet struct { Cids []cid.Cid Blocks []*BlockHeader @@ -32,6 +30,8 @@ type ExpTipSet struct { } func (ts *TipSet) MarshalJSON() ([]byte, error) { + // why didnt i just export the fields? Because the struct has methods with the + // same names already return json.Marshal(ExpTipSet{ Cids: ts.cids, Blocks: ts.blks, diff --git a/chain/types/tipset_key_test.go b/chain/types/tipset_key_test.go index 43ff1a3df..7b3ce439d 100644 --- a/chain/types/tipset_key_test.go +++ b/chain/types/tipset_key_test.go @@ -61,9 +61,9 @@ func TestTipSetKey(t *testing.T) { t.Run("JSON", func(t *testing.T) { k0 := NewTipSetKey() - verifyJson(t, "[]", k0) + verifyJSON(t, "[]", k0) k3 := NewTipSetKey(c1, c2, c3) - verifyJson(t, `[`+ + verifyJSON(t, `[`+ `{"/":"bafy2bzacecesrkxghscnq7vatble2hqdvwat6ed23vdu4vvo3uuggsoaya7ki"},`+ `{"/":"bafy2bzacebxfyh2fzoxrt6kcgc5dkaodpcstgwxxdizrww225vrhsizsfcg4g"},`+ `{"/":"bafy2bzacedwviarjtjraqakob5pslltmuo5n3xev3nt5zylezofkbbv5jclyu"}`+ @@ -71,7 +71,7 @@ func TestTipSetKey(t *testing.T) { }) } -func verifyJson(t *testing.T, expected string, k TipSetKey) { +func verifyJSON(t *testing.T, expected string, k TipSetKey) { bytes, err := json.Marshal(k) require.NoError(t, err) assert.Equal(t, expected, string(bytes)) diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index 5ad0e6ee4..8bbb27ecc 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -32,7 +32,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors/aerrors" ) -type invoker struct { +type Invoker struct { builtInCode map[cid.Cid]nativeCode builtInState map[cid.Cid]reflect.Type } @@ -40,8 +40,8 @@ type invoker struct { type invokeFunc func(rt runtime.Runtime, params []byte) ([]byte, aerrors.ActorError) type nativeCode []invokeFunc -func NewInvoker() *invoker { - inv := &invoker{ +func NewInvoker() *Invoker { + inv := &Invoker{ builtInCode: make(map[cid.Cid]nativeCode), builtInState: make(map[cid.Cid]reflect.Type), } @@ -62,7 +62,7 @@ func NewInvoker() *invoker { return inv } -func (inv *invoker) Invoke(codeCid cid.Cid, rt runtime.Runtime, method abi.MethodNum, params []byte) ([]byte, aerrors.ActorError) { +func (inv *Invoker) Invoke(codeCid cid.Cid, rt runtime.Runtime, method abi.MethodNum, params []byte) ([]byte, aerrors.ActorError) { code, ok := inv.builtInCode[codeCid] if !ok { @@ -76,7 +76,7 @@ func (inv *invoker) Invoke(codeCid cid.Cid, rt runtime.Runtime, method abi.Metho } -func (inv *invoker) Register(c cid.Cid, instance Invokee, state interface{}) { +func (inv *Invoker) Register(c cid.Cid, instance Invokee, state interface{}) { code, err := inv.transform(instance) if err != nil { panic(xerrors.Errorf("%s: %w", string(c.Hash()), err)) @@ -89,9 +89,7 @@ type Invokee interface { Exports() []interface{} } -var tAError = reflect.TypeOf((*aerrors.ActorError)(nil)).Elem() - -func (*invoker) transform(instance Invokee) (nativeCode, error) { +func (*Invoker) transform(instance Invokee) (nativeCode, error) { itype := reflect.TypeOf(instance) exports := instance.Exports() for i, m := range exports { diff --git a/chain/vm/invoker_test.go b/chain/vm/invoker_test.go index b46b445a2..55b276421 100644 --- a/chain/vm/invoker_test.go +++ b/chain/vm/invoker_test.go @@ -76,7 +76,7 @@ func (basicContract) InvokeSomething10(rt runtime.Runtime, params *basicParams) } func TestInvokerBasic(t *testing.T) { - inv := invoker{} + inv := Invoker{} code, err := inv.transform(basicContract{}) assert.NoError(t, err) diff --git a/chain/vm/mkactor.go b/chain/vm/mkactor.go index 6f3274114..1a3fd97de 100644 --- a/chain/vm/mkactor.go +++ b/chain/vm/mkactor.go @@ -2,6 +2,7 @@ package vm import ( "context" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/specs-actors/actors/abi/big" "github.com/filecoin-project/specs-actors/actors/builtin" @@ -27,7 +28,7 @@ func init() { var EmptyObjectCid cid.Cid -// Creates account actors from only BLS/SECP256K1 addresses. +// TryCreateAccountActor creates account actors from only BLS/SECP256K1 addresses. func TryCreateAccountActor(rt *Runtime, addr address.Address) (*types.Actor, aerrors.ActorError) { addrID, err := rt.state.RegisterNewAddress(addr) if err != nil { diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 403a92fa7..b6598c5fe 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -106,8 +106,8 @@ type notFoundErr interface { IsNotFound() bool } -func (rs *Runtime) Get(c cid.Cid, o vmr.CBORUnmarshaler) bool { - if err := rs.cst.Get(context.TODO(), c, o); err != nil { +func (rt *Runtime) Get(c cid.Cid, o vmr.CBORUnmarshaler) bool { + if err := rt.cst.Get(context.TODO(), c, o); err != nil { var nfe notFoundErr if xerrors.As(err, &nfe) && nfe.IsNotFound() { if xerrors.As(err, new(cbor.SerializationError)) { @@ -121,8 +121,8 @@ func (rs *Runtime) Get(c cid.Cid, o vmr.CBORUnmarshaler) bool { return true } -func (rs *Runtime) Put(x vmr.CBORMarshaler) cid.Cid { - c, err := rs.cst.Put(context.TODO(), x) +func (rt *Runtime) Put(x vmr.CBORMarshaler) cid.Cid { + c, err := rt.cst.Put(context.TODO(), x) if err != nil { if xerrors.As(err, new(cbor.SerializationError)) { panic(aerrors.Newf(exitcode.ErrSerialization, "failed to marshal cbor object %s", err)) @@ -134,7 +134,7 @@ func (rs *Runtime) Put(x vmr.CBORMarshaler) cid.Cid { var _ vmr.Runtime = (*Runtime)(nil) -func (rs *Runtime) shimCall(f func() interface{}) (rval []byte, aerr aerrors.ActorError) { +func (rt *Runtime) shimCall(f func() interface{}) (rval []byte, aerr aerrors.ActorError) { defer func() { if r := recover(); r != nil { if ar, ok := r.(aerrors.ActorError); ok { @@ -149,8 +149,8 @@ func (rs *Runtime) shimCall(f func() interface{}) (rval []byte, aerr aerrors.Act ret := f() - if !rs.callerValidated { - rs.Abortf(exitcode.SysErrorIllegalActor, "Caller MUST be validated during method execution") + if !rt.callerValidated { + rt.Abortf(exitcode.SysErrorIllegalActor, "Caller MUST be validated during method execution") } switch ret := ret.(type) { @@ -171,25 +171,25 @@ func (rs *Runtime) shimCall(f func() interface{}) (rval []byte, aerr aerrors.Act } } -func (rs *Runtime) Message() vmr.Message { - return rs.vmsg +func (rt *Runtime) Message() vmr.Message { + return rt.vmsg } -func (rs *Runtime) ValidateImmediateCallerAcceptAny() { - rs.abortIfAlreadyValidated() +func (rt *Runtime) ValidateImmediateCallerAcceptAny() { + rt.abortIfAlreadyValidated() return } -func (rs *Runtime) CurrentBalance() abi.TokenAmount { - b, err := rs.GetBalance(rs.Message().Receiver()) +func (rt *Runtime) CurrentBalance() abi.TokenAmount { + b, err := rt.GetBalance(rt.Message().Receiver()) if err != nil { - rs.Abortf(err.RetCode(), "get current balance: %v", err) + rt.Abortf(err.RetCode(), "get current balance: %v", err) } return b } -func (rs *Runtime) GetActorCodeCID(addr address.Address) (ret cid.Cid, ok bool) { - act, err := rs.state.GetActor(addr) +func (rt *Runtime) GetActorCodeCID(addr address.Address) (ret cid.Cid, ok bool) { + act, err := rt.state.GetActor(addr) if err != nil { if xerrors.Is(err, types.ErrActorNotFound) { return cid.Undef, false @@ -209,8 +209,8 @@ func (rt *Runtime) GetRandomness(personalization crypto.DomainSeparationTag, ran return res } -func (rs *Runtime) Store() vmr.Store { - return rs +func (rt *Runtime) Store() vmr.Store { + return rt } func (rt *Runtime) NewActorAddress() address.Address { @@ -235,12 +235,12 @@ func (rt *Runtime) NewActorAddress() address.Address { return addr } -func (rt *Runtime) CreateActor(codeId cid.Cid, address address.Address) { - if !builtin.IsBuiltinActor(codeId) { +func (rt *Runtime) CreateActor(codeID cid.Cid, address address.Address) { + if !builtin.IsBuiltinActor(codeID) { rt.Abortf(exitcode.SysErrorIllegalArgument, "Can only create built-in actors.") } - if builtin.IsSingletonActor(codeId) { + if builtin.IsSingletonActor(codeID) { rt.Abortf(exitcode.SysErrorIllegalArgument, "Can only have one instance of singleton actors.") } @@ -252,7 +252,7 @@ func (rt *Runtime) CreateActor(codeId cid.Cid, address address.Address) { rt.ChargeGas(rt.Pricelist().OnCreateActor()) err = rt.state.SetActor(address, &types.Actor{ - Code: codeId, + Code: codeID, Head: EmptyObjectCid, Nonce: 0, Balance: big.Zero(), @@ -282,12 +282,12 @@ func (rt *Runtime) DeleteActor(addr address.Address) { } } -func (rs *Runtime) Syscalls() vmr.Syscalls { +func (rt *Runtime) Syscalls() vmr.Syscalls { // TODO: Make sure this is wrapped in something that charges gas for each of the calls - return rs.sys + return rt.sys } -func (rs *Runtime) StartSpan(name string) vmr.TraceSpan { +func (rt *Runtime) StartSpan(name string) vmr.TraceSpan { panic("implement me") } @@ -307,12 +307,12 @@ func (rt *Runtime) Context() context.Context { return rt.ctx } -func (rs *Runtime) Abortf(code exitcode.ExitCode, msg string, args ...interface{}) { +func (rt *Runtime) Abortf(code exitcode.ExitCode, msg string, args ...interface{}) { log.Error("Abortf: ", fmt.Sprintf(msg, args...)) panic(aerrors.NewfSkip(2, code, msg, args...)) } -func (rs *Runtime) AbortStateMsg(msg string) { +func (rt *Runtime) AbortStateMsg(msg string) { panic(aerrors.NewfSkip(3, 101, msg)) } @@ -330,8 +330,8 @@ func (rt *Runtime) ValidateImmediateCallerType(ts ...cid.Cid) { rt.Abortf(exitcode.SysErrForbidden, "caller cid type %q was not one of %v", callerCid, ts) } -func (rs *Runtime) CurrEpoch() abi.ChainEpoch { - return rs.height +func (rt *Runtime) CurrEpoch() abi.ChainEpoch { + return rt.height } type dumbWrapperType struct { @@ -342,20 +342,20 @@ func (dwt *dumbWrapperType) Into(um vmr.CBORUnmarshaler) error { return um.UnmarshalCBOR(bytes.NewReader(dwt.val)) } -func (rs *Runtime) Send(to address.Address, method abi.MethodNum, m vmr.CBORMarshaler, value abi.TokenAmount) (vmr.SendReturn, exitcode.ExitCode) { - if !rs.allowInternal { - rs.Abortf(exitcode.SysErrorIllegalActor, "runtime.Send() is currently disallowed") +func (rt *Runtime) Send(to address.Address, method abi.MethodNum, m vmr.CBORMarshaler, value abi.TokenAmount) (vmr.SendReturn, exitcode.ExitCode) { + if !rt.allowInternal { + rt.Abortf(exitcode.SysErrorIllegalActor, "runtime.Send() is currently disallowed") } var params []byte if m != nil { buf := new(bytes.Buffer) if err := m.MarshalCBOR(buf); err != nil { - rs.Abortf(exitcode.SysErrInvalidParameters, "failed to marshal input parameters: %s", err) + rt.Abortf(exitcode.SysErrInvalidParameters, "failed to marshal input parameters: %s", err) } params = buf.Bytes() } - ret, err := rs.internalSend(rs.Message().Receiver(), to, method, value, params) + ret, err := rt.internalSend(rt.Message().Receiver(), to, method, value, params) if err != nil { if err.IsFatal() { panic(err) @@ -422,48 +422,48 @@ func (rt *Runtime) internalSend(from, to address.Address, method abi.MethodNum, return ret, errSend } -func (rs *Runtime) State() vmr.StateHandle { - return &shimStateHandle{rs: rs} +func (rt *Runtime) State() vmr.StateHandle { + return &shimStateHandle{rt: rt} } type shimStateHandle struct { - rs *Runtime + rt *Runtime } func (ssh *shimStateHandle) Create(obj vmr.CBORMarshaler) { - c := ssh.rs.Put(obj) + c := ssh.rt.Put(obj) // TODO: handle error below - ssh.rs.stateCommit(EmptyObjectCid, c) + ssh.rt.stateCommit(EmptyObjectCid, c) } func (ssh *shimStateHandle) Readonly(obj vmr.CBORUnmarshaler) { - act, err := ssh.rs.state.GetActor(ssh.rs.Message().Receiver()) + act, err := ssh.rt.state.GetActor(ssh.rt.Message().Receiver()) if err != nil { - ssh.rs.Abortf(exitcode.SysErrorIllegalArgument, "failed to get actor for Readonly state: %s", err) + ssh.rt.Abortf(exitcode.SysErrorIllegalArgument, "failed to get actor for Readonly state: %s", err) } - ssh.rs.Get(act.Head, obj) + ssh.rt.Get(act.Head, obj) } func (ssh *shimStateHandle) Transaction(obj vmr.CBORer, f func() interface{}) interface{} { if obj == nil { - ssh.rs.Abortf(exitcode.SysErrorIllegalActor, "Must not pass nil to Transaction()") + ssh.rt.Abortf(exitcode.SysErrorIllegalActor, "Must not pass nil to Transaction()") } - act, err := ssh.rs.state.GetActor(ssh.rs.Message().Receiver()) + act, err := ssh.rt.state.GetActor(ssh.rt.Message().Receiver()) if err != nil { - ssh.rs.Abortf(exitcode.SysErrorIllegalActor, "failed to get actor for Transaction: %s", err) + ssh.rt.Abortf(exitcode.SysErrorIllegalActor, "failed to get actor for Transaction: %s", err) } baseState := act.Head - ssh.rs.Get(baseState, obj) + ssh.rt.Get(baseState, obj) - ssh.rs.allowInternal = false + ssh.rt.allowInternal = false out := f() - ssh.rs.allowInternal = true + ssh.rt.allowInternal = true - c := ssh.rs.Put(obj) + c := ssh.rt.Put(obj) // TODO: handle error below - ssh.rs.stateCommit(baseState, c) + ssh.rt.stateCommit(baseState, c) return out } diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index 07d086d13..100472c69 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -184,7 +184,7 @@ func (ss *syscallShim) VerifyBlockSig(blk *types.BlockHeader) error { return err } - if err := sigs.CheckBlockSignature(blk, ss.ctx, waddr); err != nil { + if err := sigs.CheckBlockSignature(ss.ctx, blk, waddr); err != nil { return err } @@ -202,20 +202,6 @@ func (ss *syscallShim) VerifyPoSt(proof abi.WindowPoStVerifyInfo) error { return nil } -func cidToCommD(c cid.Cid) [32]byte { - b := c.Bytes() - var out [32]byte - copy(out[:], b[len(b)-32:]) - return out -} - -func cidToCommR(c cid.Cid) [32]byte { - b := c.Bytes() - var out [32]byte - copy(out[:], b[len(b)-32:]) - return out -} - func (ss *syscallShim) VerifySeal(info abi.SealVerifyInfo) error { //_, span := trace.StartSpan(ctx, "ValidatePoRep") //defer span.End() diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 763708eca..da38fcbf3 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -131,7 +131,7 @@ type VM struct { cst *cbor.BasicIpldStore buf *bufbstore.BufferedBS blockHeight abi.ChainEpoch - inv *invoker + inv *Invoker rand Rand Syscalls runtime.Syscalls @@ -454,7 +454,7 @@ func (vm *VM) Flush(ctx context.Context) (cid.Cid, error) { return root, nil } -// vm.MutateState(idAddr, func(cst cbor.IpldStore, st *ActorStateType) error {...}) +// MutateState usage: MutateState(ctx, idAddr, func(cst cbor.IpldStore, st *ActorStateType) error {...}) func (vm *VM) MutateState(ctx context.Context, addr address.Address, fn interface{}) error { act, err := vm.cstate.GetActor(addr) if err != nil { @@ -591,7 +591,7 @@ func (vm *VM) Invoke(act *types.Actor, rt *Runtime, method abi.MethodNum, params return ret, nil } -func (vm *VM) SetInvoker(i *invoker) { +func (vm *VM) SetInvoker(i *Invoker) { vm.inv = i } @@ -607,17 +607,17 @@ func (vm *VM) transfer(from, to address.Address, amt types.BigInt) aerrors.Actor return nil } - fromId, err := vm.cstate.LookupID(from) + fromID, err := vm.cstate.LookupID(from) if err != nil { return aerrors.Fatalf("transfer failed when resolving sender address: %s", err) } - toId, err := vm.cstate.LookupID(to) + toID, err := vm.cstate.LookupID(to) if err != nil { return aerrors.Fatalf("transfer failed when resolving receiver address: %s", err) } - if fromId == toId { + if fromID == toID { return nil } @@ -625,12 +625,12 @@ func (vm *VM) transfer(from, to address.Address, amt types.BigInt) aerrors.Actor return aerrors.Newf(exitcode.SysErrForbidden, "attempted to transfer negative value: %s", amt) } - f, err := vm.cstate.GetActor(fromId) + f, err := vm.cstate.GetActor(fromID) if err != nil { return aerrors.Fatalf("transfer failed when retrieving sender actor: %s", err) } - t, err := vm.cstate.GetActor(toId) + t, err := vm.cstate.GetActor(toID) if err != nil { return aerrors.Fatalf("transfer failed when retrieving receiver actor: %s", err) } @@ -640,11 +640,11 @@ func (vm *VM) transfer(from, to address.Address, amt types.BigInt) aerrors.Actor } depositFunds(t, amt) - if err := vm.cstate.SetActor(fromId, f); err != nil { + if err := vm.cstate.SetActor(fromID, f); err != nil { return aerrors.Fatalf("transfer failed when setting receiver actor: %s", err) } - if err := vm.cstate.SetActor(toId, t); err != nil { + if err := vm.cstate.SetActor(toID, t); err != nil { return aerrors.Fatalf("transfer failed when setting sender actor: %s", err) } diff --git a/cmd/lotus-chainwatch/blockssub.go b/cmd/lotus-chainwatch/blockssub.go index 2147639a3..c569f1885 100644 --- a/cmd/lotus-chainwatch/blockssub.go +++ b/cmd/lotus-chainwatch/blockssub.go @@ -21,7 +21,7 @@ func subBlocks(ctx context.Context, api aapi.FullNode, st *storage) { bh.Cid(): bh, }, false) if err != nil { - //log.Errorf("%+v", err) + log.Errorf("%+v", err) } } } diff --git a/cmd/lotus-chainwatch/dot.go b/cmd/lotus-chainwatch/dot.go index e898f7cc3..22c8b7f05 100644 --- a/cmd/lotus-chainwatch/dot.go +++ b/cmd/lotus-chainwatch/dot.go @@ -20,7 +20,13 @@ var dotCmd = &cli.Command{ } minH, err := strconv.ParseInt(cctx.Args().Get(0), 10, 32) + if err != nil { + return err + } tosee, err := strconv.ParseInt(cctx.Args().Get(1), 10, 32) + if err != nil { + return err + } maxH := minH + tosee res, err := st.db.Query(`select block, parent, b.miner, b.height, p.height from block_parents diff --git a/cmd/lotus-chainwatch/main.go b/cmd/lotus-chainwatch/main.go index 4896cb366..faedb3bea 100644 --- a/cmd/lotus-chainwatch/main.go +++ b/cmd/lotus-chainwatch/main.go @@ -17,7 +17,7 @@ import ( var log = logging.Logger("chainwatch") func main() { - logging.SetLogLevel("*", "INFO") + _ = logging.SetLogLevel("*", "INFO") log.Info("Starting chainwatch") diff --git a/cmd/lotus-chainwatch/sync.go b/cmd/lotus-chainwatch/sync.go index ab2c5560c..d42a72b9b 100644 --- a/cmd/lotus-chainwatch/sync.go +++ b/cmd/lotus-chainwatch/sync.go @@ -146,7 +146,7 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS } if len(bh.Parents) == 0 { // genesis case - ts, err := types.NewTipSet([]*types.BlockHeader{bh}) + ts, _ := types.NewTipSet([]*types.BlockHeader{bh}) aadrs, err := api.StateListActors(ctx, ts.Key()) if err != nil { log.Error(err) diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go index 8882ba3f1..d4e41895b 100644 --- a/cmd/lotus-fountain/main.go +++ b/cmd/lotus-fountain/main.go @@ -215,14 +215,14 @@ func (h *handler) mkminer(w http.ResponseWriter, r *http.Request) { owner, err := address.NewFromString(r.FormValue("address")) if err != nil { w.WriteHeader(400) - w.Write([]byte(err.Error())) + _, _ = w.Write([]byte(err.Error())) return } if owner.Protocol() != address.BLS { w.WriteHeader(400) - w.Write([]byte("Miner address must use BLS. A BLS address starts with the prefix 't3'.")) - w.Write([]byte("Please create a BLS address by running \"lotus wallet new bls\" while connected to a Lotus node.")) + _, _ = w.Write([]byte("Miner address must use BLS. A BLS address starts with the prefix 't3'.")) + _, _ = w.Write([]byte("Please create a BLS address by running \"lotus wallet new bls\" while connected to a Lotus node.")) return } diff --git a/lib/parmap/parmap.go b/lib/parmap/parmap.go index 50613509e..b08594739 100644 --- a/lib/parmap/parmap.go +++ b/lib/parmap/parmap.go @@ -5,6 +5,7 @@ import ( "sync" ) +// MapArr transforms map into slice of map values func MapArr(in interface{}) interface{} { rin := reflect.ValueOf(in) rout := reflect.MakeSlice(reflect.SliceOf(rin.Type().Elem()), rin.Len(), rin.Len()) @@ -19,6 +20,7 @@ func MapArr(in interface{}) interface{} { return rout.Interface() } +// KMapArr transforms map into slice of map keys func KMapArr(in interface{}) interface{} { rin := reflect.ValueOf(in) rout := reflect.MakeSlice(reflect.SliceOf(rin.Type().Key()), rin.Len(), rin.Len()) @@ -33,7 +35,8 @@ func KMapArr(in interface{}) interface{} { return rout.Interface() } -// map[k]v => []func() (k, v) +// KMapArr transforms map into slice of functions returning (key, val) pairs. +// map[A]B => []func()(A, B) func KVMapArr(in interface{}) interface{} { rin := reflect.ValueOf(in) diff --git a/lib/sigs/doc.go b/lib/sigs/doc.go index 637cd2bcd..ca3093f39 100644 --- a/lib/sigs/doc.go +++ b/lib/sigs/doc.go @@ -1,4 +1,4 @@ -// Sigs package allows for signing, verifying signatures and key generation +// Package sigs allows for signing, verifying signatures and key generation // using key types selected by package user. // // For support of secp256k1 import: diff --git a/lib/sigs/sigs.go b/lib/sigs/sigs.go index 2f624ed76..4a4fd7340 100644 --- a/lib/sigs/sigs.go +++ b/lib/sigs/sigs.go @@ -68,7 +68,7 @@ func ToPublic(sigType crypto.SigType, pk []byte) ([]byte, error) { return sv.ToPublic(pk) } -func CheckBlockSignature(blk *types.BlockHeader, ctx context.Context, worker address.Address) error { +func CheckBlockSignature(ctx context.Context, blk *types.BlockHeader, worker address.Address) error { _, span := trace.StartSpan(ctx, "checkBlockSignature") defer span.End() @@ -103,7 +103,7 @@ type SigShim interface { var sigs map[crypto.SigType]SigShim -// RegisterSig should be only used during init +// RegisterSignature should be only used during init func RegisterSignature(typ crypto.SigType, vs SigShim) { if sigs == nil { sigs = make(map[crypto.SigType]SigShim) diff --git a/lotuspond/api.go b/lotuspond/api.go index 1f1432ca1..169cec1de 100644 --- a/lotuspond/api.go +++ b/lotuspond/api.go @@ -32,7 +32,7 @@ type api struct { type nodeInfo struct { Repo string ID int32 - ApiPort int32 + APIPort int32 State NodeState FullNode string // only for storage nodes diff --git a/lotuspond/main.go b/lotuspond/main.go index 3abcda635..8551a25dc 100644 --- a/lotuspond/main.go +++ b/lotuspond/main.go @@ -37,7 +37,7 @@ var onCmd = &cli.Command{ return err } - node := nodeById(client.Nodes(), int(nd)) + node := nodeByID(client.Nodes(), int(nd)) var cmd *exec.Cmd if !node.Storage { cmd = exec.Command("./lotus", cctx.Args().Slice()[1:]...) @@ -75,7 +75,7 @@ var shCmd = &cli.Command{ return err } - node := nodeById(client.Nodes(), int(nd)) + node := nodeByID(client.Nodes(), int(nd)) shcmd := exec.Command("/bin/bash") if !node.Storage { shcmd.Env = []string{ @@ -102,7 +102,7 @@ var shCmd = &cli.Command{ }, } -func nodeById(nodes []nodeInfo, i int) nodeInfo { +func nodeByID(nodes []nodeInfo, i int) nodeInfo { for _, n := range nodes { if n.ID == int32(i) { return n diff --git a/lotuspond/spawn.go b/lotuspond/spawn.go index 76b3f3aa2..32fc69b05 100644 --- a/lotuspond/spawn.go +++ b/lotuspond/spawn.go @@ -3,7 +3,6 @@ package main import ( "encoding/json" "fmt" - "github.com/filecoin-project/lotus/chain/types" "io" "io/ioutil" "os" @@ -12,6 +11,8 @@ import ( "sync/atomic" "time" + "github.com/filecoin-project/lotus/chain/types" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -122,7 +123,7 @@ func (api *api) Spawn() (nodeInfo, error) { info := nodeInfo{ Repo: dir, ID: id, - ApiPort: 2500 + id, + APIPort: 2500 + id, State: NodeRunning, } @@ -198,7 +199,7 @@ func (api *api) SpawnStorage(fullNodeRepo string) (nodeInfo, error) { info := nodeInfo{ Repo: dir, ID: id, - ApiPort: 2500 + id, + APIPort: 2500 + id, State: NodeRunning, FullNode: fullNodeRepo, diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 4e613a1b0..9b83e6ff9 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -682,7 +682,7 @@ func (a *StateAPI) MsigGetAvailableBalance(ctx context.Context, addr address.Add return act.Balance, nil } - minBalance := types.BigDiv(types.BigInt(st.InitialBalance), types.NewInt(uint64(st.UnlockDuration))) + minBalance := types.BigDiv(st.InitialBalance, types.NewInt(uint64(st.UnlockDuration))) minBalance = types.BigMul(minBalance, types.NewInt(uint64(offset))) return types.BigSub(act.Balance, minBalance), nil } diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 491ce27bb..cca706746 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -46,7 +46,7 @@ type StorageMinerAPI struct { func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) { if !auth.HasPerm(r.Context(), nil, apistruct.PermAdmin) { w.WriteHeader(401) - json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing write permission"}) + _ = json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing write permission"}) return } @@ -185,7 +185,7 @@ func (sm *StorageMinerAPI) MarketImportDealData(ctx context.Context, propCid cid if err != nil { return xerrors.Errorf("failed to open file: %w", err) } - defer fi.Close() + defer fi.Close() //nolint:errcheck return sm.StorageProvider.ImportDataForDeal(ctx, propCid, fi) } @@ -211,7 +211,7 @@ func (sm *StorageMinerAPI) DealsImportData(ctx context.Context, deal cid.Cid, fn if err != nil { return xerrors.Errorf("failed to open given file: %w", err) } - defer fi.Close() + defer fi.Close() //nolint:errcheck return sm.StorageProvider.ImportDataForDeal(ctx, deal, fi) } diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 75b560d65..6733b0868 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -63,8 +63,7 @@ var ErrRepoExists = xerrors.New("repo exists") // FsRepo is struct for repo, use NewFS to create type FsRepo struct { - path string - repoType RepoType + path string } var _ Repo = &FsRepo{} diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index 9c6887723..eb653e1d7 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -38,8 +38,7 @@ func (pl *PointList) Points() []models.Point { } type InfluxWriteQueue struct { - influx client.Client - ch chan client.BatchPoints + ch chan client.BatchPoints } func NewInfluxWriteQueue(ctx context.Context, influx client.Client) *InfluxWriteQueue { From 96ade5f2dd1e5c7bc67cf7b4b7e7b17a1ffce96c Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 2 Jun 2020 18:38:41 +0200 Subject: [PATCH 040/160] Cleanup more lint warnings Signed-off-by: Jakub Sztandera --- chain/events/events_called.go | 2 +- chain/events/events_test.go | 4 ---- chain/events/utils.go | 2 +- chain/messagepool/messagepool_test.go | 6 +++--- chain/stmgr/stmgr.go | 2 +- lib/parmap/parmap.go | 2 +- 6 files changed, 7 insertions(+), 11 deletions(-) diff --git a/chain/events/events_called.go b/chain/events/events_called.go index c43130dac..04e7be715 100644 --- a/chain/events/events_called.go +++ b/chain/events/events_called.go @@ -23,7 +23,7 @@ type msgH = abi.ChainEpoch // message (msgH+confidence) type triggerH = abi.ChainEpoch -// CalledHandler +// CalledHandler arguments: // `ts` is the tipset, in which the `msg` is included. // `curH`-`ts.Height` = `confidence` type CalledHandler func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (more bool, err error) diff --git a/chain/events/events_test.go b/chain/events/events_test.go index 7533494e9..c81c04b32 100644 --- a/chain/events/events_test.go +++ b/chain/events/events_test.go @@ -209,10 +209,6 @@ func (fcs *fakeCS) advance(rev, app int, msgs map[int]cid.Cid, nulls ...int) { / fcs.sync.Lock() fcs.sub(revs, apps) - - fcs.sync.Lock() - fcs.sync.Unlock() - } func (fcs *fakeCS) notifDone() { diff --git a/chain/events/utils.go b/chain/events/utils.go index 1f4ca381b..d525e5368 100644 --- a/chain/events/utils.go +++ b/chain/events/utils.go @@ -36,7 +36,7 @@ func (e *calledEvents) CheckMsg(ctx context.Context, smsg types.ChainMsg, hnd Ca func (e *calledEvents) MatchMsg(inmsg *types.Message) MatchFunc { return func(msg *types.Message) (bool, error) { if msg.From == inmsg.From && msg.Nonce == inmsg.Nonce && !inmsg.Equals(msg) { - return false, xerrors.Errorf("matching msg %s from %s, nonce %d: got duplicate origin/nonce msg %s", inmsg.Cid(), inmsg.From, inmsg.Nonce, msg.Nonce) + return false, xerrors.Errorf("matching msg %s from %s, nonce %d: got duplicate origin/nonce msg %d", inmsg.Cid(), inmsg.From, inmsg.Nonce, msg.Nonce) } return inmsg.Equals(msg), nil diff --git a/chain/messagepool/messagepool_test.go b/chain/messagepool/messagepool_test.go index 1667cdc5c..d1dedd534 100644 --- a/chain/messagepool/messagepool_test.go +++ b/chain/messagepool/messagepool_test.go @@ -25,7 +25,7 @@ type testMpoolAPI struct { tipsets []*types.TipSet } -func newTestMpoolApi() *testMpoolAPI { +func newTestMpoolAPI() *testMpoolAPI { return &testMpoolAPI{ bmsgs: make(map[cid.Cid][]*types.SignedMessage), statenonce: make(map[address.Address]uint64), @@ -138,7 +138,7 @@ func mustAdd(t *testing.T, mp *MessagePool, msg *types.SignedMessage) { } func TestMessagePool(t *testing.T) { - tma := newTestMpoolApi() + tma := newTestMpoolAPI() w, err := wallet.NewWallet(wallet.NewMemKeyStore()) if err != nil { @@ -179,7 +179,7 @@ func TestMessagePool(t *testing.T) { } func TestRevertMessages(t *testing.T) { - tma := newTestMpoolApi() + tma := newTestMpoolAPI() w, err := wallet.NewWallet(wallet.NewMemKeyStore()) if err != nil { diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 17a9b58c1..a4bef46a2 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -405,7 +405,7 @@ func (sm *StateManager) LoadActorStateRaw(ctx context.Context, a address.Address return act, nil } -// ResolveToKeyAddr is similar to `vm.ResolveToKeyAddr` but does not allow `Actor` type of addresses. +// ResolveToKeyAddress is similar to `vm.ResolveToKeyAddr` but does not allow `Actor` type of addresses. // Uses the `TipSet` `ts` to generate the VM state. func (sm *StateManager) ResolveToKeyAddress(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) { switch addr.Protocol() { diff --git a/lib/parmap/parmap.go b/lib/parmap/parmap.go index b08594739..dcf0ef3c8 100644 --- a/lib/parmap/parmap.go +++ b/lib/parmap/parmap.go @@ -35,7 +35,7 @@ func KMapArr(in interface{}) interface{} { return rout.Interface() } -// KMapArr transforms map into slice of functions returning (key, val) pairs. +// KVMapArr transforms map into slice of functions returning (key, val) pairs. // map[A]B => []func()(A, B) func KVMapArr(in interface{}) interface{} { rin := reflect.ValueOf(in) From 22851df145479460ba3b0d05a618530cbf3fdfd2 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 2 Jun 2020 18:45:50 +0200 Subject: [PATCH 041/160] Undo change in events_test Signed-off-by: Jakub Sztandera --- chain/events/events_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/chain/events/events_test.go b/chain/events/events_test.go index c81c04b32..a048789ec 100644 --- a/chain/events/events_test.go +++ b/chain/events/events_test.go @@ -209,6 +209,9 @@ func (fcs *fakeCS) advance(rev, app int, msgs map[int]cid.Cid, nulls ...int) { / fcs.sync.Lock() fcs.sub(revs, apps) + + fcs.sync.Lock() + fcs.sync.Unlock() //nolint:staticcheck } func (fcs *fakeCS) notifDone() { From d7a2556292243984c2aa0539b3185f27f347f08d Mon Sep 17 00:00:00 2001 From: acruikshank Date: Tue, 2 Jun 2020 16:43:39 -0400 Subject: [PATCH 042/160] expand tilda before passing file name into chain import --- cmd/lotus/daemon.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 91af88a4e..8506b521d 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -150,6 +150,11 @@ var DaemonCmd = &cli.Command{ chainfile := cctx.String("import-chain") if chainfile != "" { + chainfile, err := homedir.Expand(chainfile) + if err != nil { + return err + } + if err := ImportChain(r, chainfile); err != nil { return err } From 1fd20b636bedbce56714d3090c6fffa2e28f49b3 Mon Sep 17 00:00:00 2001 From: acruikshank Date: Tue, 2 Jun 2020 16:48:50 -0400 Subject: [PATCH 043/160] tidy --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index ed91f8560..dc0908832 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,6 @@ github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1 github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA= github.com/filecoin-project/go-fil-markets v0.2.7 h1:bgdK/e+xW15aVZLtdFLzAHdrx1hqtGF9veg2lstLK6o= github.com/filecoin-project/go-fil-markets v0.2.7/go.mod h1:LI3VFHse33aU0djAmFQ8+Hg39i0J8ibAoppGu6TbgkA= -github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200520183639-7c6ee2e066b4 h1:H8AVYu0MV9m3CSnKMxeILMfh8xJtnqVdXfBF/qbzgu0= -github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200520183639-7c6ee2e066b4/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM= github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs= From 3423d298156dcce5a587f5b77457762aceedd6cd Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 2 Jun 2020 22:54:29 +0200 Subject: [PATCH 044/160] Prune go mod cache Circle reports 1.9GB, should be closer to 300MB Signed-off-by: Jakub Sztandera --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 700f1fbc9..858b21123 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -115,7 +115,7 @@ jobs: - go/mod-download - restore_cache: name: restore go mod cache - key: v1-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/go.mod" }} + key: v2-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/go.mod" }} - run: command: make debug @@ -162,7 +162,7 @@ jobs: - go/mod-download - restore_cache: name: restore go mod cache - key: v1-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/go.mod" }} + key: v2-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/go.mod" }} - run: command: make deps lotus no_output_timeout: 30m From 415b69155bbfcaf9431858819832f0518d998f19 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 2 Jun 2020 23:02:37 +0200 Subject: [PATCH 045/160] Disable caches Signed-off-by: Jakub Sztandera --- .circleci/config.yml | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 858b21123..3b22557de 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ version: 2.1 orbs: - go: gotest/tools@0.0.9 + go: gotest/tools@0.0.13 executors: golang: @@ -79,7 +79,6 @@ jobs: steps: - install-deps - prepare - - go/mod-download - go/mod-tidy-check build-all: @@ -90,9 +89,6 @@ jobs: - go/mod-download - run: sudo apt-get update - run: sudo apt-get install npm - - restore_cache: - name: restore go mod cache - key: v1-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/go.mod" }} - run: command: make buildall - store_artifacts: @@ -112,10 +108,6 @@ jobs: steps: - install-deps - prepare - - go/mod-download - - restore_cache: - name: restore go mod cache - key: v2-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/go.mod" }} - run: command: make debug @@ -160,9 +152,6 @@ jobs: - install-deps - prepare - go/mod-download - - restore_cache: - name: restore go mod cache - key: v2-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/go.mod" }} - run: command: make deps lotus no_output_timeout: 30m @@ -193,13 +182,6 @@ jobs: shell: /bin/bash -eo pipefail command: | bash <(curl -s https://codecov.io/bash) - - save_cache: - name: save go mod cache - key: v1-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/go.mod" }} - paths: - - "~/go/pkg" - - "~/go/src/github.com" - - "~/go/src/golang.org" test-short: <<: *test @@ -234,7 +216,7 @@ jobs: curl --location https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 --output /usr/local/bin/jq chmod +x /usr/local/bin/jq - restore_cache: - name: restore go mod and cargo cache + name: restore cargo cache key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }} - install-deps - go/mod-download From 9326cb5411a80915593aa200cf70e570143f4ab5 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 2 Jun 2020 23:07:48 +0200 Subject: [PATCH 046/160] Update golangci-lint, increase its timeout Signed-off-by: Jakub Sztandera --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3b22557de..1da5c7952 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -259,7 +259,7 @@ jobs: default: golang golangci-lint-version: type: string - default: 1.23.8 + default: 1.27.0 concurrency: type: string default: '2' @@ -285,7 +285,7 @@ jobs: - run: name: Lint command: | - $HOME/.local/bin/golangci-lint run -v \ + $HOME/.local/bin/golangci-lint run -v --timeout 2m \ --concurrency << parameters.concurrency >> << parameters.args >> lint-changes: <<: *lint From 42c76aaebcce87d9d7e6d8d002b445b1cdd07c8a Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 2 Jun 2020 23:11:07 +0200 Subject: [PATCH 047/160] Remove mod-download Signed-off-by: Jakub Sztandera --- .circleci/config.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1da5c7952..cbbc9514a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -86,7 +86,6 @@ jobs: steps: - install-deps - prepare - - go/mod-download - run: sudo apt-get update - run: sudo apt-get install npm - run: @@ -151,7 +150,6 @@ jobs: steps: - install-deps - prepare - - go/mod-download - run: command: make deps lotus no_output_timeout: 30m @@ -219,7 +217,6 @@ jobs: name: restore cargo cache key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }} - install-deps - - go/mod-download - run: command: make build no_output_timeout: 30m @@ -246,7 +243,6 @@ jobs: steps: - install-deps - prepare - - go/mod-download - run: command: "! go fmt ./... 2>&1 | read" @@ -275,7 +271,6 @@ jobs: steps: - install-deps - prepare - - go/mod-download - run: command: make deps no_output_timeout: 30m From 2fdcf5347b9ddf9381e26525d99d179107e9fdd3 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 2 Jun 2020 17:35:23 -0700 Subject: [PATCH 048/160] reuse GetTipsetByHeight in GetRandomness --- chain/store/store.go | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index 1d0668032..c9dac2e31 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -909,25 +909,26 @@ func (cs *ChainStore) GetRandomness(ctx context.Context, blks []cid.Cid, pers cr defer span.End() span.AddAttributes(trace.Int64Attribute("round", int64(round))) - //defer func() { - //log.Infof("getRand %v %d %d %x -> %x", blks, pers, round, entropy, out) - //}() - for { - nts, err := cs.LoadTipSet(types.NewTipSetKey(blks...)) - if err != nil { - return nil, err - } - - mtb := nts.MinTicketBlock() - - // if at (or just past -- for null epochs) appropriate epoch - // or at genesis (works for negative epochs) - if nts.Height() <= round || mtb.Height == 0 { - return DrawRandomness(nts.MinTicketBlock().Ticket.VRFProof, pers, round, entropy) - } - - blks = mtb.Parents + ts, err := cs.LoadTipSet(types.NewTipSetKey(blks...)) + if err != nil { + return nil, err } + + searchHeight := round + if searchHeight < 0 { + searchHeight = 0 + } + + randTs, err := cs.GetTipsetByHeight(ctx, searchHeight, ts, true) + if err != nil { + return nil, err + } + + mtb := randTs.MinTicketBlock() + + // if at (or just past -- for null epochs) appropriate epoch + // or at genesis (works for negative epochs) + return DrawRandomness(mtb.Ticket.VRFProof, pers, round, entropy) } // GetTipsetByHeight returns the tipset on the chain behind 'ts' at the given From 186fd4da74e019ce1bbdacbca1b237e1b0ec4e83 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 3 Jun 2020 03:13:49 +0200 Subject: [PATCH 049/160] Add `lotus net scores` for pubsub score visibility Signed-off-by: Jakub Sztandera --- api/api_common.go | 1 + api/apistruct/struct.go | 4 ++++ api/types.go | 6 ++++++ cli/net.go | 26 +++++++++++++++++++++++++- node/builder.go | 1 + node/impl/common/common.go | 18 ++++++++++++++++++ node/modules/dtypes/scorekeeper.go | 24 ++++++++++++++++++++++++ node/modules/lp2p/pubsub.go | 7 ++++++- 8 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 node/modules/dtypes/scorekeeper.go diff --git a/api/api_common.go b/api/api_common.go index a49397247..7f0514fdf 100644 --- a/api/api_common.go +++ b/api/api_common.go @@ -25,6 +25,7 @@ type Common interface { NetAddrsListen(context.Context) (peer.AddrInfo, error) NetDisconnect(context.Context, peer.ID) error NetFindPeer(context.Context, peer.ID) (peer.AddrInfo, error) + NetPubsubScores(context.Context) ([]PubsubScore, error) // ID returns peerID of libp2p node backing this API ID(context.Context) (peer.ID, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 1f2191ae8..8532cad11 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -42,6 +42,7 @@ type CommonStruct struct { NetAddrsListen func(context.Context) (peer.AddrInfo, error) `perm:"read"` NetDisconnect func(context.Context, peer.ID) error `perm:"write"` NetFindPeer func(context.Context, peer.ID) (peer.AddrInfo, error) `perm:"read"` + NetPubsubScores func(context.Context) ([]api.PubsubScore, error) `perm:"read"` ID func(context.Context) (peer.ID, error) `perm:"read"` Version func(context.Context) (api.Version, error) `perm:"read"` @@ -256,6 +257,9 @@ func (c *CommonStruct) AuthNew(ctx context.Context, perms []auth.Permission) ([] return c.Internal.AuthNew(ctx, perms) } +func (c *CommonStruct) NetPubsubScores(ctx context.Context) ([]api.PubsubScore, error) { + return c.Internal.NetPubsubScores(ctx) +} func (c *CommonStruct) NetConnectedness(ctx context.Context, pid peer.ID) (network.Connectedness, error) { return c.Internal.NetConnectedness(ctx, pid) } diff --git a/api/types.go b/api/types.go index 26d0695e1..fcc013441 100644 --- a/api/types.go +++ b/api/types.go @@ -3,6 +3,7 @@ package api import ( "encoding/json" + "github.com/libp2p/go-libp2p-core/peer" ma "github.com/multiformats/go-multiaddr" ) @@ -33,3 +34,8 @@ type ObjStat struct { Size uint64 Links uint64 } + +type PubsubScore struct { + ID peer.ID + Score float64 +} diff --git a/cli/net.go b/cli/net.go index b49d9604f..366162c3f 100644 --- a/cli/net.go +++ b/cli/net.go @@ -21,6 +21,7 @@ var netCmd = &cli.Command{ netListen, netId, netFindPeer, + netScores, }, } @@ -44,7 +45,30 @@ var netPeers = &cli.Command{ }) for _, peer := range peers { - fmt.Println(peer) + fmt.Printf("%s, %s\n", peer.ID, peer.Addrs) + } + + return nil + }, +} + +var netScores = &cli.Command{ + Name: "scores", + Usage: "Print peers' pubsub scores", + Action: func(cctx *cli.Context) error { + api, closer, err := GetAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + scores, err := api.NetPubsubScores(ctx) + if err != nil { + return err + } + + for _, peer := range scores { + fmt.Printf("%s, %f\n", peer.ID, peer.Score) } return nil diff --git a/node/builder.go b/node/builder.go index 3b8e0d62f..292cf39e6 100644 --- a/node/builder.go +++ b/node/builder.go @@ -180,6 +180,7 @@ func libp2p() Option { Override(ConnectionManagerKey, lp2p.ConnectionManager(50, 200, 20*time.Second, nil)), Override(AutoNATSvcKey, lp2p.AutoNATService), + Override(new(*dtypes.ScoreKeeper), lp2p.ScoreKeeper), Override(new(*pubsub.PubSub), lp2p.GossipSub), Override(new(*config.Pubsub), func(bs dtypes.Bootstrapper) *config.Pubsub { return &config.Pubsub{ diff --git a/node/impl/common/common.go b/node/impl/common/common.go index 505466a8e..6de137d9f 100644 --- a/node/impl/common/common.go +++ b/node/impl/common/common.go @@ -2,6 +2,8 @@ package common import ( "context" + "sort" + "strings" logging "github.com/ipfs/go-log/v2" @@ -28,6 +30,7 @@ type CommonAPI struct { APISecret *dtypes.APIAlg Host host.Host Router lp2p.BaseIpfsRouting + Sk *dtypes.ScoreKeeper } type jwtPayload struct { @@ -54,6 +57,21 @@ func (a *CommonAPI) AuthNew(ctx context.Context, perms []auth.Permission) ([]byt func (a *CommonAPI) NetConnectedness(ctx context.Context, pid peer.ID) (network.Connectedness, error) { return a.Host.Network().Connectedness(pid), nil } +func (a *CommonAPI) NetPubsubScores(context.Context) ([]api.PubsubScore, error) { + scores := a.Sk.Get() + out := make([]api.PubsubScore, len(scores)) + i := 0 + for k, v := range scores { + out[i] = api.PubsubScore{k, v} + i++ + } + + sort.Slice(out, func(i, j int) bool { + return strings.Compare(string(out[i].ID), string(out[j].ID)) > 0 + }) + + return out, nil +} func (a *CommonAPI) NetPeers(context.Context) ([]peer.AddrInfo, error) { conns := a.Host.Network().Conns() diff --git a/node/modules/dtypes/scorekeeper.go b/node/modules/dtypes/scorekeeper.go new file mode 100644 index 000000000..221821bf8 --- /dev/null +++ b/node/modules/dtypes/scorekeeper.go @@ -0,0 +1,24 @@ +package dtypes + +import ( + "sync" + + peer "github.com/libp2p/go-libp2p-peer" +) + +type ScoreKeeper struct { + lk sync.Mutex + scores map[peer.ID]float64 +} + +func (sk *ScoreKeeper) Update(scores map[peer.ID]float64) { + sk.lk.Lock() + sk.scores = scores + sk.lk.Unlock() +} + +func (sk *ScoreKeeper) Get() map[peer.ID]float64 { + sk.lk.Lock() + defer sk.lk.Unlock() + return sk.scores +} diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go index 35285f5c1..739bbd115 100644 --- a/node/modules/lp2p/pubsub.go +++ b/node/modules/lp2p/pubsub.go @@ -28,7 +28,11 @@ func init() { pubsub.GossipSubDirectConnectInitialDelay = 30 * time.Second } -func GossipSub(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, nn dtypes.NetworkName, bp dtypes.BootstrapPeers, cfg *config.Pubsub) (service *pubsub.PubSub, err error) { +func ScoreKeeper() *dtypes.ScoreKeeper { + return new(dtypes.ScoreKeeper) +} + +func GossipSub(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, nn dtypes.NetworkName, bp dtypes.BootstrapPeers, cfg *config.Pubsub, sk *dtypes.ScoreKeeper) (service *pubsub.PubSub, err error) { bootstrappers := make(map[peer.ID]struct{}) for _, pi := range bp { bootstrappers[pi.ID] = struct{}{} @@ -161,6 +165,7 @@ func GossipSub(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, nn dtyp OpportunisticGraftThreshold: 5, }, ), + pubsub.WithPeerScoreInspect(sk.Update, 10*time.Second), } // enable Peer eXchange on bootstrappers From 7f962e19c2821601bcfaec6bfc9dd52fa717982e Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 3 Jun 2020 03:19:18 +0200 Subject: [PATCH 050/160] Fix lint Signed-off-by: Jakub Sztandera --- node/impl/common/common.go | 2 +- node/modules/dtypes/scorekeeper.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/node/impl/common/common.go b/node/impl/common/common.go index 6de137d9f..a0620c65b 100644 --- a/node/impl/common/common.go +++ b/node/impl/common/common.go @@ -62,7 +62,7 @@ func (a *CommonAPI) NetPubsubScores(context.Context) ([]api.PubsubScore, error) out := make([]api.PubsubScore, len(scores)) i := 0 for k, v := range scores { - out[i] = api.PubsubScore{k, v} + out[i] = api.PubsubScore{ID: k, Score: v} i++ } diff --git a/node/modules/dtypes/scorekeeper.go b/node/modules/dtypes/scorekeeper.go index 221821bf8..74bcb3f46 100644 --- a/node/modules/dtypes/scorekeeper.go +++ b/node/modules/dtypes/scorekeeper.go @@ -3,7 +3,7 @@ package dtypes import ( "sync" - peer "github.com/libp2p/go-libp2p-peer" + peer "github.com/libp2p/go-libp2p-core/peer" ) type ScoreKeeper struct { From 5574e4f11bd4b333e5170ba7c432556ce9e09813 Mon Sep 17 00:00:00 2001 From: acruikshank Date: Wed, 3 Jun 2020 15:38:37 -0400 Subject: [PATCH 051/160] add confidence and timeout to message wait --- api/api_full.go | 2 +- api/apistruct/struct.go | 6 ++-- build/params_shared.go | 2 ++ chain/stmgr/stmgr.go | 46 ++++++++++++++++++++++++++---- cli/multisig.go | 7 +++-- cli/paych.go | 3 +- cli/state.go | 5 ++-- cmd/lotus-fountain/main.go | 4 +-- cmd/lotus-shed/verifreg.go | 5 ++-- cmd/lotus-storage-miner/init.go | 4 +-- markets/retrievaladapter/client.go | 5 ++-- markets/storageadapter/client.go | 4 +-- markets/storageadapter/provider.go | 2 +- node/impl/full/state.go | 6 ++-- paychmgr/simple.go | 5 ++-- storage/adapter_storage_miner.go | 3 +- storage/miner.go | 2 +- storage/wdpost_run.go | 4 +-- 18 files changed, 79 insertions(+), 36 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index cf7f5b79b..ca032d81e 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -155,7 +155,7 @@ type FullNode interface { StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) StatePledgeCollateral(context.Context, types.TipSetKey) (types.BigInt, error) - StateWaitMsg(context.Context, cid.Cid) (*MsgLookup, error) + StateWaitMsg(context.Context, cid.Cid, uint64, uint64) (*MsgLookup, error) StateSearchMsg(context.Context, cid.Cid) (*MsgLookup, error) StateListMiners(context.Context, types.TipSetKey) ([]address.Address, error) StateListActors(context.Context, types.TipSetKey) ([]address.Address, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 5903f8ee0..ebd890f82 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -133,7 +133,7 @@ type FullNodeStruct struct { StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` StateReadState func(context.Context, *types.Actor, types.TipSetKey) (*api.ActorState, error) `perm:"read"` StatePledgeCollateral func(context.Context, types.TipSetKey) (types.BigInt, error) `perm:"read"` - StateWaitMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"` + StateWaitMsg func(context.Context, cid.Cid, uint64, uint64) (*api.MsgLookup, error) `perm:"read"` StateSearchMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"` StateListMiners func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` StateListActors func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` @@ -581,8 +581,8 @@ func (c *FullNodeStruct) StatePledgeCollateral(ctx context.Context, tsk types.Ti return c.Internal.StatePledgeCollateral(ctx, tsk) } -func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid) (*api.MsgLookup, error) { - return c.Internal.StateWaitMsg(ctx, msgc) +func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid, confidence uint64, timeout uint64) (*api.MsgLookup, error) { + return c.Internal.StateWaitMsg(ctx, msgc, confidence, timeout) } func (c *FullNodeStruct) StateSearchMsg(ctx context.Context, msgc cid.Cid) (*api.MsgLookup, error) { diff --git a/build/params_shared.go b/build/params_shared.go index 2b2f3e985..3411880de 100644 --- a/build/params_shared.go +++ b/build/params_shared.go @@ -59,6 +59,8 @@ var BlocksPerEpoch = uint64(builtin.ExpectedLeadersPerEpoch) // Epochs const Finality = miner.ChainFinalityish +const MessageConfidence = 5 +const MessageTimeout = 72 // constants for Weight calculation // The ratio of weight contributed by short-term vs long-term factors in a given round diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index e291fe623..d783e6ea7 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -480,7 +480,10 @@ func (sm *StateManager) GetReceipt(ctx context.Context, msg cid.Cid, ts *types.T return r, nil } -func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid) (*types.TipSet, *types.MessageReceipt, error) { +// WaitForMessage blocks until a message appears on chain. It looks backwards in the chain to see if this has already +// happened. It guarantees that the message has been on chain for at least confidence epochs without being reverted +// before returning. +func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid, confidence uint64, timeout uint64) (*types.TipSet, *types.MessageReceipt, error) { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -528,6 +531,11 @@ func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid) (*type close(backSearchWait) }() + var candidateTs *types.TipSet + var candidateRcp *types.MessageReceipt + heightOfHead := head[0].Val.Height() + reverts := map[types.TipSetKey]bool{} + for { select { case notif, ok := <-tsub: @@ -537,21 +545,49 @@ func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid) (*type for _, val := range notif { switch val.Type { case store.HCRevert: - continue + if val.Val.Equals(candidateTs) { + candidateTs = nil + candidateRcp = nil + } + if backSearchWait != nil { + reverts[val.Val.Key()] = true + } case store.HCApply: + if candidateTs != nil && val.Val.Height() >= candidateTs.Height() + abi.ChainEpoch(confidence) { + return candidateTs, candidateRcp, nil + } r, err := sm.tipsetExecutedMessage(val.Val, mcid, msg.VMMessage()) if err != nil { return nil, nil, err } if r != nil { - return val.Val, r, nil + if confidence == 0 { + return val.Val, r, err + } + candidateTs = val.Val + candidateRcp = r + } + heightOfHead = val.Val.Height() + + // check for timeout + if heightOfHead >= head[0].Val.Height() + abi.ChainEpoch(timeout) { + return nil, nil, nil } } } case <-backSearchWait: - if backTs != nil { - return backTs, backRcp, nil + // check if we found the message in the chain and that is hasn't been reverted since we started searching + if backTs != nil && !reverts[backTs.Key()] { + // if head is at or past confidence interval, return immediately + if heightOfHead >= backTs.Height() + abi.ChainEpoch(confidence) { + return backTs, backRcp, nil + } + + // wait for confidence interval + candidateTs = backTs + candidateRcp = backRcp } + reverts = nil backSearchWait = nil case <-ctx.Done(): return nil, nil, ctx.Err() diff --git a/cli/multisig.go b/cli/multisig.go index ff2ff2fce..43696736e 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "encoding/hex" "fmt" + "github.com/filecoin-project/lotus/build" "os" "sort" "strconv" @@ -117,7 +118,7 @@ var msigCreateCmd = &cli.Command{ } // wait for it to get mined into a block - wait, err := api.StateWaitMsg(ctx, msgCid) + wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence, build.MessageTimeout) if err != nil { return err } @@ -333,7 +334,7 @@ var msigProposeCmd = &cli.Command{ fmt.Println("send proposal in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid) + wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence, build.MessageTimeout) if err != nil { return err } @@ -449,7 +450,7 @@ var msigApproveCmd = &cli.Command{ fmt.Println("sent approval in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid) + wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence, build.MessageTimeout) if err != nil { return err } diff --git a/cli/paych.go b/cli/paych.go index e150d6769..e2ea67fad 100644 --- a/cli/paych.go +++ b/cli/paych.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/base64" "fmt" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/builtin/paych" @@ -361,7 +362,7 @@ var paychVoucherSubmitCmd = &cli.Command{ return err } - mwait, err := api.StateWaitMsg(ctx, mcid) + mwait, err := api.StateWaitMsg(ctx, mcid, build.MessageConfidence, build.MessageTimeout) if err != nil { return err } diff --git a/cli/state.go b/cli/state.go index 102c8d0bb..b8c319ef3 100644 --- a/cli/state.go +++ b/cli/state.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/filecoin-project/lotus/build" "reflect" "sort" "strconv" @@ -392,7 +393,7 @@ var stateReplaySetCmd = &cli.Command{ ts, err = types.NewTipSet(headers) } else { - r, err := api.StateWaitMsg(ctx, mcid) + r, err := api.StateWaitMsg(ctx, mcid, build.MessageConfidence, build.MessageTimeout) if err != nil { return xerrors.Errorf("finding message in chain: %w", err) } @@ -1156,7 +1157,7 @@ var stateWaitMsgCmd = &cli.Command{ return err } - mw, err := api.StateWaitMsg(ctx, msg) + mw, err := api.StateWaitMsg(ctx, msg, build.MessageConfidence, build.MessageTimeout) if err != nil { return err } diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go index b2275b9ca..e4c756d0b 100644 --- a/cmd/lotus-fountain/main.go +++ b/cmd/lotus-fountain/main.go @@ -334,7 +334,7 @@ func (h *handler) msgwait(w http.ResponseWriter, r *http.Request) { return } - mw, err := h.api.StateWaitMsg(r.Context(), c) + mw, err := h.api.StateWaitMsg(r.Context(), c, build.MessageConfidence, build.MessageTimeout) if err != nil { w.WriteHeader(400) w.Write([]byte(err.Error())) @@ -357,7 +357,7 @@ func (h *handler) msgwaitaddr(w http.ResponseWriter, r *http.Request) { return } - mw, err := h.api.StateWaitMsg(r.Context(), c) + mw, err := h.api.StateWaitMsg(r.Context(), c, build.MessageConfidence, build.MessageTimeout) if err != nil { w.WriteHeader(400) w.Write([]byte(err.Error())) diff --git a/cmd/lotus-shed/verifreg.go b/cmd/lotus-shed/verifreg.go index 41de8c879..a00839d69 100644 --- a/cmd/lotus-shed/verifreg.go +++ b/cmd/lotus-shed/verifreg.go @@ -3,6 +3,7 @@ package main import ( "bytes" "fmt" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/go-address" "gopkg.in/urfave/cli.v2" @@ -85,7 +86,7 @@ var verifRegAddVerifierCmd = &cli.Command{ fmt.Printf("message sent, now waiting on cid: %s\n", smsg.Cid()) - mwait, err := api.StateWaitMsg(ctx, smsg.Cid()) + mwait, err := api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence, build.MessageTimeout) if err != nil { return err } @@ -161,7 +162,7 @@ var verifRegVerifyClientCmd = &cli.Command{ fmt.Printf("message sent, now waiting on cid: %s\n", smsg.Cid()) - mwait, err := api.StateWaitMsg(ctx, smsg.Cid()) + mwait, err := api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence, build.MessageTimeout) if err != nil { return err } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index f834a09a5..a5e6f1c13 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -577,7 +577,7 @@ func configureStorageMiner(ctx context.Context, api lapi.FullNode, addr address. } log.Info("Waiting for message: ", smsg.Cid()) - ret, err := api.StateWaitMsg(ctx, smsg.Cid()) + ret, err := api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence, build.MessageTimeout) if err != nil { return err } @@ -659,7 +659,7 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, log.Infof("Pushed StorageMarket.CreateStorageMiner, %s to Mpool", signed.Cid()) log.Infof("Waiting for confirmation") - mw, err := api.StateWaitMsg(ctx, signed.Cid()) + mw, err := api.StateWaitMsg(ctx, signed.Cid(), build.MessageConfidence, build.MessageTimeout) if err != nil { return address.Undef, err } diff --git a/markets/retrievaladapter/client.go b/markets/retrievaladapter/client.go index dcd42d414..21ac3af6c 100644 --- a/markets/retrievaladapter/client.go +++ b/markets/retrievaladapter/client.go @@ -3,6 +3,7 @@ package retrievaladapter import ( "bytes" "context" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" @@ -72,7 +73,7 @@ func (rcn *retrievalClientNode) GetChainHead(ctx context.Context) (shared.TipSet // WaitForPaymentChannelAddFunds waits messageCID to appear on chain. If it doesn't appear within // defaultMsgWaitTimeout it returns error func (rcn *retrievalClientNode) WaitForPaymentChannelAddFunds(messageCID cid.Cid) error { - _, mr, err := rcn.chainapi.StateManager.WaitForMessage(context.TODO(), messageCID) + _, mr, err := rcn.chainapi.StateManager.WaitForMessage(context.TODO(), messageCID, build.MessageConfidence, build.MessageTimeout) if err != nil { return err @@ -84,7 +85,7 @@ func (rcn *retrievalClientNode) WaitForPaymentChannelAddFunds(messageCID cid.Cid } func (rcn *retrievalClientNode) WaitForPaymentChannelCreation(messageCID cid.Cid) (address.Address, error) { - _, mr, err := rcn.chainapi.StateManager.WaitForMessage(context.TODO(), messageCID) + _, mr, err := rcn.chainapi.StateManager.WaitForMessage(context.TODO(), messageCID, build.MessageConfidence, build.MessageTimeout) if err != nil { return address.Undef, err diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 754327076..64f3d6686 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -211,7 +211,7 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor } // TODO: timeout - _, ret, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage) + _, ret, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage, build.MessageConfidence, build.MessageTimeout) if err != nil { return 0, xerrors.Errorf("waiting for deal publish message: %w", err) } @@ -397,7 +397,7 @@ func (n *ClientNodeAdapter) GetChainHead(ctx context.Context) (shared.TipSetToke } func (n *ClientNodeAdapter) WaitForMessage(ctx context.Context, mcid cid.Cid, cb func(code exitcode.ExitCode, bytes []byte, err error) error) error { - receipt, err := n.StateWaitMsg(ctx, mcid) + receipt, err := n.StateWaitMsg(ctx, mcid, build.MessageConfidence, build.MessageTimeout) if err != nil { return cb(0, nil, err) } diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index ead0d99e6..67d3b9920 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -338,7 +338,7 @@ func (n *ProviderNodeAdapter) GetChainHead(ctx context.Context) (shared.TipSetTo } func (n *ProviderNodeAdapter) WaitForMessage(ctx context.Context, mcid cid.Cid, cb func(code exitcode.ExitCode, bytes []byte, err error) error) error { - receipt, err := n.StateWaitMsg(ctx, mcid) + receipt, err := n.StateWaitMsg(ctx, mcid, build.MessageConfidence, build.MessageTimeout) if err != nil { return cb(0, nil, err) } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 546434d8c..da4f3e290 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -344,10 +344,8 @@ func (a *StateAPI) MinerCreateBlock(ctx context.Context, bt *api.BlockTemplate) return &out, nil } -func (a *StateAPI) StateWaitMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error) { - // TODO: consider using event system for this, expose confidence - - ts, recpt, err := a.StateManager.WaitForMessage(ctx, msg) +func (a *StateAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64, timeout uint64) (*api.MsgLookup, error) { + ts, recpt, err := a.StateManager.WaitForMessage(ctx, msg, confidence, timeout) if err != nil { return nil, err } diff --git a/paychmgr/simple.go b/paychmgr/simple.go index ff537d425..d528a94aa 100644 --- a/paychmgr/simple.go +++ b/paychmgr/simple.go @@ -3,6 +3,7 @@ package paychmgr import ( "bytes" "context" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/specs-actors/actors/builtin" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" @@ -55,7 +56,7 @@ func (pm *Manager) createPaych(ctx context.Context, from, to address.Address, am // (tricky because we need to setup channel tracking before we know its address) func (pm *Manager) waitForPaychCreateMsg(ctx context.Context, mcid cid.Cid) { defer pm.store.lk.Unlock() - mwait, err := pm.state.StateWaitMsg(ctx, mcid) + mwait, err := pm.state.StateWaitMsg(ctx, mcid, build.MessageConfidence, build.MessageTimeout) if err != nil { log.Errorf("wait msg: %w", err) } @@ -105,7 +106,7 @@ func (pm *Manager) addFunds(ctx context.Context, ch address.Address, from addres // (tricky because we need to setup channel tracking before we know it's address) func (pm *Manager) waitForAddFundsMsg(ctx context.Context, mcid cid.Cid) { defer pm.store.lk.Unlock() - mwait, err := pm.state.StateWaitMsg(ctx, mcid) + mwait, err := pm.state.StateWaitMsg(ctx, mcid, build.MessageConfidence, build.MessageTimeout) if err != nil { log.Error(err) } diff --git a/storage/adapter_storage_miner.go b/storage/adapter_storage_miner.go index 391a38b37..76e89b748 100644 --- a/storage/adapter_storage_miner.go +++ b/storage/adapter_storage_miner.go @@ -3,6 +3,7 @@ package storage import ( "bytes" "context" + "github.com/filecoin-project/lotus/build" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -81,7 +82,7 @@ func (s SealingAPIAdapter) StateMinerDeadlines(ctx context.Context, maddr addres } func (s SealingAPIAdapter) StateWaitMsg(ctx context.Context, mcid cid.Cid) (sealing.MsgLookup, error) { - wmsg, err := s.delegate.StateWaitMsg(ctx, mcid) + wmsg, err := s.delegate.StateWaitMsg(ctx, mcid, build.MessageConfidence, build.MessageTimeout) if err != nil { return sealing.MsgLookup{}, err } diff --git a/storage/miner.go b/storage/miner.go index 779f981e9..c692db853 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -53,7 +53,7 @@ type storageMinerApi interface { StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error) StateMinerInitialPledgeCollateral(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (types.BigInt, error) - StateWaitMsg(context.Context, cid.Cid) (*api.MsgLookup, error) // TODO: removeme eventually + StateWaitMsg(context.Context, cid.Cid, uint64, uint64) (*api.MsgLookup, error) // TODO: removeme eventually StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) StateMarketStorageDeal(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 63109e5f5..63cf809ad 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -167,7 +167,7 @@ func (s *WindowPoStScheduler) checkRecoveries(ctx context.Context, deadline uint log.Warnw("declare faults recovered Message CID", "cid", sm.Cid()) - rec, err := s.api.StateWaitMsg(context.TODO(), sm.Cid()) + rec, err := s.api.StateWaitMsg(context.TODO(), sm.Cid(), build.MessageConfidence, build.MessageTimeout) if err != nil { return xerrors.Errorf("declare faults recovered wait error: %w", err) } @@ -406,7 +406,7 @@ func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.Submi log.Infof("Submitted window post: %s", sm.Cid()) go func() { - rec, err := s.api.StateWaitMsg(context.TODO(), sm.Cid()) + rec, err := s.api.StateWaitMsg(context.TODO(), sm.Cid(), build.MessageConfidence, build.MessageTimeout) if err != nil { log.Error(err) return From 57567db0174ef53c9c758c151a4415ddf702ef55 Mon Sep 17 00:00:00 2001 From: acruikshank Date: Wed, 3 Jun 2020 15:51:11 -0400 Subject: [PATCH 052/160] tidy --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index ed91f8560..dc0908832 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,6 @@ github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1 github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA= github.com/filecoin-project/go-fil-markets v0.2.7 h1:bgdK/e+xW15aVZLtdFLzAHdrx1hqtGF9veg2lstLK6o= github.com/filecoin-project/go-fil-markets v0.2.7/go.mod h1:LI3VFHse33aU0djAmFQ8+Hg39i0J8ibAoppGu6TbgkA= -github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200520183639-7c6ee2e066b4 h1:H8AVYu0MV9m3CSnKMxeILMfh8xJtnqVdXfBF/qbzgu0= -github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200520183639-7c6ee2e066b4/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM= github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs= From 04f0130530baa6c2a237fc3397041f4fd3eda6d2 Mon Sep 17 00:00:00 2001 From: acruikshank Date: Wed, 3 Jun 2020 17:42:06 -0400 Subject: [PATCH 053/160] remove timeout --- api/api_full.go | 2 +- api/apistruct/struct.go | 6 +++--- build/params_shared.go | 1 - chain/stmgr/stmgr.go | 7 +------ cli/multisig.go | 6 +++--- cli/paych.go | 2 +- cli/state.go | 4 ++-- cmd/lotus-fountain/main.go | 4 ++-- cmd/lotus-shed/verifreg.go | 4 ++-- cmd/lotus-storage-miner/init.go | 4 ++-- markets/retrievaladapter/client.go | 4 ++-- markets/storageadapter/client.go | 4 ++-- markets/storageadapter/provider.go | 2 +- node/impl/full/state.go | 4 ++-- paychmgr/simple.go | 4 ++-- storage/adapter_storage_miner.go | 2 +- storage/miner.go | 2 +- storage/wdpost_run.go | 4 ++-- 18 files changed, 30 insertions(+), 36 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index ca032d81e..4b091caa1 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -155,7 +155,7 @@ type FullNode interface { StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) StatePledgeCollateral(context.Context, types.TipSetKey) (types.BigInt, error) - StateWaitMsg(context.Context, cid.Cid, uint64, uint64) (*MsgLookup, error) + StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*MsgLookup, error) StateSearchMsg(context.Context, cid.Cid) (*MsgLookup, error) StateListMiners(context.Context, types.TipSetKey) ([]address.Address, error) StateListActors(context.Context, types.TipSetKey) ([]address.Address, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index ebd890f82..93a5ca8cf 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -133,7 +133,7 @@ type FullNodeStruct struct { StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` StateReadState func(context.Context, *types.Actor, types.TipSetKey) (*api.ActorState, error) `perm:"read"` StatePledgeCollateral func(context.Context, types.TipSetKey) (types.BigInt, error) `perm:"read"` - StateWaitMsg func(context.Context, cid.Cid, uint64, uint64) (*api.MsgLookup, error) `perm:"read"` + StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"` StateSearchMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"` StateListMiners func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` StateListActors func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` @@ -581,8 +581,8 @@ func (c *FullNodeStruct) StatePledgeCollateral(ctx context.Context, tsk types.Ti return c.Internal.StatePledgeCollateral(ctx, tsk) } -func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid, confidence uint64, timeout uint64) (*api.MsgLookup, error) { - return c.Internal.StateWaitMsg(ctx, msgc, confidence, timeout) +func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid, confidence uint64) (*api.MsgLookup, error) { + return c.Internal.StateWaitMsg(ctx, msgc, confidence) } func (c *FullNodeStruct) StateSearchMsg(ctx context.Context, msgc cid.Cid) (*api.MsgLookup, error) { diff --git a/build/params_shared.go b/build/params_shared.go index 3411880de..97ca46986 100644 --- a/build/params_shared.go +++ b/build/params_shared.go @@ -60,7 +60,6 @@ var BlocksPerEpoch = uint64(builtin.ExpectedLeadersPerEpoch) // Epochs const Finality = miner.ChainFinalityish const MessageConfidence = 5 -const MessageTimeout = 72 // constants for Weight calculation // The ratio of weight contributed by short-term vs long-term factors in a given round diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index d783e6ea7..b4f27bea2 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -483,7 +483,7 @@ func (sm *StateManager) GetReceipt(ctx context.Context, msg cid.Cid, ts *types.T // WaitForMessage blocks until a message appears on chain. It looks backwards in the chain to see if this has already // happened. It guarantees that the message has been on chain for at least confidence epochs without being reverted // before returning. -func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid, confidence uint64, timeout uint64) (*types.TipSet, *types.MessageReceipt, error) { +func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid, confidence uint64) (*types.TipSet, *types.MessageReceipt, error) { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -568,11 +568,6 @@ func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid, confid candidateRcp = r } heightOfHead = val.Val.Height() - - // check for timeout - if heightOfHead >= head[0].Val.Height() + abi.ChainEpoch(timeout) { - return nil, nil, nil - } } } case <-backSearchWait: diff --git a/cli/multisig.go b/cli/multisig.go index 43696736e..4d6d6603d 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -118,7 +118,7 @@ var msigCreateCmd = &cli.Command{ } // wait for it to get mined into a block - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence, build.MessageTimeout) + wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) if err != nil { return err } @@ -334,7 +334,7 @@ var msigProposeCmd = &cli.Command{ fmt.Println("send proposal in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence, build.MessageTimeout) + wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) if err != nil { return err } @@ -450,7 +450,7 @@ var msigApproveCmd = &cli.Command{ fmt.Println("sent approval in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence, build.MessageTimeout) + wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) if err != nil { return err } diff --git a/cli/paych.go b/cli/paych.go index e2ea67fad..a1998fff1 100644 --- a/cli/paych.go +++ b/cli/paych.go @@ -362,7 +362,7 @@ var paychVoucherSubmitCmd = &cli.Command{ return err } - mwait, err := api.StateWaitMsg(ctx, mcid, build.MessageConfidence, build.MessageTimeout) + mwait, err := api.StateWaitMsg(ctx, mcid, build.MessageConfidence) if err != nil { return err } diff --git a/cli/state.go b/cli/state.go index b8c319ef3..9b7021520 100644 --- a/cli/state.go +++ b/cli/state.go @@ -393,7 +393,7 @@ var stateReplaySetCmd = &cli.Command{ ts, err = types.NewTipSet(headers) } else { - r, err := api.StateWaitMsg(ctx, mcid, build.MessageConfidence, build.MessageTimeout) + r, err := api.StateWaitMsg(ctx, mcid, build.MessageConfidence) if err != nil { return xerrors.Errorf("finding message in chain: %w", err) } @@ -1157,7 +1157,7 @@ var stateWaitMsgCmd = &cli.Command{ return err } - mw, err := api.StateWaitMsg(ctx, msg, build.MessageConfidence, build.MessageTimeout) + mw, err := api.StateWaitMsg(ctx, msg, build.MessageConfidence) if err != nil { return err } diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go index e4c756d0b..593dc2e1d 100644 --- a/cmd/lotus-fountain/main.go +++ b/cmd/lotus-fountain/main.go @@ -334,7 +334,7 @@ func (h *handler) msgwait(w http.ResponseWriter, r *http.Request) { return } - mw, err := h.api.StateWaitMsg(r.Context(), c, build.MessageConfidence, build.MessageTimeout) + mw, err := h.api.StateWaitMsg(r.Context(), c, build.MessageConfidence) if err != nil { w.WriteHeader(400) w.Write([]byte(err.Error())) @@ -357,7 +357,7 @@ func (h *handler) msgwaitaddr(w http.ResponseWriter, r *http.Request) { return } - mw, err := h.api.StateWaitMsg(r.Context(), c, build.MessageConfidence, build.MessageTimeout) + mw, err := h.api.StateWaitMsg(r.Context(), c, build.MessageConfidence) if err != nil { w.WriteHeader(400) w.Write([]byte(err.Error())) diff --git a/cmd/lotus-shed/verifreg.go b/cmd/lotus-shed/verifreg.go index a00839d69..e375abe73 100644 --- a/cmd/lotus-shed/verifreg.go +++ b/cmd/lotus-shed/verifreg.go @@ -86,7 +86,7 @@ var verifRegAddVerifierCmd = &cli.Command{ fmt.Printf("message sent, now waiting on cid: %s\n", smsg.Cid()) - mwait, err := api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence, build.MessageTimeout) + mwait, err := api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence) if err != nil { return err } @@ -162,7 +162,7 @@ var verifRegVerifyClientCmd = &cli.Command{ fmt.Printf("message sent, now waiting on cid: %s\n", smsg.Cid()) - mwait, err := api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence, build.MessageTimeout) + mwait, err := api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence) if err != nil { return err } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index a5e6f1c13..fe3c1e0dc 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -577,7 +577,7 @@ func configureStorageMiner(ctx context.Context, api lapi.FullNode, addr address. } log.Info("Waiting for message: ", smsg.Cid()) - ret, err := api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence, build.MessageTimeout) + ret, err := api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence) if err != nil { return err } @@ -659,7 +659,7 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, log.Infof("Pushed StorageMarket.CreateStorageMiner, %s to Mpool", signed.Cid()) log.Infof("Waiting for confirmation") - mw, err := api.StateWaitMsg(ctx, signed.Cid(), build.MessageConfidence, build.MessageTimeout) + mw, err := api.StateWaitMsg(ctx, signed.Cid(), build.MessageConfidence) if err != nil { return address.Undef, err } diff --git a/markets/retrievaladapter/client.go b/markets/retrievaladapter/client.go index 21ac3af6c..0c1992236 100644 --- a/markets/retrievaladapter/client.go +++ b/markets/retrievaladapter/client.go @@ -73,7 +73,7 @@ func (rcn *retrievalClientNode) GetChainHead(ctx context.Context) (shared.TipSet // WaitForPaymentChannelAddFunds waits messageCID to appear on chain. If it doesn't appear within // defaultMsgWaitTimeout it returns error func (rcn *retrievalClientNode) WaitForPaymentChannelAddFunds(messageCID cid.Cid) error { - _, mr, err := rcn.chainapi.StateManager.WaitForMessage(context.TODO(), messageCID, build.MessageConfidence, build.MessageTimeout) + _, mr, err := rcn.chainapi.StateManager.WaitForMessage(context.TODO(), messageCID, build.MessageConfidence) if err != nil { return err @@ -85,7 +85,7 @@ func (rcn *retrievalClientNode) WaitForPaymentChannelAddFunds(messageCID cid.Cid } func (rcn *retrievalClientNode) WaitForPaymentChannelCreation(messageCID cid.Cid) (address.Address, error) { - _, mr, err := rcn.chainapi.StateManager.WaitForMessage(context.TODO(), messageCID, build.MessageConfidence, build.MessageTimeout) + _, mr, err := rcn.chainapi.StateManager.WaitForMessage(context.TODO(), messageCID, build.MessageConfidence) if err != nil { return address.Undef, err diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 64f3d6686..a0c7899e6 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -211,7 +211,7 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor } // TODO: timeout - _, ret, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage, build.MessageConfidence, build.MessageTimeout) + _, ret, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage, build.MessageConfidence) if err != nil { return 0, xerrors.Errorf("waiting for deal publish message: %w", err) } @@ -397,7 +397,7 @@ func (n *ClientNodeAdapter) GetChainHead(ctx context.Context) (shared.TipSetToke } func (n *ClientNodeAdapter) WaitForMessage(ctx context.Context, mcid cid.Cid, cb func(code exitcode.ExitCode, bytes []byte, err error) error) error { - receipt, err := n.StateWaitMsg(ctx, mcid, build.MessageConfidence, build.MessageTimeout) + receipt, err := n.StateWaitMsg(ctx, mcid, build.MessageConfidence) if err != nil { return cb(0, nil, err) } diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 67d3b9920..560fbf202 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -338,7 +338,7 @@ func (n *ProviderNodeAdapter) GetChainHead(ctx context.Context) (shared.TipSetTo } func (n *ProviderNodeAdapter) WaitForMessage(ctx context.Context, mcid cid.Cid, cb func(code exitcode.ExitCode, bytes []byte, err error) error) error { - receipt, err := n.StateWaitMsg(ctx, mcid, build.MessageConfidence, build.MessageTimeout) + receipt, err := n.StateWaitMsg(ctx, mcid, build.MessageConfidence) if err != nil { return cb(0, nil, err) } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index da4f3e290..da8f90234 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -344,8 +344,8 @@ func (a *StateAPI) MinerCreateBlock(ctx context.Context, bt *api.BlockTemplate) return &out, nil } -func (a *StateAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64, timeout uint64) (*api.MsgLookup, error) { - ts, recpt, err := a.StateManager.WaitForMessage(ctx, msg, confidence, timeout) +func (a *StateAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { + ts, recpt, err := a.StateManager.WaitForMessage(ctx, msg, confidence) if err != nil { return nil, err } diff --git a/paychmgr/simple.go b/paychmgr/simple.go index d528a94aa..209217c0c 100644 --- a/paychmgr/simple.go +++ b/paychmgr/simple.go @@ -56,7 +56,7 @@ func (pm *Manager) createPaych(ctx context.Context, from, to address.Address, am // (tricky because we need to setup channel tracking before we know its address) func (pm *Manager) waitForPaychCreateMsg(ctx context.Context, mcid cid.Cid) { defer pm.store.lk.Unlock() - mwait, err := pm.state.StateWaitMsg(ctx, mcid, build.MessageConfidence, build.MessageTimeout) + mwait, err := pm.state.StateWaitMsg(ctx, mcid, build.MessageConfidence) if err != nil { log.Errorf("wait msg: %w", err) } @@ -106,7 +106,7 @@ func (pm *Manager) addFunds(ctx context.Context, ch address.Address, from addres // (tricky because we need to setup channel tracking before we know it's address) func (pm *Manager) waitForAddFundsMsg(ctx context.Context, mcid cid.Cid) { defer pm.store.lk.Unlock() - mwait, err := pm.state.StateWaitMsg(ctx, mcid, build.MessageConfidence, build.MessageTimeout) + mwait, err := pm.state.StateWaitMsg(ctx, mcid, build.MessageConfidence) if err != nil { log.Error(err) } diff --git a/storage/adapter_storage_miner.go b/storage/adapter_storage_miner.go index 76e89b748..8d9bf7774 100644 --- a/storage/adapter_storage_miner.go +++ b/storage/adapter_storage_miner.go @@ -82,7 +82,7 @@ func (s SealingAPIAdapter) StateMinerDeadlines(ctx context.Context, maddr addres } func (s SealingAPIAdapter) StateWaitMsg(ctx context.Context, mcid cid.Cid) (sealing.MsgLookup, error) { - wmsg, err := s.delegate.StateWaitMsg(ctx, mcid, build.MessageConfidence, build.MessageTimeout) + wmsg, err := s.delegate.StateWaitMsg(ctx, mcid, build.MessageConfidence) if err != nil { return sealing.MsgLookup{}, err } diff --git a/storage/miner.go b/storage/miner.go index c692db853..961e1ab8f 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -53,7 +53,7 @@ type storageMinerApi interface { StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error) StateMinerInitialPledgeCollateral(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (types.BigInt, error) - StateWaitMsg(context.Context, cid.Cid, uint64, uint64) (*api.MsgLookup, error) // TODO: removeme eventually + StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) // TODO: removeme eventually StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) StateMarketStorageDeal(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 63cf809ad..b7004ea9f 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -167,7 +167,7 @@ func (s *WindowPoStScheduler) checkRecoveries(ctx context.Context, deadline uint log.Warnw("declare faults recovered Message CID", "cid", sm.Cid()) - rec, err := s.api.StateWaitMsg(context.TODO(), sm.Cid(), build.MessageConfidence, build.MessageTimeout) + rec, err := s.api.StateWaitMsg(context.TODO(), sm.Cid(), build.MessageConfidence) if err != nil { return xerrors.Errorf("declare faults recovered wait error: %w", err) } @@ -406,7 +406,7 @@ func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.Submi log.Infof("Submitted window post: %s", sm.Cid()) go func() { - rec, err := s.api.StateWaitMsg(context.TODO(), sm.Cid(), build.MessageConfidence, build.MessageTimeout) + rec, err := s.api.StateWaitMsg(context.TODO(), sm.Cid(), build.MessageConfidence) if err != nil { log.Error(err) return From 29dbc26dbd948a40611541a3868c360d9baa6459 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 2 Jun 2020 20:12:53 +0200 Subject: [PATCH 054/160] Update cli Signed-off-by: Jakub Sztandera --- cli/auth.go | 2 +- cli/chain.go | 2 +- cli/client.go | 2 +- cli/cmd.go | 2 +- cli/log.go | 2 +- cli/mpool.go | 2 +- cli/multisig.go | 2 +- cli/net.go | 2 +- cli/params.go | 2 +- cli/paych.go | 2 +- cli/send.go | 2 +- cli/state.go | 2 +- cli/sync.go | 2 +- cli/version.go | 2 +- cli/wait.go | 2 +- cli/wallet.go | 2 +- cmd/chain-noise/main.go | 2 +- cmd/lotus-bench/import.go | 2 +- cmd/lotus-bench/main.go | 2 +- cmd/lotus-chainwatch/dot.go | 2 +- cmd/lotus-chainwatch/main.go | 2 +- cmd/lotus-fountain/main.go | 2 +- cmd/lotus-health/main.go | 2 +- cmd/lotus-seal-worker/main.go | 2 +- cmd/lotus-seed/genesis.go | 2 +- cmd/lotus-seed/main.go | 2 +- cmd/lotus-shed/base16.go | 2 +- cmd/lotus-shed/base32.go | 2 +- cmd/lotus-shed/bigint.go | 2 +- cmd/lotus-shed/bitfield.go | 2 +- cmd/lotus-shed/commp.go | 2 +- cmd/lotus-shed/import-car.go | 2 +- cmd/lotus-shed/keyinfo.go | 2 +- cmd/lotus-shed/main.go | 2 +- cmd/lotus-shed/nonce-fix.go | 2 +- cmd/lotus-shed/params.go | 2 +- cmd/lotus-shed/peerkey.go | 2 +- cmd/lotus-shed/proofs.go | 2 +- cmd/lotus-shed/stateroot-stats.go | 2 +- cmd/lotus-shed/verifreg.go | 2 +- cmd/lotus-storage-miner/info.go | 2 +- cmd/lotus-storage-miner/init.go | 2 +- cmd/lotus-storage-miner/main.go | 10 +++++----- cmd/lotus-storage-miner/market.go | 2 +- cmd/lotus-storage-miner/proving.go | 2 +- cmd/lotus-storage-miner/rewards.go | 2 +- cmd/lotus-storage-miner/run.go | 2 +- cmd/lotus-storage-miner/sectors.go | 2 +- cmd/lotus-storage-miner/storage.go | 2 +- cmd/lotus-storage-miner/workers.go | 2 +- cmd/lotus/daemon.go | 2 +- cmd/lotus/daemon_nodaemon.go | 2 +- cmd/lotus/debug_advance.go | 2 +- cmd/lotus/main.go | 10 +++++----- go.mod | 2 +- go.sum | 2 -- lotuspond/main.go | 2 +- 57 files changed, 64 insertions(+), 66 deletions(-) diff --git a/cli/auth.go b/cli/auth.go index a40e0a328..e3497f17c 100644 --- a/cli/auth.go +++ b/cli/auth.go @@ -4,7 +4,7 @@ import ( "fmt" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/go-jsonrpc/auth" diff --git a/cli/chain.go b/cli/chain.go index 9e245c875..4d296cf78 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -25,7 +25,7 @@ import ( cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" diff --git a/cli/client.go b/cli/client.go index 3bfc5de70..ea6241b9b 100644 --- a/cli/client.go +++ b/cli/client.go @@ -10,7 +10,7 @@ import ( "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/storagemarket" diff --git a/cli/cmd.go b/cli/cmd.go index fd0ab1e78..3953f1372 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -14,7 +14,7 @@ import ( "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/go-jsonrpc" diff --git a/cli/log.go b/cli/log.go index 561d949d5..3e1027ccf 100644 --- a/cli/log.go +++ b/cli/log.go @@ -4,7 +4,7 @@ import ( "fmt" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" ) var logCmd = &cli.Command{ diff --git a/cli/mpool.go b/cli/mpool.go index 0c5102d01..1b8a1e0d1 100644 --- a/cli/mpool.go +++ b/cli/mpool.go @@ -6,7 +6,7 @@ import ( "sort" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/go-address" diff --git a/cli/multisig.go b/cli/multisig.go index 37106dff5..19d1aa420 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -19,7 +19,7 @@ import ( cbor "github.com/ipfs/go-ipld-cbor" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apibstore" diff --git a/cli/net.go b/cli/net.go index b49d9604f..96185ad07 100644 --- a/cli/net.go +++ b/cli/net.go @@ -7,7 +7,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/lib/addrutil" ) diff --git a/cli/params.go b/cli/params.go index f6acbaad7..e1ad3e5d9 100644 --- a/cli/params.go +++ b/cli/params.go @@ -4,7 +4,7 @@ import ( "github.com/docker/go-units" paramfetch "github.com/filecoin-project/go-paramfetch" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/build" ) diff --git a/cli/paych.go b/cli/paych.go index 5fe5fac0c..597f80837 100644 --- a/cli/paych.go +++ b/cli/paych.go @@ -7,7 +7,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/builtin/paych" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" types "github.com/filecoin-project/lotus/chain/types" ) diff --git a/cli/send.go b/cli/send.go index f43e90c36..9f9c70dde 100644 --- a/cli/send.go +++ b/cli/send.go @@ -5,7 +5,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" ) var sendCmd = &cli.Command{ diff --git a/cli/state.go b/cli/state.go index 102c8d0bb..9e97037ad 100644 --- a/cli/state.go +++ b/cli/state.go @@ -17,7 +17,7 @@ import ( "github.com/multiformats/go-multihash" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi" diff --git a/cli/sync.go b/cli/sync.go index 891f61da3..2a062cbcd 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -7,7 +7,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/abi" cid "github.com/ipfs/go-cid" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" diff --git a/cli/version.go b/cli/version.go index 50bc0d259..d257f5ba9 100644 --- a/cli/version.go +++ b/cli/version.go @@ -3,7 +3,7 @@ package cli import ( "fmt" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" ) var versionCmd = &cli.Command{ diff --git a/cli/wait.go b/cli/wait.go index ce4c4dccb..ca8cdce3f 100644 --- a/cli/wait.go +++ b/cli/wait.go @@ -4,7 +4,7 @@ import ( "fmt" "time" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" ) var waitApiCmd = &cli.Command{ diff --git a/cli/wallet.go b/cli/wallet.go index 688a2829f..a731c1b2c 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -15,7 +15,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/crypto" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" ) var walletCmd = &cli.Command{ diff --git a/cmd/chain-noise/main.go b/cmd/chain-noise/main.go index e152ccf75..a1e92ee96 100644 --- a/cmd/chain-noise/main.go +++ b/cmd/chain-noise/main.go @@ -14,7 +14,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" ) func main() { diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index f33cc1d9d..11d58d9b8 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -24,7 +24,7 @@ import ( "github.com/ipfs/go-datastore" badger "github.com/ipfs/go-ds-badger2" blockstore "github.com/ipfs/go-ipfs-blockstore" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" ) type TipSetExec struct { diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 4ac52b497..5e34c36b0 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -16,7 +16,7 @@ import ( "github.com/minio/blake2b-simd" "github.com/mitchellh/go-homedir" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/go-address" paramfetch "github.com/filecoin-project/go-paramfetch" diff --git a/cmd/lotus-chainwatch/dot.go b/cmd/lotus-chainwatch/dot.go index 22c8b7f05..87dab62ad 100644 --- a/cmd/lotus-chainwatch/dot.go +++ b/cmd/lotus-chainwatch/dot.go @@ -6,7 +6,7 @@ import ( "strconv" "github.com/ipfs/go-cid" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" ) var dotCmd = &cli.Command{ diff --git a/cmd/lotus-chainwatch/main.go b/cmd/lotus-chainwatch/main.go index faedb3bea..b5ca379e0 100644 --- a/cmd/lotus-chainwatch/main.go +++ b/cmd/lotus-chainwatch/main.go @@ -8,7 +8,7 @@ import ( logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go index d4e41895b..050b64c87 100644 --- a/cmd/lotus-fountain/main.go +++ b/cmd/lotus-fountain/main.go @@ -15,7 +15,7 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/abi" diff --git a/cmd/lotus-health/main.go b/cmd/lotus-health/main.go index 264b3b84e..9860b5b7c 100644 --- a/cmd/lotus-health/main.go +++ b/cmd/lotus-health/main.go @@ -10,7 +10,7 @@ import ( cid "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/go-jsonrpc" diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index e9721e9dd..5ca1483c1 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -13,7 +13,7 @@ import ( "github.com/gorilla/mux" logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/go-jsonrpc" "github.com/filecoin-project/go-jsonrpc/auth" diff --git a/cmd/lotus-seed/genesis.go b/cmd/lotus-seed/genesis.go index 3dc6e15d2..471fb99df 100644 --- a/cmd/lotus-seed/genesis.go +++ b/cmd/lotus-seed/genesis.go @@ -7,7 +7,7 @@ import ( "github.com/google/uuid" "github.com/mitchellh/go-homedir" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi/big" diff --git a/cmd/lotus-seed/main.go b/cmd/lotus-seed/main.go index 47dd850c0..1e4f8fe0d 100644 --- a/cmd/lotus-seed/main.go +++ b/cmd/lotus-seed/main.go @@ -12,7 +12,7 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi" diff --git a/cmd/lotus-shed/base16.go b/cmd/lotus-shed/base16.go index e1b5a6502..adfdfeddb 100644 --- a/cmd/lotus-shed/base16.go +++ b/cmd/lotus-shed/base16.go @@ -8,7 +8,7 @@ import ( "os" "strings" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" ) var base16Cmd = &cli.Command{ diff --git a/cmd/lotus-shed/base32.go b/cmd/lotus-shed/base32.go index ff7fab858..cd30a2146 100644 --- a/cmd/lotus-shed/base32.go +++ b/cmd/lotus-shed/base32.go @@ -7,7 +7,7 @@ import ( "os" "strings" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/multiformats/go-base32" ) diff --git a/cmd/lotus-shed/bigint.go b/cmd/lotus-shed/bigint.go index 3db0de68d..e087a341a 100644 --- a/cmd/lotus-shed/bigint.go +++ b/cmd/lotus-shed/bigint.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/filecoin-project/lotus/chain/types" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" ) var bigIntParseCmd = &cli.Command{ diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index 96116b3b3..0cf9e4ea0 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -8,7 +8,7 @@ import ( "os" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/go-bitfield" rlepluslazy "github.com/filecoin-project/go-bitfield/rle" diff --git a/cmd/lotus-shed/commp.go b/cmd/lotus-shed/commp.go index acc1e73e6..9b0cab75d 100644 --- a/cmd/lotus-shed/commp.go +++ b/cmd/lotus-shed/commp.go @@ -5,7 +5,7 @@ import ( "fmt" commcid "github.com/filecoin-project/go-fil-commcid" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" ) var commpToCidCmd = &cli.Command{ diff --git a/cmd/lotus-shed/import-car.go b/cmd/lotus-shed/import-car.go index 970341a3a..091ab9054 100644 --- a/cmd/lotus-shed/import-car.go +++ b/cmd/lotus-shed/import-car.go @@ -8,7 +8,7 @@ import ( blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipld/go-car" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/node/repo" ) diff --git a/cmd/lotus-shed/keyinfo.go b/cmd/lotus-shed/keyinfo.go index f1a42894a..42ab91601 100644 --- a/cmd/lotus-shed/keyinfo.go +++ b/cmd/lotus-shed/keyinfo.go @@ -9,7 +9,7 @@ import ( "strings" "text/template" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index e7264c92c..09cffac5d 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -4,7 +4,7 @@ import ( "os" logging "github.com/ipfs/go-log/v2" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/build" ) diff --git a/cmd/lotus-shed/nonce-fix.go b/cmd/lotus-shed/nonce-fix.go index aaecdcf58..4fb30300a 100644 --- a/cmd/lotus-shed/nonce-fix.go +++ b/cmd/lotus-shed/nonce-fix.go @@ -5,7 +5,7 @@ import ( "math" "github.com/filecoin-project/go-address" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" diff --git a/cmd/lotus-shed/params.go b/cmd/lotus-shed/params.go index 210711b91..a81110b12 100644 --- a/cmd/lotus-shed/params.go +++ b/cmd/lotus-shed/params.go @@ -4,7 +4,7 @@ import ( "github.com/docker/go-units" paramfetch "github.com/filecoin-project/go-paramfetch" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/build" ) diff --git a/cmd/lotus-shed/peerkey.go b/cmd/lotus-shed/peerkey.go index 65220209d..6d9ee99b6 100644 --- a/cmd/lotus-shed/peerkey.go +++ b/cmd/lotus-shed/peerkey.go @@ -6,7 +6,7 @@ import ( "os" "strings" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/lp2p" diff --git a/cmd/lotus-shed/proofs.go b/cmd/lotus-shed/proofs.go index 9f86d9fa7..3a4615658 100644 --- a/cmd/lotus-shed/proofs.go +++ b/cmd/lotus-shed/proofs.go @@ -4,7 +4,7 @@ import ( "encoding/hex" "fmt" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" ffi "github.com/filecoin-project/filecoin-ffi" "github.com/filecoin-project/go-address" diff --git a/cmd/lotus-shed/stateroot-stats.go b/cmd/lotus-shed/stateroot-stats.go index 89bd0bb47..0546e5315 100644 --- a/cmd/lotus-shed/stateroot-stats.go +++ b/cmd/lotus-shed/stateroot-stats.go @@ -3,7 +3,7 @@ package main import ( "fmt" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/ipfs/go-cid" diff --git a/cmd/lotus-shed/verifreg.go b/cmd/lotus-shed/verifreg.go index 41de8c879..44adc7297 100644 --- a/cmd/lotus-shed/verifreg.go +++ b/cmd/lotus-shed/verifreg.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/filecoin-project/go-address" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 0257be850..264128cee 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -8,7 +8,7 @@ import ( "github.com/fatih/color" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/specs-actors/actors/builtin/miner" sealing "github.com/filecoin-project/storage-fsm" diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index f1677801b..e63e983b9 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -19,7 +19,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/mitchellh/go-homedir" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index e3e908fd1..10ffb85ad 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -4,8 +4,8 @@ import ( "os" logging "github.com/ipfs/go-log/v2" + "github.com/urfave/cli/v2" "go.opencensus.io/trace" - "gopkg.in/urfave/cli.v2" "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" @@ -55,10 +55,10 @@ func main() { } app := &cli.App{ - Name: "lotus-storage-miner", - Usage: "Filecoin decentralized storage network storage miner", - Version: build.UserVersion(), - EnableShellCompletion: true, + Name: "lotus-storage-miner", + Usage: "Filecoin decentralized storage network storage miner", + Version: build.UserVersion(), + EnableBashCompletion: true, Flags: []cli.Flag{ &cli.StringFlag{ Name: "repo", diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index b918937db..68b626360 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -7,7 +7,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" "github.com/ipfs/go-cid" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" ) var setPriceCmd = &cli.Command{ diff --git a/cmd/lotus-storage-miner/proving.go b/cmd/lotus-storage-miner/proving.go index 7cfa010b9..9f9ca600a 100644 --- a/cmd/lotus-storage-miner/proving.go +++ b/cmd/lotus-storage-miner/proving.go @@ -10,8 +10,8 @@ import ( lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/urfave/cli/v2" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" ) var provingCmd = &cli.Command{ diff --git a/cmd/lotus-storage-miner/rewards.go b/cmd/lotus-storage-miner/rewards.go index 68b7ab9d9..4152880ae 100644 --- a/cmd/lotus-storage-miner/rewards.go +++ b/cmd/lotus-storage-miner/rewards.go @@ -3,7 +3,7 @@ package main import ( "fmt" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 467033f9c..f3c7fa8ef 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -12,7 +12,7 @@ import ( "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/go-jsonrpc" "github.com/filecoin-project/go-jsonrpc/auth" diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index d923089ec..1a500cc8b 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -9,7 +9,7 @@ import ( "time" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/specs-actors/actors/abi" diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index 40205531a..a8ceda6c2 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -15,7 +15,7 @@ import ( "github.com/google/uuid" "github.com/mitchellh/go-homedir" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi" diff --git a/cmd/lotus-storage-miner/workers.go b/cmd/lotus-storage-miner/workers.go index 8114b74e6..009d2fb90 100644 --- a/cmd/lotus-storage-miner/workers.go +++ b/cmd/lotus-storage-miner/workers.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/fatih/color" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/sector-storage/storiface" diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 5aa3fe33d..4d75bdd84 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -22,7 +22,7 @@ import ( "go.opencensus.io/stats/view" "go.opencensus.io/tag" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" diff --git a/cmd/lotus/daemon_nodaemon.go b/cmd/lotus/daemon_nodaemon.go index 05a8f97da..a11d92c6c 100644 --- a/cmd/lotus/daemon_nodaemon.go +++ b/cmd/lotus/daemon_nodaemon.go @@ -5,7 +5,7 @@ package main import ( "errors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" ) // DaemonCmd is the `go-lotus daemon` command diff --git a/cmd/lotus/debug_advance.go b/cmd/lotus/debug_advance.go index 990bbe95d..ae4064982 100644 --- a/cmd/lotus/debug_advance.go +++ b/cmd/lotus/debug_advance.go @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/crypto" "golang.org/x/xerrors" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" ) func init() { diff --git a/cmd/lotus/main.go b/cmd/lotus/main.go index 4bc13d6fe..5376ce02a 100644 --- a/cmd/lotus/main.go +++ b/cmd/lotus/main.go @@ -4,8 +4,8 @@ import ( "context" "os" + "github.com/urfave/cli/v2" "go.opencensus.io/trace" - "gopkg.in/urfave/cli.v2" "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" @@ -50,10 +50,10 @@ func main() { defer span.End() app := &cli.App{ - Name: "lotus", - Usage: "Filecoin decentralized storage network client", - Version: build.UserVersion(), - EnableShellCompletion: true, + Name: "lotus", + Usage: "Filecoin decentralized storage network client", + Version: build.UserVersion(), + EnableBashCompletion: true, Flags: []cli.Flag{ &cli.StringFlag{ Name: "repo", diff --git a/go.mod b/go.mod index 56309be9e..ba737b67b 100644 --- a/go.mod +++ b/go.mod @@ -104,6 +104,7 @@ require ( github.com/stretchr/objx v0.2.0 // indirect github.com/stretchr/testify v1.5.1 github.com/syndtr/goleveldb v1.0.0 + github.com/urfave/cli/v2 v2.2.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba github.com/whyrusleeping/cbor-gen v0.0.0-20200504204219-64967432584d github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 @@ -117,7 +118,6 @@ require ( golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 - gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect ) diff --git a/go.sum b/go.sum index cb372e94d..c234fadc3 100644 --- a/go.sum +++ b/go.sum @@ -1456,8 +1456,6 @@ gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVY gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 h1:Ggy3mWN4l3PUFPfSG0YB3n5fVYggzysUmiUQ89SnX6Y= -gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8/go.mod h1:cKXr3E0k4aosgycml1b5z33BVV6hai1Kh7uDgFOkbcs= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/lotuspond/main.go b/lotuspond/main.go index 8551a25dc..fee36d86a 100644 --- a/lotuspond/main.go +++ b/lotuspond/main.go @@ -8,7 +8,7 @@ import ( "path" "strconv" - "gopkg.in/urfave/cli.v2" + "github.com/urfave/cli/v2" "github.com/filecoin-project/go-jsonrpc" ) From cd817906b200583a12da09fc5a3709db82433ef6 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 3 Jun 2020 18:30:43 -0700 Subject: [PATCH 055/160] handle sourcing randomness from a future block --- chain/store/store.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chain/store/store.go b/chain/store/store.go index c9dac2e31..72c77ee36 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -914,6 +914,10 @@ func (cs *ChainStore) GetRandomness(ctx context.Context, blks []cid.Cid, pers cr return nil, err } + if round > ts.Height() { + return DrawRandomness(ts.MinTicket().VRFProof, pers, round, entropy) + } + searchHeight := round if searchHeight < 0 { searchHeight = 0 From 5f95a92135a4557de791d8a07b83f438d8d9dbab Mon Sep 17 00:00:00 2001 From: acruikshank Date: Wed, 3 Jun 2020 22:18:38 -0400 Subject: [PATCH 056/160] fix race (sort of) by increasing confidence --- markets/storageadapter/client.go | 2 +- markets/storageadapter/provider.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index a0c7899e6..3f13978c2 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -321,7 +321,7 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider } } - if err := c.ev.Called(checkFunc, called, revert, 3, build.SealRandomnessLookbackLimit, matchEvent); err != nil { + if err := c.ev.Called(checkFunc, called, revert, build.MessageConfidence + 1, build.SealRandomnessLookbackLimit, matchEvent); err != nil { return xerrors.Errorf("failed to set up called handler: %w", err) } diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 560fbf202..7c7310cd9 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -321,7 +321,7 @@ func (n *ProviderNodeAdapter) OnDealSectorCommitted(ctx context.Context, provide } - if err := n.ev.Called(checkFunc, called, revert, 3, build.SealRandomnessLookbackLimit, matchEvent); err != nil { + if err := n.ev.Called(checkFunc, called, revert, build.MessageConfidence + 1, build.SealRandomnessLookbackLimit, matchEvent); err != nil { return xerrors.Errorf("failed to set up called handler: %w", err) } From ce330746269f7fb39f88c087c87f22caaec9e1a7 Mon Sep 17 00:00:00 2001 From: acruikshank Date: Wed, 3 Jun 2020 22:30:09 -0400 Subject: [PATCH 057/160] go fmt --- chain/stmgr/stmgr.go | 4 ++-- markets/storageadapter/client.go | 2 +- markets/storageadapter/provider.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index b4f27bea2..e016bcce9 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -553,7 +553,7 @@ func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid, confid reverts[val.Val.Key()] = true } case store.HCApply: - if candidateTs != nil && val.Val.Height() >= candidateTs.Height() + abi.ChainEpoch(confidence) { + if candidateTs != nil && val.Val.Height() >= candidateTs.Height()+abi.ChainEpoch(confidence) { return candidateTs, candidateRcp, nil } r, err := sm.tipsetExecutedMessage(val.Val, mcid, msg.VMMessage()) @@ -574,7 +574,7 @@ func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid, confid // check if we found the message in the chain and that is hasn't been reverted since we started searching if backTs != nil && !reverts[backTs.Key()] { // if head is at or past confidence interval, return immediately - if heightOfHead >= backTs.Height() + abi.ChainEpoch(confidence) { + if heightOfHead >= backTs.Height()+abi.ChainEpoch(confidence) { return backTs, backRcp, nil } diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 3f13978c2..d9deaedf2 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -321,7 +321,7 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider } } - if err := c.ev.Called(checkFunc, called, revert, build.MessageConfidence + 1, build.SealRandomnessLookbackLimit, matchEvent); err != nil { + if err := c.ev.Called(checkFunc, called, revert, build.MessageConfidence+1, build.SealRandomnessLookbackLimit, matchEvent); err != nil { return xerrors.Errorf("failed to set up called handler: %w", err) } diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 7c7310cd9..ddbc826eb 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -321,7 +321,7 @@ func (n *ProviderNodeAdapter) OnDealSectorCommitted(ctx context.Context, provide } - if err := n.ev.Called(checkFunc, called, revert, build.MessageConfidence + 1, build.SealRandomnessLookbackLimit, matchEvent); err != nil { + if err := n.ev.Called(checkFunc, called, revert, build.MessageConfidence+1, build.SealRandomnessLookbackLimit, matchEvent); err != nil { return xerrors.Errorf("failed to set up called handler: %w", err) } From ea111b058e3dbcc4adacd9426f24b952ea9586c4 Mon Sep 17 00:00:00 2001 From: acruikshank Date: Wed, 3 Jun 2020 22:34:57 -0400 Subject: [PATCH 058/160] imports --- cli/multisig.go | 2 +- cli/state.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/multisig.go b/cli/multisig.go index 4d6d6603d..5672c40a6 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -6,7 +6,6 @@ import ( "encoding/binary" "encoding/hex" "fmt" - "github.com/filecoin-project/lotus/build" "os" "sort" "strconv" @@ -24,6 +23,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apibstore" + "github.com/filecoin-project/lotus/build" types "github.com/filecoin-project/lotus/chain/types" ) diff --git a/cli/state.go b/cli/state.go index 9b7021520..e69a6d750 100644 --- a/cli/state.go +++ b/cli/state.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/filecoin-project/lotus/build" "reflect" "sort" "strconv" @@ -36,6 +35,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/miner" ) From da960a9de4a968211bb48f27a4bf4dda4190451d Mon Sep 17 00:00:00 2001 From: acruikshank Date: Thu, 4 Jun 2020 09:54:37 -0400 Subject: [PATCH 059/160] more import rearrangement --- markets/retrievaladapter/client.go | 2 +- paychmgr/simple.go | 2 +- storage/adapter_storage_miner.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/markets/retrievaladapter/client.go b/markets/retrievaladapter/client.go index 0c1992236..709254d49 100644 --- a/markets/retrievaladapter/client.go +++ b/markets/retrievaladapter/client.go @@ -3,7 +3,6 @@ package retrievaladapter import ( "bytes" "context" - "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" @@ -15,6 +14,7 @@ import ( "github.com/ipfs/go-cid" "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/node/impl/full" payapi "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/paychmgr" diff --git a/paychmgr/simple.go b/paychmgr/simple.go index 209217c0c..191a4c474 100644 --- a/paychmgr/simple.go +++ b/paychmgr/simple.go @@ -3,7 +3,6 @@ package paychmgr import ( "bytes" "context" - "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/specs-actors/actors/builtin" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" @@ -13,6 +12,7 @@ import ( "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" ) diff --git a/storage/adapter_storage_miner.go b/storage/adapter_storage_miner.go index 8d9bf7774..809d13d3d 100644 --- a/storage/adapter_storage_miner.go +++ b/storage/adapter_storage_miner.go @@ -3,7 +3,6 @@ package storage import ( "bytes" "context" - "github.com/filecoin-project/lotus/build" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -19,6 +18,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api/apibstore" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" From c625b307818752dc3d287f0f61e50a3e8ccae6df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 4 Jun 2020 15:54:36 +0200 Subject: [PATCH 060/160] Update fsm adapter --- node/impl/storminer.go | 2 +- storage/adapter_storage_miner.go | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/node/impl/storminer.go b/node/impl/storminer.go index cca706746..e48c9fcad 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -120,7 +120,7 @@ func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid abi.SectorNumb Value: info.SeedValue, Epoch: info.SeedEpoch, }, - Retries: info.Nonce, + Retries: info.InvalidProofs, LastErr: info.LastErr, Log: log, diff --git a/storage/adapter_storage_miner.go b/storage/adapter_storage_miner.go index 09197e71a..314fe7faf 100644 --- a/storage/adapter_storage_miner.go +++ b/storage/adapter_storage_miner.go @@ -163,9 +163,13 @@ func (s SealingAPIAdapter) StateSectorPreCommitInfo(ctx context.Context, maddr a } var pci miner.SectorPreCommitOnChainInfo - if _, err := precommits.Get(adt.UIntKey(uint64(sectorNumber)), &pci); err != nil { + ok, err := precommits.Get(adt.UIntKey(uint64(sectorNumber)), &pci) + if err != nil { return nil, err } + if !ok { + return nil, nil + } return &pci, nil } From fa3b1552cf67cfea96962ddc312828dba8a82309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 4 Jun 2020 15:54:54 +0200 Subject: [PATCH 061/160] Update sector-storage intefraces --- api/api_worker.go | 2 ++ api/apistruct/struct.go | 10 ++++++++++ cmd/lotus-seal-worker/main.go | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/api/api_worker.go b/api/api_worker.go index 182b2fe6a..a130824ab 100644 --- a/api/api_worker.go +++ b/api/api_worker.go @@ -25,6 +25,8 @@ type WorkerAPI interface { storage.Sealer + MoveStorage(ctx context.Context, sector abi.SectorID) error + UnsealPiece(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error ReadPiece(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) error diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 1f2191ae8..1cbf44bd4 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -213,6 +213,7 @@ type StorageMinerStruct struct { StorageInfo func(context.Context, stores.ID) (stores.StorageInfo, error) `perm:"admin"` StorageBestAlloc func(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredProof, sealing stores.PathType) ([]stores.StorageInfo, error) `perm:"admin"` StorageReportHealth func(ctx context.Context, id stores.ID, report stores.HealthReport) error `perm:"admin"` + StorageLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error `perm:"admin"` DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` @@ -236,6 +237,7 @@ type WorkerStruct struct { SealCommit1 func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) `perm:"admin"` SealCommit2 func(context.Context, abi.SectorID, storage.Commit1Out) (storage.Proof, error) `perm:"admin"` FinalizeSector func(context.Context, abi.SectorID) error `perm:"admin"` + MoveStorage func(ctx context.Context, sector abi.SectorID) error `perm:"admin"` UnsealPiece func(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error `perm:"admin"` ReadPiece func(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) error `perm:"admin"` @@ -802,6 +804,10 @@ func (c *StorageMinerStruct) StorageReportHealth(ctx context.Context, id stores. return c.Internal.StorageReportHealth(ctx, id, report) } +func (c *StorageMinerStruct) StorageLock(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error { + return c.Internal.StorageLock(ctx, sector, read, write) +} + func (c *StorageMinerStruct) MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error { return c.Internal.MarketImportDealData(ctx, propcid, path) } @@ -868,6 +874,10 @@ func (w *WorkerStruct) FinalizeSector(ctx context.Context, sector abi.SectorID) return w.Internal.FinalizeSector(ctx, sector) } +func (w *WorkerStruct) MoveStorage(ctx context.Context, sector abi.SectorID) error { + return w.Internal.MoveStorage(ctx, sector) +} + func (w *WorkerStruct) UnsealPiece(ctx context.Context, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, c cid.Cid) error { return w.Internal.UnsealPiece(ctx, id, index, size, randomness, c) } diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index e9721e9dd..f11a51b78 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -155,7 +155,7 @@ var runCmd = &cli.Command{ var taskTypes []sealtasks.TaskType - taskTypes = append(taskTypes, sealtasks.TTFetch) + taskTypes = append(taskTypes, sealtasks.TTFetch, sealtasks.TTCommit1, sealtasks.TTFinalize) if cctx.Bool("precommit1") { taskTypes = append(taskTypes, sealtasks.TTPreCommit1) From 8bbe63bd9d723ca67ae13106a7392ac60ccd2012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 4 Jun 2020 17:10:41 +0200 Subject: [PATCH 062/160] Add expected win rate to miner-info --- cmd/lotus-storage-miner/info.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 0257be850..aa6feaa96 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "sort" + "time" "github.com/fatih/color" "golang.org/x/xerrors" @@ -14,6 +15,7 @@ import ( sealing "github.com/filecoin-project/storage-fsm" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" ) @@ -117,6 +119,18 @@ var infoCmd = &cli.Command{ faultyPercentage) } + expWinChance := float64(types.BigMul(qpercI, types.NewInt(build.BlocksPerEpoch)).Int64()) / 1000000 + if expWinChance > 0 { + if expWinChance > 1 { + expWinChance = 1 + } + winRate := time.Duration(float64(time.Second * build.BlockDelay) / expWinChance) + winPerDay := float64(time.Hour * 24) / float64(winRate) + + fmt.Print("Expected block win rate: ") + color.Blue("%.4f/day (every %s)", winPerDay, winRate.Truncate(time.Second)) + } + fmt.Println() fmt.Printf("Miner Balance: %s\n", color.YellowString("%s", types.FIL(mact.Balance))) @@ -168,6 +182,7 @@ var stateList = []stateMeta{ {col: color.FgYellow, state: sealing.PreCommit1}, {col: color.FgYellow, state: sealing.PreCommit2}, {col: color.FgYellow, state: sealing.PreCommitting}, + {col: color.FgYellow, state: sealing.PreCommitWait}, {col: color.FgYellow, state: sealing.WaitSeed}, {col: color.FgYellow, state: sealing.Committing}, {col: color.FgYellow, state: sealing.CommitWait}, @@ -179,6 +194,7 @@ var stateList = []stateMeta{ {col: color.FgRed, state: sealing.ComputeProofFailed}, {col: color.FgRed, state: sealing.CommitFailed}, {col: color.FgRed, state: sealing.PackingFailed}, + {col: color.FgRed, state: sealing.FinalizeFailed}, {col: color.FgRed, state: sealing.Faulty}, {col: color.FgRed, state: sealing.FaultReported}, {col: color.FgRed, state: sealing.FaultedFinal}, From 0bc174697a7e4fd45843f65b93e7ff031bf11c57 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 5 Jun 2020 00:52:45 +0200 Subject: [PATCH 063/160] Fix WindowPoStScheduler Signed-off-by: Jakub Sztandera --- storage/wdpost_run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 3613736b8..ea95245dd 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -176,7 +176,7 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, deadline log.Warnw("declare faults recovered Message CID", "cid", sm.Cid()) - rec, err := s.api.StateWaitMsg(context.TODO(), sm.Cid()) + rec, err := s.api.StateWaitMsg(context.TODO(), sm.Cid(), build.MessageConfidence) if err != nil { return xerrors.Errorf("declare faults recovered wait error: %w", err) } From 9e5cc7c910837bebdf84ef3541ca5efa9401a909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 4 Jun 2020 21:22:53 +0200 Subject: [PATCH 064/160] Update fsm, sector-storage --- api/api_worker.go | 2 +- api/apistruct/struct.go | 6 +++--- cmd/lotus-storage-miner/info.go | 3 ++- go.mod | 4 ++-- go.sum | 10 ++++++---- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/api/api_worker.go b/api/api_worker.go index a130824ab..69a5aed5f 100644 --- a/api/api_worker.go +++ b/api/api_worker.go @@ -30,7 +30,7 @@ type WorkerAPI interface { UnsealPiece(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error ReadPiece(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) error - Fetch(context.Context, abi.SectorID, stores.SectorFileType, bool, stores.AcquireMode) error + Fetch(context.Context, abi.SectorID, stores.SectorFileType, stores.PathType, stores.AcquireMode) error Closing(context.Context) (<-chan struct{}, error) } diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 1cbf44bd4..ff27475e5 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -242,7 +242,7 @@ type WorkerStruct struct { UnsealPiece func(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error `perm:"admin"` ReadPiece func(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) error `perm:"admin"` - Fetch func(context.Context, abi.SectorID, stores.SectorFileType, bool, stores.AcquireMode) error `perm:"admin"` + Fetch func(context.Context, abi.SectorID, stores.SectorFileType, stores.PathType, stores.AcquireMode) error `perm:"admin"` Closing func(context.Context) (<-chan struct{}, error) `perm:"admin"` } @@ -886,8 +886,8 @@ func (w *WorkerStruct) ReadPiece(ctx context.Context, writer io.Writer, id abi.S return w.Internal.ReadPiece(ctx, writer, id, index, size) } -func (w *WorkerStruct) Fetch(ctx context.Context, id abi.SectorID, fileType stores.SectorFileType, b bool, am stores.AcquireMode) error { - return w.Internal.Fetch(ctx, id, fileType, b, am) +func (w *WorkerStruct) Fetch(ctx context.Context, id abi.SectorID, fileType stores.SectorFileType, ptype stores.PathType, am stores.AcquireMode) error { + return w.Internal.Fetch(ctx, id, fileType, ptype, am) } func (w *WorkerStruct) Closing(ctx context.Context) (<-chan struct{}, error) { diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index aa6feaa96..dfb53d898 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -189,7 +189,8 @@ var stateList = []stateMeta{ {col: color.FgYellow, state: sealing.FinalizeSector}, {col: color.FgRed, state: sealing.FailedUnrecoverable}, - {col: color.FgRed, state: sealing.SealFailed}, + {col: color.FgRed, state: sealing.SealPreCommit1Failed}, + {col: color.FgRed, state: sealing.SealPreCommit2Failed}, {col: color.FgRed, state: sealing.PreCommitFailed}, {col: color.FgRed, state: sealing.ComputeProofFailed}, {col: color.FgRed, state: sealing.CommitFailed}, diff --git a/go.mod b/go.mod index e4a4a9993..96919c680 100644 --- a/go.mod +++ b/go.mod @@ -29,10 +29,10 @@ require ( github.com/filecoin-project/go-paramfetch v0.0.2-0.20200505180321-973f8949ea8e github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/sector-storage v0.0.0-20200529175241-9df0cdf19326 + github.com/filecoin-project/sector-storage v0.0.0-20200604213020-13176e8b537d github.com/filecoin-project/specs-actors v0.5.4 github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 - github.com/filecoin-project/storage-fsm v0.0.0-20200528050623-cdada6e88960 + github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index 57da8c1ce..bf77833c4 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZO github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= github.com/filecoin-project/sector-storage v0.0.0-20200508203401-a74812ba12f3/go.mod h1:B+xzopr/oWZJz2hBL5Ekb7Obcum5ntmfbaAUlaaho28= -github.com/filecoin-project/sector-storage v0.0.0-20200529175241-9df0cdf19326 h1:WIIvOzK9KaWMFXhvivUxeg4cdMzQ/5/MW+bNpYj43wU= -github.com/filecoin-project/sector-storage v0.0.0-20200529175241-9df0cdf19326/go.mod h1:hUQo3z/5B8t8Yl+XgIxBqgNfYtIcCYzKjWWY27dHBCk= +github.com/filecoin-project/sector-storage v0.0.0-20200604213020-13176e8b537d h1:Lh6iYC2xhsV5hvACi6joUIBLabzCcOZiAxg67ZLP5dE= +github.com/filecoin-project/sector-storage v0.0.0-20200604213020-13176e8b537d/go.mod h1:hUQo3z/5B8t8Yl+XgIxBqgNfYtIcCYzKjWWY27dHBCk= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= @@ -218,8 +218,10 @@ github.com/filecoin-project/specs-actors v0.5.4 h1:Od00SYb/pTXC84oZGz9h2xGBm8GIH github.com/filecoin-project/specs-actors v0.5.4/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 h1:T3f/zkuvgtgqcXrb0NO3BicuveGOxxUAMPa/Yif2kuE= github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102/go.mod h1:xJ1/xl9+8zZeSSSFmDC3Wr6uusCTxyYPI0VeNVSFmPE= -github.com/filecoin-project/storage-fsm v0.0.0-20200528050623-cdada6e88960 h1:KXkOn1r3lyxKn2rHUcVox6WYtfIEsgyZdBVoIVJARjY= -github.com/filecoin-project/storage-fsm v0.0.0-20200528050623-cdada6e88960/go.mod h1:S0u14Wr55mpe22lElCSKbXrhtWg/jquVTTMhefQ8f4Q= +github.com/filecoin-project/storage-fsm v0.0.0-20200604153036-8ad1af106d62 h1:QDnMGcx4QisW/8tPHM4Uy9K4LkYBIijaHFxhk5Cpmxs= +github.com/filecoin-project/storage-fsm v0.0.0-20200604153036-8ad1af106d62/go.mod h1:S0u14Wr55mpe22lElCSKbXrhtWg/jquVTTMhefQ8f4Q= +github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa h1:kTCIBKMhhhVcjCh1ws72vpdDr4cdZjCyIUDFijuWuvA= +github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa/go.mod h1:S0u14Wr55mpe22lElCSKbXrhtWg/jquVTTMhefQ8f4Q= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= From 813961b458e184e1c5f187260c6a6a64c8b2ff28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 5 Jun 2020 14:57:37 +0200 Subject: [PATCH 065/160] compute-state html: Allow linking to internal executions --- cli/state.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/cli/state.go b/cli/state.go index c8425f645..fbc0b109b 100644 --- a/cli/state.go +++ b/cli/state.go @@ -1042,7 +1042,7 @@ func computeStateHtml(ts *types.TipSet, o *api.ComputeStateOutput, getCode func( if len(ir.InternalExecutions) > 0 { fmt.Println("
Internal executions:
") - if err := printInternalExecutionsHtml(ir.InternalExecutions, getCode); err != nil { + if err := printInternalExecutionsHtml(cid.String(), ir.InternalExecutions, getCode); err != nil { return err } } @@ -1054,8 +1054,10 @@ func computeStateHtml(ts *types.TipSet, o *api.ComputeStateOutput, getCode func( return nil } -func printInternalExecutionsHtml(trace []*types.ExecutionResult, getCode func(addr address.Address) (cid.Cid, error)) error { - for _, im := range trace { +func printInternalExecutionsHtml(hashName string, trace []*types.ExecutionResult, getCode func(addr address.Address) (cid.Cid, error)) error { + for i, im := range trace { + hashName := fmt.Sprintf("%s-r%d", hashName, i) + toCode, err := getCode(im.Msg.To) if err != nil { return xerrors.Errorf("getting code for %s: %w", toCode, err) @@ -1086,18 +1088,18 @@ func printInternalExecutionsHtml(trace []*types.ExecutionResult, getCode func(ad slow := im.Duration > 10*time.Millisecond veryslow := im.Duration > 50*time.Millisecond - fmt.Printf(`
-

%s:%s

+ fmt.Printf(`
+
%s -> %s (%s FIL), M%d
%s
Took %s, Exit: %d%s -`, codeStr(toCode), methods[toCode][im.Msg.Method].name, im.Msg.From, im.Msg.To, types.FIL(im.Msg.Value), im.Msg.Method, params, slow, veryslow, im.Duration, im.MsgRct.ExitCode, im.MsgRct.ExitCode, ret) +`, hashName, hashName, codeStr(toCode), methods[toCode][im.Msg.Method].name, im.Msg.From, im.Msg.To, types.FIL(im.Msg.Value), im.Msg.Method, params, slow, veryslow, im.Duration, im.MsgRct.ExitCode, im.MsgRct.ExitCode, ret) if im.MsgRct.ExitCode != 0 { fmt.Printf(`
Error: 
%s
`, im.Error) } if len(im.Subcalls) > 0 { fmt.Println("
Subcalls:
") - if err := printInternalExecutionsHtml(im.Subcalls, getCode); err != nil { + if err := printInternalExecutionsHtml(hashName, im.Subcalls, getCode); err != nil { return err } } From 6b06ae528c44130c5bc9fe9d7b05d7a7ea1b1b06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 5 Jun 2020 17:41:21 +0200 Subject: [PATCH 066/160] chain get: @state selector to prettify actor state --- cli/chain.go | 3 ++- node/impl/full/chain.go | 51 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/cli/chain.go b/cli/chain.go index 4d296cf78..a174e75f2 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -442,7 +442,8 @@ var chainGetCmd = &cli.Command{ - /ipfs/[cid]/@Hi:123 - get varint elem 123 from hamt - /ipfs/[cid]/@Hu:123 - get uvarint elem 123 from hamt - /ipfs/[cid]/@Ha:t01 - get element under Addr(t01).Bytes - - /ipfs/[cid]/@A:10 - get 10th amt element + - /ipfs/[cid]/@A:10 - get 10th amt element + - .../@Ha:t01/@state - get pretty map-based actor state List of --as-type types: - raw diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index f6553fa54..d5c8f385b 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -1,7 +1,9 @@ package full import ( + "bytes" "context" + "encoding/json" "fmt" "io" "strconv" @@ -33,6 +35,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" ) var log = logging.Logger("fullnode") @@ -377,6 +380,54 @@ func resolveOnce(bs blockstore.Blockstore) func(ctx context.Context, ds ipld.Nod return resolveOnce(bs)(ctx, ds, n, names[1:]) } + if names[0] == "@state" { + var act types.Actor + if err := act.UnmarshalCBOR(bytes.NewReader(nd.RawData())); err != nil { + return nil, nil, xerrors.Errorf("unmarshaling actor struct for @state: %w", err) + } + + head, err := ds.Get(ctx, act.Head) + if err != nil { + return nil, nil, xerrors.Errorf("getting actor head for @state: %w", err) + } + + m, err := vm.DumpActorState(act.Code, head.RawData()) + if err != nil { + return nil, nil, err + } + + // a hack to workaround struct aliasing in refmt + ms := map[string]interface{}{} + { + mstr, err := json.Marshal(m) + if err != nil { + return nil, nil, err + } + if err := json.Unmarshal(mstr, &ms); err != nil { + return nil, nil, err + } + } + + n, err := cbor.WrapObject(ms, mh.SHA2_256, 32) + if err != nil { + return nil, nil, err + } + + if err := bs.Put(n); err != nil { + return nil, nil, xerrors.Errorf("put amt val: %w", err) + } + + if len(names) == 1 { + return &ipld.Link{ + Name: "state", + Size: 0, + Cid: n.Cid(), + }, nil, nil + } + + return resolveOnce(bs)(ctx, ds, n, names[1:]) + } + return nd.ResolveLink(names) } } From 51f1712564a89707f237c4fa3db1ebc705e229d7 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 5 Jun 2020 10:47:49 -0700 Subject: [PATCH 067/160] add a simple semaphore to batch verify seals --- chain/vm/syscalls.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index 100472c69..143807ab4 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + goruntime "runtime" "sync" "github.com/filecoin-project/go-address" @@ -243,6 +244,8 @@ func (ss *syscallShim) VerifySignature(sig crypto.Signature, addr address.Addres func (ss *syscallShim) BatchVerifySeals(inp map[address.Address][]abi.SealVerifyInfo) (map[address.Address][]bool, error) { out := make(map[address.Address][]bool) + sema := make(chan struct{}, goruntime.NumCPU()) + var wg sync.WaitGroup for addr, seals := range inp { results := make([]bool, len(seals)) @@ -252,12 +255,16 @@ func (ss *syscallShim) BatchVerifySeals(inp map[address.Address][]abi.SealVerify wg.Add(1) go func(ma address.Address, ix int, svi abi.SealVerifyInfo, res []bool) { defer wg.Done() + sema <- struct{}{} + if err := ss.VerifySeal(svi); err != nil { log.Warnw("seal verify in batch failed", "miner", ma, "index", ix, "err", err) res[ix] = false } else { res[ix] = true } + + <-sema }(addr, i, s, results) } } From cd6670205c43a2c4d9b73039d33e5a59f2ce49e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 5 Jun 2020 20:21:40 +0200 Subject: [PATCH 068/160] Update sector-storage --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2f2adce08..e59cdfbaa 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/filecoin-project/go-paramfetch v0.0.2-0.20200505180321-973f8949ea8e github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/sector-storage v0.0.0-20200604213020-13176e8b537d + github.com/filecoin-project/sector-storage v0.0.0-20200605192746-4b9317d1f08f github.com/filecoin-project/specs-actors v0.5.4 github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa diff --git a/go.sum b/go.sum index 9cb785560..bb16166fd 100644 --- a/go.sum +++ b/go.sum @@ -219,8 +219,8 @@ github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZO github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= github.com/filecoin-project/sector-storage v0.0.0-20200508203401-a74812ba12f3/go.mod h1:B+xzopr/oWZJz2hBL5Ekb7Obcum5ntmfbaAUlaaho28= -github.com/filecoin-project/sector-storage v0.0.0-20200604213020-13176e8b537d h1:Lh6iYC2xhsV5hvACi6joUIBLabzCcOZiAxg67ZLP5dE= -github.com/filecoin-project/sector-storage v0.0.0-20200604213020-13176e8b537d/go.mod h1:hUQo3z/5B8t8Yl+XgIxBqgNfYtIcCYzKjWWY27dHBCk= +github.com/filecoin-project/sector-storage v0.0.0-20200605192746-4b9317d1f08f h1:UASwQxRVA9LEfpFhhh87RvZMaPRFfR7QlWXNd9u6Aj0= +github.com/filecoin-project/sector-storage v0.0.0-20200605192746-4b9317d1f08f/go.mod h1:hUQo3z/5B8t8Yl+XgIxBqgNfYtIcCYzKjWWY27dHBCk= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= From 3f81ff35078020edae3c98473a20d0e7bb1f048b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 5 Jun 2020 22:06:11 +0200 Subject: [PATCH 069/160] Update specs-actors --- chain/gen/genesis/miners.go | 2 +- cli/client.go | 4 ++-- cmd/lotus-fountain/main.go | 2 +- cmd/lotus-storage-miner/init.go | 4 ++-- go.mod | 2 +- go.sum | 2 ++ markets/storageadapter/client.go | 3 ++- node/impl/client/client.go | 4 ++-- node/node_test.go | 2 +- 9 files changed, 14 insertions(+), 11 deletions(-) diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index fa71865b8..92d77d270 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -71,7 +71,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid constructorParams := &power.CreateMinerParams{ Owner: m.Worker, Worker: m.Worker, - Peer: m.PeerId, + Peer: []byte(m.PeerId), SealProofType: spt, } diff --git a/cli/client.go b/cli/client.go index ea6241b9b..8ee26f2d2 100644 --- a/cli/client.go +++ b/cli/client.go @@ -446,11 +446,11 @@ var clientQueryAskCmd = &cli.Command{ return xerrors.Errorf("failed to get peerID for miner: %w", err) } - if mi.PeerId == peer.ID("SETME") { + if peer.ID(mi.PeerId) == peer.ID("SETME") { return fmt.Errorf("the miner hasn't initialized yet") } - pid = mi.PeerId + pid = peer.ID(mi.PeerId) } ask, err := api.ClientQueryAsk(ctx, pid, maddr) diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go index f181bf173..8b13c7ba6 100644 --- a/cmd/lotus-fountain/main.go +++ b/cmd/lotus-fountain/main.go @@ -294,7 +294,7 @@ func (h *handler) mkminer(w http.ResponseWriter, r *http.Request) { Owner: owner, Worker: owner, SealProofType: spt, - Peer: h.defaultMinerPeer, + Peer: abi.PeerID(h.defaultMinerPeer), }) if err != nil { w.WriteHeader(400) diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 5d598fa5b..bcdb113d8 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -545,7 +545,7 @@ func configureStorageMiner(ctx context.Context, api lapi.FullNode, addr address. return xerrors.Errorf("getWorkerAddr returned bad address: %w", err) } - enc, err := actors.SerializeParams(&miner2.ChangePeerIDParams{NewID: peerid}) + enc, err := actors.SerializeParams(&miner2.ChangePeerIDParams{NewID: abi.PeerID(peerid)}) if err != nil { return err } @@ -622,7 +622,7 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, Owner: owner, Worker: worker, SealProofType: spt, - Peer: peerid, + Peer: abi.PeerID(peerid), }) if err != nil { return address.Undef, err diff --git a/go.mod b/go.mod index e59cdfbaa..6626da43a 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/sector-storage v0.0.0-20200605192746-4b9317d1f08f - github.com/filecoin-project/specs-actors v0.5.4 + github.com/filecoin-project/specs-actors v0.5.5-0.20200605181154-8e2301d36794 github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index bb16166fd..cbaa7e296 100644 --- a/go.sum +++ b/go.sum @@ -227,6 +227,8 @@ github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVl github.com/filecoin-project/specs-actors v0.5.4-0.20200521014528-0df536f7e461/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= github.com/filecoin-project/specs-actors v0.5.4 h1:Od00SYb/pTXC84oZGz9h2xGBm8GIHo8GXUaRKg1ClZo= github.com/filecoin-project/specs-actors v0.5.4/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= +github.com/filecoin-project/specs-actors v0.5.5-0.20200605181154-8e2301d36794 h1:JrpmpAPPjso2+RNvPYqXLaDRLyg6A1l26ub4ah+WC1o= +github.com/filecoin-project/specs-actors v0.5.5-0.20200605181154-8e2301d36794/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 h1:T3f/zkuvgtgqcXrb0NO3BicuveGOxxUAMPa/Yif2kuE= github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102/go.mod h1:xJ1/xl9+8zZeSSSFmDC3Wr6uusCTxyYPI0VeNVSFmPE= github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa h1:kTCIBKMhhhVcjCh1ws72vpdDr4cdZjCyIUDFijuWuvA= diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index d9deaedf2..5f962c171 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -19,6 +19,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/crypto" "github.com/filecoin-project/specs-actors/actors/runtime/exitcode" "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p-core/peer" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/events" @@ -79,7 +80,7 @@ func (n *ClientNodeAdapter) ListStorageProviders(ctx context.Context, encodedTs return nil, err } - storageProviderInfo := utils.NewStorageProviderInfo(addr, mi.Worker, mi.SectorSize, mi.PeerId) + storageProviderInfo := utils.NewStorageProviderInfo(addr, mi.Worker, mi.SectorSize, peer.ID(mi.PeerId)) out = append(out, &storageProviderInfo) } diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 594787ae8..59a258489 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -103,7 +103,7 @@ func (a *API) ClientStartDeal(ctx context.Context, params *api.StartDealParams) return nil, xerrors.New("data doesn't fit in a sector") } - providerInfo := utils.NewStorageProviderInfo(params.Miner, mi.Worker, mi.SectorSize, mi.PeerId) + providerInfo := utils.NewStorageProviderInfo(params.Miner, mi.Worker, mi.SectorSize, peer.ID(mi.PeerId)) dealStart := params.DealStartEpoch if dealStart <= 0 { // unset, or explicitly 'epoch undefined' @@ -309,7 +309,7 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref return err } - order.MinerPeerID = mi.PeerId + order.MinerPeerID = peer.ID(mi.PeerId) } if order.Size == 0 { diff --git a/node/node_test.go b/node/node_test.go index 6258fa943..8b17fac2e 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -97,7 +97,7 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a peerid, err := peer.IDFromPrivateKey(pk) require.NoError(t, err) - enc, err := actors.SerializeParams(&saminer.ChangePeerIDParams{NewID: peerid}) + enc, err := actors.SerializeParams(&saminer.ChangePeerIDParams{NewID: abi.PeerID(peerid)}) require.NoError(t, err) msg := &types.Message{ From 73d928751d7d2344f57c4ff3c5496fd2c2198d10 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 5 Jun 2020 15:12:13 -0700 Subject: [PATCH 070/160] disallow sourcing randomness from future epochs --- chain/store/store.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index 72c77ee36..998129d41 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -904,7 +904,7 @@ func DrawRandomness(rbase []byte, pers crypto.DomainSeparationTag, round abi.Cha return h.Sum(nil), nil } -func (cs *ChainStore) GetRandomness(ctx context.Context, blks []cid.Cid, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) (out []byte, err error) { +func (cs *ChainStore) GetRandomness(ctx context.Context, blks []cid.Cid, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { _, span := trace.StartSpan(ctx, "store.GetRandomness") defer span.End() span.AddAttributes(trace.Int64Attribute("round", int64(round))) @@ -915,7 +915,7 @@ func (cs *ChainStore) GetRandomness(ctx context.Context, blks []cid.Cid, pers cr } if round > ts.Height() { - return DrawRandomness(ts.MinTicket().VRFProof, pers, round, entropy) + return nil, xerrors.Errorf("cannot draw randomness from the future") } searchHeight := round From 2571480bb67e9c3e7aa20cac1848beed8ef8f983 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 5 Jun 2020 18:10:50 -0400 Subject: [PATCH 071/160] Update chain-val, use tagged specs-actors --- go.mod | 4 ++-- go.sum | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 6626da43a..51f33818d 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/drand/drand v0.9.2-0.20200529123141-6b138aefaef2 github.com/drand/kyber v1.0.2 github.com/fatih/color v1.8.0 - github.com/filecoin-project/chain-validation v0.0.6-0.20200528212128-d4578ae4f874 + github.com/filecoin-project/chain-validation v0.0.6-0.20200605221044-7f78284bbc94 github.com/filecoin-project/filecoin-ffi v0.26.1-0.20200508175440-05b30afeb00d github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2 @@ -30,7 +30,7 @@ require ( github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/sector-storage v0.0.0-20200605192746-4b9317d1f08f - github.com/filecoin-project/specs-actors v0.5.5-0.20200605181154-8e2301d36794 + github.com/filecoin-project/specs-actors v0.5.5 github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index cbaa7e296..b1c3b032c 100644 --- a/go.sum +++ b/go.sum @@ -180,8 +180,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= -github.com/filecoin-project/chain-validation v0.0.6-0.20200528212128-d4578ae4f874 h1:NNzyoQjidcvvoARgaM6zo2jYXC0KwNQkaYuDiBFcAD4= -github.com/filecoin-project/chain-validation v0.0.6-0.20200528212128-d4578ae4f874/go.mod h1:aV5C4aX+kZwh5OlTHNCHxQ5vTnpTFKUuKiDTk9f/8Es= +github.com/filecoin-project/chain-validation v0.0.6-0.20200605221044-7f78284bbc94 h1:svEal/usZ/REEjEz6ije3/ALn8duM2CckR0LjT1btx0= +github.com/filecoin-project/chain-validation v0.0.6-0.20200605221044-7f78284bbc94/go.mod h1:aOmmdhO0xIRtWCwx3MyIv3tPCCorM1Bq7pNOJGMLndA= github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef h1:Wi5E+P1QfHP8IF27eUiTx5vYfqQZwfPxzq3oFEq8w8U= @@ -225,10 +225,8 @@ github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.m github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= github.com/filecoin-project/specs-actors v0.5.4-0.20200521014528-0df536f7e461/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= -github.com/filecoin-project/specs-actors v0.5.4 h1:Od00SYb/pTXC84oZGz9h2xGBm8GIHo8GXUaRKg1ClZo= -github.com/filecoin-project/specs-actors v0.5.4/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= -github.com/filecoin-project/specs-actors v0.5.5-0.20200605181154-8e2301d36794 h1:JrpmpAPPjso2+RNvPYqXLaDRLyg6A1l26ub4ah+WC1o= -github.com/filecoin-project/specs-actors v0.5.5-0.20200605181154-8e2301d36794/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= +github.com/filecoin-project/specs-actors v0.5.5 h1:bDsowem6dLRc9B7g3sgFYvHgfWTH4D9q5ehWLGLdKCw= +github.com/filecoin-project/specs-actors v0.5.5/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 h1:T3f/zkuvgtgqcXrb0NO3BicuveGOxxUAMPa/Yif2kuE= github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102/go.mod h1:xJ1/xl9+8zZeSSSFmDC3Wr6uusCTxyYPI0VeNVSFmPE= github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa h1:kTCIBKMhhhVcjCh1ws72vpdDr4cdZjCyIUDFijuWuvA= From f765fe36d7e8542ed37ce71a97871729fdbef1c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 5 Jun 2020 01:38:38 +0200 Subject: [PATCH 072/160] Make lotus-storage-miner deadlines correct --- cmd/lotus-storage-miner/proving.go | 59 ++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/cmd/lotus-storage-miner/proving.go b/cmd/lotus-storage-miner/proving.go index 9f9ca600a..3415c7ffd 100644 --- a/cmd/lotus-storage-miner/proving.go +++ b/cmd/lotus-storage-miner/proving.go @@ -3,15 +3,21 @@ package main import ( "bytes" "fmt" + "os" + "text/tabwriter" "time" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-bitfield" + rlepluslazy "github.com/filecoin-project/go-bitfield/rle" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/urfave/cli/v2" - "golang.org/x/xerrors" ) var provingCmd = &cli.Command{ @@ -191,28 +197,59 @@ var provingDeadlinesCmd = &cli.Command{ } } + tw := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0) + _, _ = fmt.Fprintln(tw, "deadline\tsectors\tpartitions\tproven") + for i, field := range deadlines.Due { c, err := field.Count() if err != nil { return err } - var info string - proven, err := mas.PostSubmissions.IsSet(uint64(i)) + firstPartition, sectorCount, err := miner.PartitionsForDeadline(deadlines, mas.Info.WindowPoStPartitionSectors, uint64(i)) if err != nil { return err } - if proven { - info += ", proven" + partitionCount := (sectorCount + mas.Info.WindowPoStPartitionSectors - 1) / mas.Info.WindowPoStPartitionSectors + + var provenPartitions uint64 + { + var maskRuns []rlepluslazy.Run + if firstPartition > 0 { + maskRuns = append(maskRuns, rlepluslazy.Run{ + Val: false, + Len: firstPartition, + }) + } + maskRuns = append(maskRuns, rlepluslazy.Run{ + Val: true, + Len: partitionCount, + }) + + ppbm, err := bitfield.NewFromIter(&rlepluslazy.RunSliceIterator{Runs: maskRuns}) + if err != nil { + return err + } + + pp, err := bitfield.IntersectBitField(ppbm, mas.PostSubmissions) + if err != nil { + return err + } + + provenPartitions, err = pp.Count() + if err != nil { + return err + } } + var cur string if di.Index == uint64(i) { - info += " (current)" + cur += "\t(current)" } - fmt.Printf("%d: %d sectors%s\n", i, c, info) + _, _ = fmt.Fprintf(tw,"%d\t%d\t%d\t%d%s\n", i, c, partitionCount, provenPartitions, cur) } - return nil + return tw.Flush() }, } From 3b133971657f3c70c3f16d426b2068ac31a716a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 6 Jun 2020 00:59:01 +0200 Subject: [PATCH 073/160] gofmt --- api/apistruct/struct.go | 4 ++-- cli/auth.go | 2 +- cli/chain.go | 2 +- cli/client.go | 2 +- cli/cmd.go | 2 +- cli/log.go | 2 +- cli/mpool.go | 2 +- cli/multisig.go | 2 +- cli/params.go | 2 +- cli/state.go | 2 +- cmd/lotus-bench/main.go | 2 +- cmd/lotus-chainwatch/main.go | 2 +- cmd/lotus-fountain/main.go | 2 +- cmd/lotus-seal-worker/main.go | 2 +- cmd/lotus-seed/genesis.go | 2 +- cmd/lotus-shed/bitfield.go | 2 +- cmd/lotus-shed/import-car.go | 2 +- cmd/lotus-shed/params.go | 2 +- cmd/lotus-storage-miner/info.go | 6 +++--- cmd/lotus-storage-miner/init.go | 2 +- cmd/lotus-storage-miner/proving.go | 2 +- cmd/lotus-storage-miner/run.go | 2 +- cmd/lotus-storage-miner/sectors.go | 2 +- cmd/lotus-storage-miner/storage.go | 2 +- cmd/lotus/daemon.go | 2 +- 25 files changed, 28 insertions(+), 28 deletions(-) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index aba0790e7..940f876e2 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -216,7 +216,7 @@ type StorageMinerStruct struct { StorageInfo func(context.Context, stores.ID) (stores.StorageInfo, error) `perm:"admin"` StorageBestAlloc func(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredProof, sealing stores.PathType) ([]stores.StorageInfo, error) `perm:"admin"` StorageReportHealth func(ctx context.Context, id stores.ID, report stores.HealthReport) error `perm:"admin"` - StorageLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error `perm:"admin"` + StorageLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error `perm:"admin"` DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` @@ -240,7 +240,7 @@ type WorkerStruct struct { SealCommit1 func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) `perm:"admin"` SealCommit2 func(context.Context, abi.SectorID, storage.Commit1Out) (storage.Proof, error) `perm:"admin"` FinalizeSector func(context.Context, abi.SectorID) error `perm:"admin"` - MoveStorage func(ctx context.Context, sector abi.SectorID) error `perm:"admin"` + MoveStorage func(ctx context.Context, sector abi.SectorID) error `perm:"admin"` UnsealPiece func(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error `perm:"admin"` ReadPiece func(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) error `perm:"admin"` diff --git a/cli/auth.go b/cli/auth.go index e3497f17c..d59ac37a5 100644 --- a/cli/auth.go +++ b/cli/auth.go @@ -3,8 +3,8 @@ package cli import ( "fmt" - "golang.org/x/xerrors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/go-jsonrpc/auth" diff --git a/cli/chain.go b/cli/chain.go index a174e75f2..d76335263 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -23,9 +23,9 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/util/adt" cid "github.com/ipfs/go-cid" + "github.com/urfave/cli/v2" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" - "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" diff --git a/cli/client.go b/cli/client.go index afd8e3e29..d6b59db53 100644 --- a/cli/client.go +++ b/cli/client.go @@ -11,8 +11,8 @@ import ( "github.com/ipfs/go-cidutil/cidenc" "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multibase" - "golang.org/x/xerrors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/storagemarket" diff --git a/cli/cmd.go b/cli/cmd.go index 3953f1372..83a1845fc 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -13,8 +13,8 @@ import ( "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" - "golang.org/x/xerrors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/go-jsonrpc" diff --git a/cli/log.go b/cli/log.go index 3e1027ccf..a7d95799d 100644 --- a/cli/log.go +++ b/cli/log.go @@ -3,8 +3,8 @@ package cli import ( "fmt" - "golang.org/x/xerrors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" ) var logCmd = &cli.Command{ diff --git a/cli/mpool.go b/cli/mpool.go index 1b8a1e0d1..fec7b2e0d 100644 --- a/cli/mpool.go +++ b/cli/mpool.go @@ -5,8 +5,8 @@ import ( "fmt" "sort" - "golang.org/x/xerrors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" diff --git a/cli/multisig.go b/cli/multisig.go index 961a14a56..e0f233db8 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -17,9 +17,9 @@ import ( cid "github.com/ipfs/go-cid" "github.com/ipfs/go-hamt-ipld" cbor "github.com/ipfs/go-ipld-cbor" + "github.com/urfave/cli/v2" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" - "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apibstore" diff --git a/cli/params.go b/cli/params.go index 93c8d9458..b0594b641 100644 --- a/cli/params.go +++ b/cli/params.go @@ -3,8 +3,8 @@ package cli import ( "github.com/docker/go-units" paramfetch "github.com/filecoin-project/go-paramfetch" - "golang.org/x/xerrors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/lotus/build" ) diff --git a/cli/state.go b/cli/state.go index fbc0b109b..c0cfd8eb4 100644 --- a/cli/state.go +++ b/cli/state.go @@ -15,9 +15,9 @@ import ( "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multihash" + "github.com/urfave/cli/v2" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" - "github.com/urfave/cli/v2" "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi" diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index e410d09c5..5951bd6fa 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -15,8 +15,8 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/minio/blake2b-simd" "github.com/mitchellh/go-homedir" - "golang.org/x/xerrors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" paramfetch "github.com/filecoin-project/go-paramfetch" diff --git a/cmd/lotus-chainwatch/main.go b/cmd/lotus-chainwatch/main.go index b5ca379e0..704c4d457 100644 --- a/cmd/lotus-chainwatch/main.go +++ b/cmd/lotus-chainwatch/main.go @@ -7,8 +7,8 @@ import ( "os" logging "github.com/ipfs/go-log/v2" - "golang.org/x/xerrors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go index 8b13c7ba6..c5c6d5857 100644 --- a/cmd/lotus-fountain/main.go +++ b/cmd/lotus-fountain/main.go @@ -14,8 +14,8 @@ import ( "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/peer" - "golang.org/x/xerrors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/abi" diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index c9f0239cf..65418ead8 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -12,8 +12,8 @@ import ( "github.com/google/uuid" "github.com/gorilla/mux" logging "github.com/ipfs/go-log/v2" - "golang.org/x/xerrors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/go-jsonrpc" "github.com/filecoin-project/go-jsonrpc/auth" diff --git a/cmd/lotus-seed/genesis.go b/cmd/lotus-seed/genesis.go index 471fb99df..748b406ac 100644 --- a/cmd/lotus-seed/genesis.go +++ b/cmd/lotus-seed/genesis.go @@ -6,8 +6,8 @@ import ( "github.com/google/uuid" "github.com/mitchellh/go-homedir" - "golang.org/x/xerrors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi/big" diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index 0cf9e4ea0..79ce214ee 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -7,8 +7,8 @@ import ( "io/ioutil" "os" - "golang.org/x/xerrors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/go-bitfield" rlepluslazy "github.com/filecoin-project/go-bitfield/rle" diff --git a/cmd/lotus-shed/import-car.go b/cmd/lotus-shed/import-car.go index 091ab9054..e54089df0 100644 --- a/cmd/lotus-shed/import-car.go +++ b/cmd/lotus-shed/import-car.go @@ -7,8 +7,8 @@ import ( blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipld/go-car" - "golang.org/x/xerrors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/lotus/node/repo" ) diff --git a/cmd/lotus-shed/params.go b/cmd/lotus-shed/params.go index 74876d778..3f7e7b6fb 100644 --- a/cmd/lotus-shed/params.go +++ b/cmd/lotus-shed/params.go @@ -4,8 +4,8 @@ import ( "github.com/docker/go-units" paramfetch "github.com/filecoin-project/go-paramfetch" lcli "github.com/filecoin-project/lotus/cli" - "golang.org/x/xerrors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/lotus/build" ) diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 161a386a0..9d98f8569 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -8,8 +8,8 @@ import ( "time" "github.com/fatih/color" - "golang.org/x/xerrors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/specs-actors/actors/builtin/miner" sealing "github.com/filecoin-project/storage-fsm" @@ -124,8 +124,8 @@ var infoCmd = &cli.Command{ if expWinChance > 1 { expWinChance = 1 } - winRate := time.Duration(float64(time.Second * build.BlockDelay) / expWinChance) - winPerDay := float64(time.Hour * 24) / float64(winRate) + winRate := time.Duration(float64(time.Second*build.BlockDelay) / expWinChance) + winPerDay := float64(time.Hour*24) / float64(winRate) fmt.Print("Expected block win rate: ") color.Blue("%.4f/day (every %s)", winPerDay, winRate.Truncate(time.Second)) diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 024b8d540..29e82cc2f 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -18,8 +18,8 @@ import ( "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" "github.com/mitchellh/go-homedir" - "golang.org/x/xerrors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" diff --git a/cmd/lotus-storage-miner/proving.go b/cmd/lotus-storage-miner/proving.go index 3415c7ffd..8adb021ea 100644 --- a/cmd/lotus-storage-miner/proving.go +++ b/cmd/lotus-storage-miner/proving.go @@ -247,7 +247,7 @@ var provingDeadlinesCmd = &cli.Command{ if di.Index == uint64(i) { cur += "\t(current)" } - _, _ = fmt.Fprintf(tw,"%d\t%d\t%d\t%d%s\n", i, c, partitionCount, provenPartitions, cur) + _, _ = fmt.Fprintf(tw, "%d\t%d\t%d\t%d%s\n", i, c, partitionCount, provenPartitions, cur) } return tw.Flush() diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 62df0a48d..29634d17d 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -11,8 +11,8 @@ import ( mux "github.com/gorilla/mux" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" - "golang.org/x/xerrors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/go-jsonrpc" "github.com/filecoin-project/go-jsonrpc/auth" diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index 1a500cc8b..4a3109f37 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -8,8 +8,8 @@ import ( "text/tabwriter" "time" - "golang.org/x/xerrors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/specs-actors/actors/abi" diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index a8ceda6c2..4ab46fdae 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -14,8 +14,8 @@ import ( "github.com/fatih/color" "github.com/google/uuid" "github.com/mitchellh/go-homedir" - "golang.org/x/xerrors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi" diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 6da80a0c8..fd9ceea85 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -18,11 +18,11 @@ import ( blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" + "github.com/urfave/cli/v2" "go.opencensus.io/stats" "go.opencensus.io/stats/view" "go.opencensus.io/tag" "golang.org/x/xerrors" - "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" From 94da3396228ff829635fe3b09376771f877eec58 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 6 Jun 2020 02:47:29 +0200 Subject: [PATCH 074/160] Fix protobuf panic by updating protobuf to 1.4.2 The panic is caused by usage of internal protobuf interface. It was changed in protobuf 1.4.0 and reluctantly fixed in 1.4.2 Signed-off-by: Jakub Sztandera --- go.mod | 1 + go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/go.mod b/go.mod index a6b832d16..9d5411681 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect + github.com/golang/protobuf v1.4.2 // indirect github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.4 github.com/gorilla/websocket v1.4.2 diff --git a/go.sum b/go.sum index fb7ad986f..7deb743d1 100644 --- a/go.sum +++ b/go.sum @@ -292,6 +292,8 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= From baa69c3c812aae2de1bbb7860b18ec30becd7e8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 6 Jun 2020 19:51:10 +0200 Subject: [PATCH 075/160] Update specs-actors to v0.5.6 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 9d5411681..786afed49 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/sector-storage v0.0.0-20200605192746-4b9317d1f08f - github.com/filecoin-project/specs-actors v0.5.5 + github.com/filecoin-project/specs-actors v0.5.6 github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index 7deb743d1..be456d1dd 100644 --- a/go.sum +++ b/go.sum @@ -227,6 +227,8 @@ github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVl github.com/filecoin-project/specs-actors v0.5.4-0.20200521014528-0df536f7e461/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= github.com/filecoin-project/specs-actors v0.5.5 h1:bDsowem6dLRc9B7g3sgFYvHgfWTH4D9q5ehWLGLdKCw= github.com/filecoin-project/specs-actors v0.5.5/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= +github.com/filecoin-project/specs-actors v0.5.6 h1:WlhtoXwFoKlP1b06NI4NJaxC4m9EXNV+qFVl43/xRN8= +github.com/filecoin-project/specs-actors v0.5.6/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 h1:T3f/zkuvgtgqcXrb0NO3BicuveGOxxUAMPa/Yif2kuE= github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102/go.mod h1:xJ1/xl9+8zZeSSSFmDC3Wr6uusCTxyYPI0VeNVSFmPE= github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa h1:kTCIBKMhhhVcjCh1ws72vpdDr4cdZjCyIUDFijuWuvA= From e91cc9cd08da76c54ce2344a245af1ab2f460939 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 3 Jun 2020 15:48:22 +0200 Subject: [PATCH 076/160] Add drand topic scores Signed-off-by: Jakub Sztandera --- node/modules/lp2p/pubsub.go | 94 ++++++++++++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 13 deletions(-) diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go index fdb4dc028..76387dd1d 100644 --- a/node/modules/lp2p/pubsub.go +++ b/node/modules/lp2p/pubsub.go @@ -2,6 +2,7 @@ package lp2p import ( "context" + "encoding/json" "time" host "github.com/libp2p/go-libp2p-core/host" @@ -11,6 +12,7 @@ import ( blake2b "github.com/minio/blake2b-simd" ma "github.com/multiformats/go-multiaddr" "go.uber.org/fx" + "golang.org/x/xerrors" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/node/config" @@ -28,17 +30,42 @@ func init() { pubsub.GossipSubDlazy = 12 pubsub.GossipSubDirectConnectInitialDelay = 30 * time.Second } - func ScoreKeeper() *dtypes.ScoreKeeper { return new(dtypes.ScoreKeeper) } -func GossipSub(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, nn dtypes.NetworkName, bp dtypes.BootstrapPeers, cfg *config.Pubsub, sk *dtypes.ScoreKeeper) (service *pubsub.PubSub, err error) { +type GossipIn struct { + fx.In + Mctx helpers.MetricsCtx + Lc fx.Lifecycle + Host host.Host + Nn dtypes.NetworkName + Bp dtypes.BootstrapPeers + Cfg *config.Pubsub + Sk *dtypes.ScoreKeeper +} + +func getDrandTopic() (string, error) { + var drandInfo = struct { + Hash string `json:"hash"` + }{} + err := json.Unmarshal([]byte(build.DrandChain), &drandInfo) + if err != nil { + return "", xerrors.Errorf("could not unmarshal drand chain info: %w", err) + } + return "/drand/pubsub/v0.0.0/" + drandInfo.Hash, nil +} + +func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { bootstrappers := make(map[peer.ID]struct{}) - for _, pi := range bp { + for _, pi := range in.Bp { bootstrappers[pi.ID] = struct{}{} } - isBootstrapNode := cfg.Bootstrapper + isBootstrapNode := in.Cfg.Bootstrapper + drandTopic, err := getDrandTopic() + if err != nil { + return nil, err + } options := []pubsub.Option{ // Gossipsubv1.1 configuration @@ -80,7 +107,48 @@ func GossipSub(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, nn dtyp // topic parameters Topics: map[string]*pubsub.TopicScoreParams{ - build.BlocksTopic(nn): { + drandTopic: { + // expected 2 beaconsn/min + TopicWeight: 0.5, // 5x block topic + + // 1 tick per second, maxes at 1 after 1 hour + TimeInMeshWeight: 0.00027, // ~1/3600 + TimeInMeshQuantum: time.Second, + TimeInMeshCap: 1, + + // deliveries decay after 1 hour, cap at 100 blocks + FirstMessageDeliveriesWeight: 5, // max value is 500 + FirstMessageDeliveriesDecay: pubsub.ScoreParameterDecay(time.Hour), + FirstMessageDeliveriesCap: 100, // 100 blocks in an hour + + // Mesh Delivery Failure is currently turned off for blocks + // This is on purpose as + // - the traffic is very low for meaningful distribution of incoming edges. + // - the reaction time needs to be very slow -- in the order of 10 min at least + // so we might as well let opportunistic grafting repair the mesh on its own + // pace. + // - the network is too small, so large asymmetries can be expected between mesh + // edges. + // We should revisit this once the network grows. + // + // // tracks deliveries in the last minute + // // penalty activates at 1 minute and expects ~0.4 blocks + // MeshMessageDeliveriesWeight: -576, // max penalty is -100 + // MeshMessageDeliveriesDecay: pubsub.ScoreParameterDecay(time.Minute), + // MeshMessageDeliveriesCap: 10, // 10 blocks in a minute + // MeshMessageDeliveriesThreshold: 0.41666, // 10/12/2 blocks/min + // MeshMessageDeliveriesWindow: 10 * time.Millisecond, + // MeshMessageDeliveriesActivation: time.Minute, + // + // // decays after 15 min + // MeshFailurePenaltyWeight: -576, + // MeshFailurePenaltyDecay: pubsub.ScoreParameterDecay(15 * time.Minute), + + // invalid messages decay after 1 hour + InvalidMessageDeliveriesWeight: -1000, + InvalidMessageDeliveriesDecay: pubsub.ScoreParameterDecay(time.Hour), + }, + build.BlocksTopic(in.Nn): { // expected 10 blocks/min TopicWeight: 0.1, // max is 50, max mesh penalty is -10, single invalid message is -100 @@ -121,7 +189,7 @@ func GossipSub(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, nn dtyp InvalidMessageDeliveriesWeight: -1000, InvalidMessageDeliveriesDecay: pubsub.ScoreParameterDecay(time.Hour), }, - build.MessagesTopic(nn): { + build.MessagesTopic(in.Nn): { // expected > 1 tx/second TopicWeight: 0.05, // max is 25, max mesh penalty is -5, single invalid message is -100 @@ -166,7 +234,7 @@ func GossipSub(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, nn dtyp OpportunisticGraftThreshold: 5, }, ), - pubsub.WithPeerScoreInspect(sk.Update, 10*time.Second), + pubsub.WithPeerScoreInspect(in.Sk.Update, 10*time.Second), } // enable Peer eXchange on bootstrappers @@ -184,10 +252,10 @@ func GossipSub(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, nn dtyp } // direct peers - if cfg.DirectPeers != nil { + if in.Cfg.DirectPeers != nil { var directPeerInfo []peer.AddrInfo - for _, addr := range cfg.DirectPeers { + for _, addr := range in.Cfg.DirectPeers { a, err := ma.NewMultiaddr(addr) if err != nil { return nil, err @@ -205,8 +273,8 @@ func GossipSub(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, nn dtyp } // tracer - if cfg.RemoteTracer != "" { - a, err := ma.NewMultiaddr(cfg.RemoteTracer) + if in.Cfg.RemoteTracer != "" { + a, err := ma.NewMultiaddr(in.Cfg.RemoteTracer) if err != nil { return nil, err } @@ -216,7 +284,7 @@ func GossipSub(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, nn dtyp return nil, err } - tr, err := pubsub.NewRemoteTracer(context.TODO(), host, *pi) + tr, err := pubsub.NewRemoteTracer(context.TODO(), in.Host, *pi) if err != nil { return nil, err } @@ -229,7 +297,7 @@ func GossipSub(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, nn dtyp // in peer scores for debugging purposes -- this might be trigged by metrics collection // options = append(options, pubsub.WithPeerScoreInspect(XXX, time.Second)) - return pubsub.NewGossipSub(helpers.LifecycleCtx(mctx, lc), host, options...) + return pubsub.NewGossipSub(helpers.LifecycleCtx(in.Mctx, in.Lc), in.Host, options...) } func HashMsgId(m *pubsub_pb.Message) string { From 7762cd0a686a808a5f0f4f3e32acac402ac7d11d Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 8 Jun 2020 11:31:33 +0200 Subject: [PATCH 077/160] Assign positive scores to drand bootstrappers Signed-off-by: Jakub Sztandera --- build/bootstrap.go | 9 +++++++++ node/builder.go | 1 + node/modules/core.go | 4 ++++ node/modules/dtypes/bootstrap.go | 1 + node/modules/lp2p/pubsub.go | 11 +++++++++++ 5 files changed, 26 insertions(+) diff --git a/build/bootstrap.go b/build/bootstrap.go index ca6cc26ce..0710f0dc0 100644 --- a/build/bootstrap.go +++ b/build/bootstrap.go @@ -34,3 +34,12 @@ func BuiltinBootstrap() ([]peer.AddrInfo, error) { }) return out, err } + +func DrandBootstrap() ([]peer.AddrInfo, error) { + addrs := []string{ + "/dnsaddr/pl-eu.testnet.drand.sh/", + "/dnsaddr/pl-us.testnet.drand.sh/", + "/dnsaddr/pl-sin.testnet.drand.sh/", + } + return addrutil.ParseAddresses(context.TODO(), addrs) +} diff --git a/node/builder.go b/node/builder.go index cf772746d..f503bff5c 100644 --- a/node/builder.go +++ b/node/builder.go @@ -218,6 +218,7 @@ func Online() Option { // TODO: Fix offline mode Override(new(dtypes.BootstrapPeers), modules.BuiltinBootstrap), + Override(new(dtypes.DrandBootstrap), modules.DrandBootstrap), Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages), diff --git a/node/modules/core.go b/node/modules/core.go index 6521b60df..ca9872d90 100644 --- a/node/modules/core.go +++ b/node/modules/core.go @@ -89,3 +89,7 @@ func ConfigBootstrap(peers []string) func() (dtypes.BootstrapPeers, error) { func BuiltinBootstrap() (dtypes.BootstrapPeers, error) { return build.BuiltinBootstrap() } + +func DrandBootstrap() (dtypes.DrandBootstrap, error) { + return build.DrandBootstrap() +} diff --git a/node/modules/dtypes/bootstrap.go b/node/modules/dtypes/bootstrap.go index 13003f029..96cd2f673 100644 --- a/node/modules/dtypes/bootstrap.go +++ b/node/modules/dtypes/bootstrap.go @@ -3,5 +3,6 @@ package dtypes import "github.com/libp2p/go-libp2p-core/peer" type BootstrapPeers []peer.AddrInfo +type DrandBootstrap []peer.AddrInfo type Bootstrapper bool diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go index 76387dd1d..bf9d88d42 100644 --- a/node/modules/lp2p/pubsub.go +++ b/node/modules/lp2p/pubsub.go @@ -41,6 +41,7 @@ type GossipIn struct { Host host.Host Nn dtypes.NetworkName Bp dtypes.BootstrapPeers + Db dtypes.DrandBootstrap Cfg *config.Pubsub Sk *dtypes.ScoreKeeper } @@ -61,6 +62,11 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { for _, pi := range in.Bp { bootstrappers[pi.ID] = struct{}{} } + drandBootstrappers := make(map[peer.ID]struct{}) + for _, pi := range in.Db { + drandBootstrappers[pi.ID] = struct{}{} + } + isBootstrapNode := in.Cfg.Bootstrapper drandTopic, err := getDrandTopic() if err != nil { @@ -83,6 +89,11 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { return 2500 } + _, ok = drandBootstrappers[p] + if ok && !isBootstrapNode { + return 1500 + } + // TODO: we want to plug the application specific score to the node itself in order // to provide feedback to the pubsub system based on observed behaviour return 0 From 9d49c6861f7be4a96000d842e1b0eedfd1cf0219 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 8 Jun 2020 09:51:47 -0700 Subject: [PATCH 078/160] don't update the mining base if mining fails for some reason --- miner/miner.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/miner/miner.go b/miner/miner.go index e906e6e70..f2059ef7f 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -153,13 +153,14 @@ func (m *Miner) mine(ctx context.Context) { m.niceSleep(build.BlockDelay * time.Second) continue } - lastBase = *base b, err := m.mineOne(ctx, base) if err != nil { log.Errorf("mining block failed: %+v", err) + m.niceSleep(time.Second) continue } + lastBase = *base onDone(b != nil) From 28f895cec2af37d8a34c8ea1671eef0029b4830f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 8 Jun 2020 20:20:58 +0200 Subject: [PATCH 079/160] Update sector-storage with better post faults --- api/apistruct/struct.go | 5 +++++ cmd/lotus-bench/main.go | 4 ++-- go.mod | 4 ++-- go.sum | 6 ++++-- storage/wdpost_run.go | 7 +++++-- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index c41b6e229..88f23d724 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -218,6 +218,7 @@ type StorageMinerStruct struct { StorageBestAlloc func(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredProof, sealing stores.PathType) ([]stores.StorageInfo, error) `perm:"admin"` StorageReportHealth func(ctx context.Context, id stores.ID, report stores.HealthReport) error `perm:"admin"` StorageLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error `perm:"admin"` + StorageTryLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) (bool, error) `perm:"admin"` DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` @@ -823,6 +824,10 @@ func (c *StorageMinerStruct) StorageLock(ctx context.Context, sector abi.SectorI return c.Internal.StorageLock(ctx, sector, read, write) } +func (c *StorageMinerStruct) StorageTryLock(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) (bool, error) { + return c.Internal.StorageTryLock(ctx, sector, read, write) +} + func (c *StorageMinerStruct) MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error { return c.Internal.MarketImportDealData(ctx, propcid, path) } diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 5951bd6fa..7f6d0c5de 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -345,7 +345,7 @@ var sealBenchCmd = &cli.Command{ verifyWinningPost2 := time.Now() log.Info("computing window post snark (cold)") - wproof1, err := sb.GenerateWindowPoSt(context.TODO(), mid, sealedSectors, challenge[:]) + wproof1, _, err := sb.GenerateWindowPoSt(context.TODO(), mid, sealedSectors, challenge[:]) if err != nil { return err } @@ -353,7 +353,7 @@ var sealBenchCmd = &cli.Command{ windowpost1 := time.Now() log.Info("computing window post snark (hot)") - wproof2, err := sb.GenerateWindowPoSt(context.TODO(), mid, sealedSectors, challenge[:]) + wproof2, _, err := sb.GenerateWindowPoSt(context.TODO(), mid, sealedSectors, challenge[:]) if err != nil { return err } diff --git a/go.mod b/go.mod index 786afed49..39f5bff3d 100644 --- a/go.mod +++ b/go.mod @@ -29,9 +29,9 @@ require ( github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/sector-storage v0.0.0-20200605192746-4b9317d1f08f + github.com/filecoin-project/sector-storage v0.0.0-20200608203246-bccaf697129c github.com/filecoin-project/specs-actors v0.5.6 - github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 + github.com/filecoin-project/specs-storage v0.1.0 github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 diff --git a/go.sum b/go.sum index be456d1dd..f86e293ef 100644 --- a/go.sum +++ b/go.sum @@ -219,8 +219,8 @@ github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZO github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= github.com/filecoin-project/sector-storage v0.0.0-20200508203401-a74812ba12f3/go.mod h1:B+xzopr/oWZJz2hBL5Ekb7Obcum5ntmfbaAUlaaho28= -github.com/filecoin-project/sector-storage v0.0.0-20200605192746-4b9317d1f08f h1:UASwQxRVA9LEfpFhhh87RvZMaPRFfR7QlWXNd9u6Aj0= -github.com/filecoin-project/sector-storage v0.0.0-20200605192746-4b9317d1f08f/go.mod h1:hUQo3z/5B8t8Yl+XgIxBqgNfYtIcCYzKjWWY27dHBCk= +github.com/filecoin-project/sector-storage v0.0.0-20200608203246-bccaf697129c h1:xWl+lxNnPmRhHPvcIM10AJHHhdAEo8lrJjWdd7kpQzA= +github.com/filecoin-project/sector-storage v0.0.0-20200608203246-bccaf697129c/go.mod h1:SfaVNw/A6LwqqEt6wEMVBN5Grn3XC50j+yjmegHIe7Y= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= @@ -231,6 +231,8 @@ github.com/filecoin-project/specs-actors v0.5.6 h1:WlhtoXwFoKlP1b06NI4NJaxC4m9EX github.com/filecoin-project/specs-actors v0.5.6/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 h1:T3f/zkuvgtgqcXrb0NO3BicuveGOxxUAMPa/Yif2kuE= github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102/go.mod h1:xJ1/xl9+8zZeSSSFmDC3Wr6uusCTxyYPI0VeNVSFmPE= +github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sgOVdJbvFjOnD5w94= +github.com/filecoin-project/specs-storage v0.1.0/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa h1:kTCIBKMhhhVcjCh1ws72vpdDr4cdZjCyIUDFijuWuvA= github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa/go.mod h1:S0u14Wr55mpe22lElCSKbXrhtWg/jquVTTMhefQ8f4Q= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index ea95245dd..8cd9fdc5f 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -431,8 +431,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo return nil, err } - // TODO: Faults! - postOut, err := s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), ssi, abi.PoStRandomness(rand)) + postOut, postSkipped, err := s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), ssi, abi.PoStRandomness(rand)) if err != nil { return nil, xerrors.Errorf("running post failed: %w", err) } @@ -441,6 +440,10 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo return nil, xerrors.Errorf("received proofs back from generate window post") } + for _, sector := range postSkipped { + skipped.Set(uint64(sector.Number)) + } + elapsed := time.Since(tsStart) log.Infow("submitting window PoSt", "elapsed", elapsed) From 19013b09e0982dc79e128d7c13e6c13309be40b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 8 Jun 2020 13:11:41 +0200 Subject: [PATCH 080/160] v27 parameters --- build/proof-params/parameters.json | 180 ++++++++++++++--------------- 1 file changed, 90 insertions(+), 90 deletions(-) diff --git a/build/proof-params/parameters.json b/build/proof-params/parameters.json index 4ca3e6d2d..b632c17e8 100644 --- a/build/proof-params/parameters.json +++ b/build/proof-params/parameters.json @@ -1,152 +1,152 @@ { - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.params": { - "cid": "QmYkygifkXnrnsN4MJsjBFHTQJHx294CyikDgDK8nYxdGh", - "digest": "df3f30442a6d6b4192f5071fb17e820c", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.params": { + "cid": "QmeDRyxek34F1H6xJY6AkFdWvPsy5F6dKTrebV3ZtWT4ky", + "digest": "f5827f2d8801c62c831e0f972f6dc8bb", "sector_size": 2048 }, - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.vk": { - "cid": "QmdXyqbmy2bkJA9Kyhh6z25GrTCq48LwX6c1mxPsm54wi7", - "digest": "0bea3951abf9557a3569f68e52a30c6c", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.vk": { + "cid": "QmUw1ZmG4BBbX19MsbH3zAEGKUc42iFJc5ZAyomDHeJTsA", + "digest": "398fecdb4b2de445125852bc3c080b35", "sector_size": 2048 }, - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0cfb4f178bbb71cf2ecfcd42accce558b27199ab4fb59cb78f2483fe21ef36d9.params": { - "cid": "Qmf5XZZtP5VcYTf65MbKjLVabcS6cYMbr2rFShmfJzh5e5", - "digest": "655e6277638edc8c658094f6f0b33d54", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0cfb4f178bbb71cf2ecfcd42accce558b27199ab4fb59cb78f2483fe21ef36d9.params": { + "cid": "QmUeNKp9YZpiAFm81RV5KuxH1FDGJx2DuwcbU2XNSZLLSv", + "digest": "2b6d2972ac9e862e8134d98fb695b0c5", "sector_size": 536870912 }, - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0cfb4f178bbb71cf2ecfcd42accce558b27199ab4fb59cb78f2483fe21ef36d9.vk": { - "cid": "QmPuhdWnAXBks43emnkqi9FQzyU1gASKyz23zrD27BPGs8", - "digest": "57690e3a6a94c3f704802a674b34f36b", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0cfb4f178bbb71cf2ecfcd42accce558b27199ab4fb59cb78f2483fe21ef36d9.vk": { + "cid": "QmQaQmTXX995Akd66ggtJY5bNx6Gkxk8P34JTdMMq8393G", + "digest": "3688c9eb256b7b17f411dad78d5ef74a", "sector_size": 536870912 }, - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-3ea05428c9d11689f23529cde32fd30aabd50f7d2c93657c1d3650bca3e8ea9e.params": { - "cid": "QmPNVgTN7N5vDtD5u7ERMTLcvUtrKRBfYVUDr6uW3pKhX7", - "digest": "3d390654f58e603b896ac70c653f5676", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-3ea05428c9d11689f23529cde32fd30aabd50f7d2c93657c1d3650bca3e8ea9e.params": { + "cid": "QmfEYTMSkwGJTumQx26iKXGNKiYh3mmAC4SkdybZpJCj5p", + "digest": "09bff16aed893349d94485cfae366a9c", "sector_size": 2048 }, - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-3ea05428c9d11689f23529cde32fd30aabd50f7d2c93657c1d3650bca3e8ea9e.vk": { - "cid": "Qmbj61Zez7v5xA7nSCnmWbyLYznWJDWeusz7Yg8EcgVdoN", - "digest": "8c170a164743c39576a7f47a1b51e6f3", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-3ea05428c9d11689f23529cde32fd30aabd50f7d2c93657c1d3650bca3e8ea9e.vk": { + "cid": "QmP4ThPieSUJyRanjibWpT5R5cCMzMAU4j8Y7kBn7CSW1Q", + "digest": "142f2f7e8f1b1779290315cabfd2c803", "sector_size": 2048 }, - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-50c7368dea9593ed0989e70974d28024efa9d156d585b7eea1be22b2e753f331.params": { - "cid": "QmRApb8RZoBK3cqicT7V3ydXg8yVvqPFMPrQNXP33aBihp", - "digest": "b1b58ff9a297b82885e8a7dfb035f83c", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-50c7368dea9593ed0989e70974d28024efa9d156d585b7eea1be22b2e753f331.params": { + "cid": "QmcAixrHsz29DgvtZiMc2kQjvPRvWxYUp36QYmRDZbmREm", + "digest": "8f987f64d434365562180b96ec12e299", "sector_size": 8388608 }, - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-50c7368dea9593ed0989e70974d28024efa9d156d585b7eea1be22b2e753f331.vk": { - "cid": "QmcytF1dTdqMFoyXi931j1RgmGtLfR9LLLaBznRt1tPQyD", - "digest": "1a09e00c641f192f55af3433a028f050", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-50c7368dea9593ed0989e70974d28024efa9d156d585b7eea1be22b2e753f331.vk": { + "cid": "QmT4iFnbL6r4txS5PXsiV7NTzbhCxHy54PvdkJJGV2VFXb", + "digest": "94b6c24ac01924f4feeecedd16b5d77d", "sector_size": 8388608 }, - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-5294475db5237a2e83c3e52fd6c2b03859a1831d45ed08c4f35dbf9a803165a9.params": { - "cid": "QmPvr54tWaVeP4WnekivzUAJitTqsQfvikBvAHNEaDNQSw", - "digest": "9380e41368ed4083dbc922b290d3b786", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-5294475db5237a2e83c3e52fd6c2b03859a1831d45ed08c4f35dbf9a803165a9.params": { + "cid": "QmbjFst6SFCK1KsTQrfwPdxf3VTNa1raed574tEZZ9PoyQ", + "digest": "2c245fe8179839dd6c6cdea207c67ae8", "sector_size": 8388608 }, - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-5294475db5237a2e83c3e52fd6c2b03859a1831d45ed08c4f35dbf9a803165a9.vk": { - "cid": "QmXyVLVDRCcxA9SjT7PeK8HFtyxZ2ZH3SHa8KoGLw8VGJt", - "digest": "f0731a7e20f90704bd38fc5d27882f6d", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-5294475db5237a2e83c3e52fd6c2b03859a1831d45ed08c4f35dbf9a803165a9.vk": { + "cid": "QmQJKmvZN1a5cQ1Nw6CDyXs3nuRPzvyU5NvCFMUL2BfcZC", + "digest": "56ae47bfda53bb8d22981ed8d8d27d72", "sector_size": 8388608 }, - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-7d739b8cf60f1b0709eeebee7730e297683552e4b69cab6984ec0285663c5781.params": { - "cid": "Qmf5f6ko3dqj7qauzXpZqxM9B2x2sL977K6gE2ppNwuJPv", - "digest": "273ebb8c896326b7c292bee8b775fd38", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-7d739b8cf60f1b0709eeebee7730e297683552e4b69cab6984ec0285663c5781.params": { + "cid": "QmQCABxeTpdvXTyjDyk7nPBxkQzCh7MXfGztWnSXEPKMLW", + "digest": "7e6b2eb5ecbb11ac651ad66ebbb2075a", "sector_size": 536870912 }, - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-7d739b8cf60f1b0709eeebee7730e297683552e4b69cab6984ec0285663c5781.vk": { - "cid": "QmfP3MQe8koW63n5MkDENENVHxib78MJYYyZvbneCsuze8", - "digest": "3dd94da9da64e51b3445bc528d84e76d", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-7d739b8cf60f1b0709eeebee7730e297683552e4b69cab6984ec0285663c5781.vk": { + "cid": "QmPBweyugh5Sx4umk8ULhgEGbjY8xmWLfU6M7EMpc8Mad6", + "digest": "94a8d9e25a9ab9674d339833664eba25", "sector_size": 536870912 }, - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-0377ded656c6f524f1618760bffe4e0a1c51d5a70c4509eedae8a27555733edc.params": { - "cid": "QmYEeeCE8uT2bsVkxcqqUYeMmMEbe6rfmo8wQCv7jFHqqm", - "digest": "c947f2021304ed43b7216f7a8436e294", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-0377ded656c6f524f1618760bffe4e0a1c51d5a70c4509eedae8a27555733edc.params": { + "cid": "QmY5yax1E9KymBnCeHksE9Zi8NieZbmwcpoDGoabkeeb9h", + "digest": "c909ea9e3fe25ab9b391a64593afdbba", "sector_size": 34359738368 }, - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-0377ded656c6f524f1618760bffe4e0a1c51d5a70c4509eedae8a27555733edc.vk": { - "cid": "QmXB63ExriFjB4ywWnXTnFwCcLFfCeEP3h15qtL5i7F4aX", - "digest": "ab20d7b253e7e9a0d2ccdf7599ec8ec3", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-0377ded656c6f524f1618760bffe4e0a1c51d5a70c4509eedae8a27555733edc.vk": { + "cid": "QmXnPo4yH5mwMguwrvqgRfduSttbmPrXtbBfbwU21wQWHt", + "digest": "caf900461e988bbf86dbcaca087b7864", "sector_size": 34359738368 }, - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-559e581f022bb4e4ec6e719e563bf0e026ad6de42e56c18714a2c692b1b88d7e.params": { - "cid": "QmW5Yxg3L1NSzuQVcRMHMbG3uvVoi4dTLzVaDpnEUPQpnA", - "digest": "079ba19645828ae42b22b0e3f4866e8d", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-559e581f022bb4e4ec6e719e563bf0e026ad6de42e56c18714a2c692b1b88d7e.params": { + "cid": "QmZtzzPWwmZEgR7MSMvXRbt9KVK8k4XZ5RLWHybHJW9SdE", + "digest": "a2844f0703f186d143a06146a04577d8", "sector_size": 34359738368 }, - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-559e581f022bb4e4ec6e719e563bf0e026ad6de42e56c18714a2c692b1b88d7e.vk": { - "cid": "QmQzZ5dJ11tcSBees38WX41tZLXS9BqpEti253m5QcnTNs", - "digest": "c76125a50a7de315165de359b5174ae4", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-559e581f022bb4e4ec6e719e563bf0e026ad6de42e56c18714a2c692b1b88d7e.vk": { + "cid": "QmWxEA7EdQCUJTzjNpxg5XTF45D2uVyYnN1QRUb5TRYU8M", + "digest": "2306247a1e616dbe07f01b88196c2044", "sector_size": 34359738368 }, - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-2627e4006b67f99cef990c0a47d5426cb7ab0a0ad58fc1061547bf2d28b09def.params": { - "cid": "QmNk3wga1tS53FUu1QnkK8ehWA2cqpCnSEAPv3KLxdJxNa", - "digest": "421e4790c0b80e0107a7ff67acf14084", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-2627e4006b67f99cef990c0a47d5426cb7ab0a0ad58fc1061547bf2d28b09def.params": { + "cid": "QmP676KwuvyF9Y64uJnXvLtvD1xcuWQ6wD23RzYtQ6dd4f", + "digest": "215b1c667a4f46a1d0178338df568615", "sector_size": 68719476736 }, - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-2627e4006b67f99cef990c0a47d5426cb7ab0a0ad58fc1061547bf2d28b09def.vk": { - "cid": "QmVQCHGsrUtbn9RjHs1e6GXfeXDW5m9w4ge48PSX3Z2as2", - "digest": "8b60e9cc1470a6729c687d6cf0a1f79c", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-2627e4006b67f99cef990c0a47d5426cb7ab0a0ad58fc1061547bf2d28b09def.vk": { + "cid": "QmPvPwbJtcSGyqB1rQJhSF5yvFbX9ZBSsHVej5F8JUyHUJ", + "digest": "0c9c423b28b1455fcbc329a1045fd4dd", "sector_size": 68719476736 }, - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-b62098629d07946e9028127e70295ed996fe3ed25b0f9f88eb610a0ab4385a3c.params": { - "cid": "QmTL3VvydaMFWKvE5VzxjgKsJYgL9JMM4JVYNtQxdj9JK1", - "digest": "2685f31124b22ea6b2857e5a5e87ffa3", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-b62098629d07946e9028127e70295ed996fe3ed25b0f9f88eb610a0ab4385a3c.params": { + "cid": "QmUxPQfvckzm1t6MFRdDZ1fDK5UJzAjK7pTZ97cwyachdr", + "digest": "965132f51ae445b0e6d32692b7561995", "sector_size": 68719476736 }, - "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-b62098629d07946e9028127e70295ed996fe3ed25b0f9f88eb610a0ab4385a3c.vk": { - "cid": "QmSVWbLqQYbUbbJyfsRMzEib2rfSqMtnPks1Nw22omcBQm", - "digest": "efe703cd2839597c7ca5c2a906b74296", + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-b62098629d07946e9028127e70295ed996fe3ed25b0f9f88eb610a0ab4385a3c.vk": { + "cid": "QmTxq2EBnQWb5R8tS4MHdchj4vNfLYGoSXxwJFvs5xgW4K", + "digest": "fc8c3d26e0e56373ad96cb41520d55a6", "sector_size": 68719476736 }, - "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-032d3138d22506ec0082ed72b2dcba18df18477904e35bafee82b3793b06832f.params": { - "cid": "QmU9dH31nZZUJnsogR4Ld4ySUcH6wm2RgmGiujwnqtbU6k", - "digest": "fcef8e87ae2afd7a28aae44347b804cf", + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-032d3138d22506ec0082ed72b2dcba18df18477904e35bafee82b3793b06832f.params": { + "cid": "QmRjgZHERgqGoRagR788Kh6ybi26csVYa8mqbqhmZm57Jx", + "digest": "cfc7b0897d1eee48c586f7beb89e67f7", "sector_size": 2048 }, - "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-032d3138d22506ec0082ed72b2dcba18df18477904e35bafee82b3793b06832f.vk": { - "cid": "QmdJ15DMGPooye5NaPcRfXUdHUDibcN7hKjbmTGuu1K4AQ", - "digest": "2ee2b3518229680db15161d4f582af37", + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-032d3138d22506ec0082ed72b2dcba18df18477904e35bafee82b3793b06832f.vk": { + "cid": "QmNjvnvFP7KgovHUddULoB19fBHT81iz7NcUbzEHZUUPsm", + "digest": "fb59bd061c987eac7068008c44de346b", "sector_size": 2048 }, - "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-6babf46ce344ae495d558e7770a585b2382d54f225af8ed0397b8be7c3fcd472.params": { - "cid": "QmZgtxcY3tMXXQxZTA7ZTUDXLVUnfxNcerXgeW4gG2NnfP", - "digest": "3273c7135cb75684248b475781b738ee", + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-6babf46ce344ae495d558e7770a585b2382d54f225af8ed0397b8be7c3fcd472.params": { + "cid": "QmTpRPBA4dt8fgGpcVzi4L1KA1U2eBHCE8WVmS2GUygMvT", + "digest": "36d465915b0afbf96bd08e7915e00952", "sector_size": 536870912 }, - "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-6babf46ce344ae495d558e7770a585b2382d54f225af8ed0397b8be7c3fcd472.vk": { - "cid": "QmSS6ZkAV2aGZcgKgdPpEEgihXF1ryZX8PSAZDWSoeL1d4", - "digest": "1519b5f61d9044a59f2bdc57537c094b", + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-6babf46ce344ae495d558e7770a585b2382d54f225af8ed0397b8be7c3fcd472.vk": { + "cid": "QmRzDyVfQCLsxspoVsed5bcQRsG6KiktngJfcNBL3TJPZe", + "digest": "99d16df0eb6a7e227a4f4570c4f6b6f1", "sector_size": 536870912 }, - "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-ecd683648512ab1765faa2a5f14bab48f676e633467f0aa8aad4b55dcb0652bb.params": { - "cid": "QmQBGXeiNn6hVwbR6qFarQqiNGDdKk4h9ucfyvcXyfYz2N", - "digest": "7d5f896f435c38e93bcda6dd168d860b", + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-ecd683648512ab1765faa2a5f14bab48f676e633467f0aa8aad4b55dcb0652bb.params": { + "cid": "QmV8ZjTSGzDUWmFvsq9NSyPBR7eDDUcvCPNgj2yE7HMAFu", + "digest": "34f3ddf1d1c9f41c0cd73b91e8b4bc27", "sector_size": 8388608 }, - "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-ecd683648512ab1765faa2a5f14bab48f676e633467f0aa8aad4b55dcb0652bb.vk": { - "cid": "QmPrZgBVGMckEAeu5eSJnLmiAwcPQjKjZe5ir6VaQ5AxKs", - "digest": "fe6d2de44580a0db5a4934688899b92f", + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-ecd683648512ab1765faa2a5f14bab48f676e633467f0aa8aad4b55dcb0652bb.vk": { + "cid": "QmTa3VbjTiqJWU6r4WKayaQrUaaBsrpp5UDqYvPDd2C5hs", + "digest": "ec62d59651daa5631d3d1e9c782dd940", "sector_size": 8388608 }, - "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-0-sha256_hasher-82a357d2f2ca81dc61bb45f4a762807aedee1b0a53fd6c4e77b46a01bfef7820.params": { - "cid": "QmZL2cq45XJn5BFzagAZwgFmLrcM1W6CXoiEF9C5j5tjEF", - "digest": "acdfed9f0512bc85a01a9fb871d475d5", + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-0-sha256_hasher-82a357d2f2ca81dc61bb45f4a762807aedee1b0a53fd6c4e77b46a01bfef7820.params": { + "cid": "Qmf8ngfArxrv9tFWDqBcNegdBMymvuakwyHKd1pbW3pbsb", + "digest": "a16d6f4c6424fb280236739f84b24f97", "sector_size": 34359738368 }, - "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-0-sha256_hasher-82a357d2f2ca81dc61bb45f4a762807aedee1b0a53fd6c4e77b46a01bfef7820.vk": { - "cid": "QmQ4zB7nNa1tDYNifBkExRnZtwtxZw775iaqvVsZyRi6Q2", - "digest": "524a2f3e9d6826593caebc41bb545c40", + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-0-sha256_hasher-82a357d2f2ca81dc61bb45f4a762807aedee1b0a53fd6c4e77b46a01bfef7820.vk": { + "cid": "QmfQgVFerArJ6Jupwyc9tKjLD9n1J9ajLHBdpY465tRM7M", + "digest": "7a139d82b8a02e35279d657e197f5c1f", "sector_size": 34359738368 }, - "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-2-sha256_hasher-96f1b4a04c5c51e4759bbf224bbc2ef5a42c7100f16ec0637123f16a845ddfb2.params": { - "cid": "QmY7DitNKXFeLQt9QoVQkfjM1EvRnprqUVxjmkTXkHDNka", - "digest": "f27271c0537ba65ade2ec045f8fbd069", + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-2-sha256_hasher-96f1b4a04c5c51e4759bbf224bbc2ef5a42c7100f16ec0637123f16a845ddfb2.params": { + "cid": "QmfDha8271nXJn14Aq3qQeghjMBWbs6HNSGa6VuzCVk4TW", + "digest": "5d3cd3f107a3bea8a96d1189efd2965c", "sector_size": 68719476736 }, - "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-2-sha256_hasher-96f1b4a04c5c51e4759bbf224bbc2ef5a42c7100f16ec0637123f16a845ddfb2.vk": { - "cid": "QmUJsvoCuQ4LszPmeRVAkMYb5qY95ctz3UXKhu8xLzyFKo", - "digest": "576b292938c6c9d0a0e721bd867a543b", + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-2-sha256_hasher-96f1b4a04c5c51e4759bbf224bbc2ef5a42c7100f16ec0637123f16a845ddfb2.vk": { + "cid": "QmRVtTtiFzHJTHurYzaCvetGAchux9cktixT4aGHthN6Zt", + "digest": "62c366405404e60f171e661492740b1c", "sector_size": 68719476736 } } \ No newline at end of file From 6e11ff37962d101f5462a4bc5862b27112bc7ac6 Mon Sep 17 00:00:00 2001 From: Jim Pick Date: Mon, 8 Jun 2020 17:03:11 -0700 Subject: [PATCH 081/160] Make libp2p swarm Announce / NoAnnounce addresses configurable In go-ipfs, it is possible to manually configure addresses to announce or not announce to the swarm. https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#addressesannounce This PR adds the same feature to Lotus. It can be configured from the [libp2p] section of either the .lotus or .lotusstorage config.toml file. Example: [Libp2p] ListenAddresses = ["/ip4/0.0.0.0/tcp/34515", "/ip6/::/tcp/34515"] AnnounceAddresses = ["/ip4/52.13.91.110/tcp/34515"] --- documentation/en/setting-a-static-port.md | 14 ++++++++++++++ node/builder.go | 3 +++ node/config/def.go | 10 +++++++--- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/documentation/en/setting-a-static-port.md b/documentation/en/setting-a-static-port.md index 43a38daa9..714f455b0 100644 --- a/documentation/en/setting-a-static-port.md +++ b/documentation/en/setting-a-static-port.md @@ -15,6 +15,20 @@ To change the port to `1347`: After changing the port value, restart your **daemon**. +## Announce Addresses + +If the **swarm port** is port-forwarded from another address, it is possible to control what addresses +are announced to the network. + +```sh +[Libp2p] + AnnounceAddresses = ["/ip4//tcp/1347"] +``` + +If non-empty, this array specifies the swarm addresses to announce to the network. If empty, the daemon will announce inferred swarm addresses. + +Similarly, it is possible to set `NoAnnounceAddresses` with an array of addresses to not announce to the network. + ## Ubuntu's Uncomplicated Firewall Open firewall manually: diff --git a/node/builder.go b/node/builder.go index f503bff5c..35ef69bfe 100644 --- a/node/builder.go +++ b/node/builder.go @@ -371,6 +371,9 @@ func ConfigCommon(cfg *config.Common) Option { Override(new(dtypes.BootstrapPeers), modules.ConfigBootstrap(cfg.Libp2p.BootstrapPeers)), ), ), + Override(AddrsFactoryKey, lp2p.AddrsFactory( + cfg.Libp2p.AnnounceAddresses, + cfg.Libp2p.NoAnnounceAddresses)), ) } diff --git a/node/config/def.go b/node/config/def.go index 9ca97bb47..575284b95 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -39,9 +39,11 @@ type API struct { // Libp2p contains configs for libp2p type Libp2p struct { - ListenAddresses []string - BootstrapPeers []string - ProtectedPeers []string + ListenAddresses []string + AnnounceAddresses []string + NoAnnounceAddresses []string + BootstrapPeers []string + ProtectedPeers []string ConnMgrLow uint ConnMgrHigh uint @@ -78,6 +80,8 @@ func defCommon() Common { "/ip4/0.0.0.0/tcp/0", "/ip6/::/tcp/0", }, + AnnounceAddresses: []string{}, + NoAnnounceAddresses: []string{}, ConnMgrLow: 150, ConnMgrHigh: 180, From 5feea2e0f8fbbdadf2b5afd88171265ef214cd88 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 9 Jun 2020 22:55:56 +0200 Subject: [PATCH 082/160] Update go-log to master It restores line informations in log messages Signed-off-by: Jakub Sztandera --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 786afed49..70a5c839b 100644 --- a/go.mod +++ b/go.mod @@ -67,7 +67,7 @@ require ( github.com/ipfs/go-ipld-cbor v0.0.5-0.20200428170625-a0bd04d3cbdf github.com/ipfs/go-ipld-format v0.2.0 github.com/ipfs/go-log v1.0.4 - github.com/ipfs/go-log/v2 v2.1.1 + github.com/ipfs/go-log/v2 v2.1.2-0.20200609205458-f8d20c392cb7 github.com/ipfs/go-merkledag v0.3.1 github.com/ipfs/go-path v0.0.7 github.com/ipfs/go-unixfs v0.2.4 diff --git a/go.sum b/go.sum index be456d1dd..1b6b9f0f0 100644 --- a/go.sum +++ b/go.sum @@ -527,8 +527,8 @@ github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBW github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= github.com/ipfs/go-log/v2 v2.0.8/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= github.com/ipfs/go-log/v2 v2.1.0/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= -github.com/ipfs/go-log/v2 v2.1.1 h1:G4TtqN+V9y9HY9TA6BwbCVyyBZ2B9MbCjR2MtGx8FR0= -github.com/ipfs/go-log/v2 v2.1.1/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= +github.com/ipfs/go-log/v2 v2.1.2-0.20200609205458-f8d20c392cb7 h1:LtL/rvdfbKSthZGmAAD9o4KKg6HA6Qn8gXCCdgnj7lw= +github.com/ipfs/go-log/v2 v2.1.2-0.20200609205458-f8d20c392cb7/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/ipfs/go-merkledag v0.0.3/go.mod h1:Oc5kIXLHokkE1hWGMBHw+oxehkAaTOqtEb7Zbh6BhLA= github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto= github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= From 5cf76ba8b3737ce8d6c6ab1c2dd63d75b57cb517 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 8 Jun 2020 21:03:44 -0400 Subject: [PATCH 083/160] Create an api.MinerInfo that has peerID as a Peer ID --- api/api_full.go | 35 ++++++++++++++++++++++++++++++++++- api/apistruct/struct.go | 4 ++-- cli/state.go | 14 +------------- node/impl/full/state.go | 11 ++++++++--- storage/miner.go | 2 +- 5 files changed, 46 insertions(+), 20 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index dca416b94..746e2b166 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -146,7 +146,7 @@ type FullNode interface { StateMinerProvingSet(context.Context, address.Address, types.TipSetKey) ([]*ChainSectorInfo, error) StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error) StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error) - StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) + StateMinerInfo(context.Context, address.Address, types.TipSetKey) (MinerInfo, error) StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) (*miner.Deadlines, error) StateMinerFaults(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error) // Returns all non-expired Faults that occur within lookback epochs of the given tipset @@ -299,6 +299,39 @@ type MinerPower struct { TotalPower power.Claim } +type MinerInfo struct { + Owner address.Address // Must be an ID-address. + Worker address.Address // Must be an ID-address. + NewWorker address.Address // Must be an ID-address. + WorkerChangeEpoch abi.ChainEpoch + PeerId peer.ID + Multiaddrs []abi.Multiaddrs + SealProofType abi.RegisteredProof + SectorSize abi.SectorSize + WindowPoStPartitionSectors uint64 +} + +func NewApiMinerInfo(info miner.MinerInfo) MinerInfo { + mi := MinerInfo{ + Owner: info.Owner, + Worker: info.Worker, + NewWorker: address.Undef, + WorkerChangeEpoch: -1, + PeerId: peer.ID(info.PeerId), + Multiaddrs: info.Multiaddrs, + SealProofType: info.SealProofType, + SectorSize: info.SectorSize, + WindowPoStPartitionSectors: info.WindowPoStPartitionSectors, + } + + if info.PendingWorkerKey != nil { + mi.NewWorker = info.PendingWorkerKey.NewWorker + mi.WorkerChangeEpoch = info.PendingWorkerKey.EffectiveAt + } + + return mi +} + type QueryOffer struct { Err string diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 88f23d724..31cd6badd 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -125,7 +125,7 @@ type FullNodeStruct struct { StateMinerProvingSet func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"` StateMinerProvingDeadline func(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error) `perm:"read"` StateMinerPower func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) `perm:"read"` - StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) `perm:"read"` + StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) `perm:"read"` StateMinerDeadlines func(context.Context, address.Address, types.TipSetKey) (*miner.Deadlines, error) `perm:"read"` StateMinerFaults func(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error) `perm:"read"` StateAllMinerFaults func(context.Context, abi.ChainEpoch, types.TipSetKey) ([]*api.Fault, error) `perm:"read"` @@ -552,7 +552,7 @@ func (c *FullNodeStruct) StateMinerPower(ctx context.Context, a address.Address, return c.Internal.StateMinerPower(ctx, a, tsk) } -func (c *FullNodeStruct) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) { +func (c *FullNodeStruct) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (api.MinerInfo, error) { return c.Internal.StateMinerInfo(ctx, actor, tsk) } diff --git a/cli/state.go b/cli/state.go index c0cfd8eb4..6720ceacd 100644 --- a/cli/state.go +++ b/cli/state.go @@ -145,23 +145,11 @@ var stateMinerInfo = &cli.Command{ return err } - act, err := api.StateGetActor(ctx, addr, ts.Key()) + mi, err := api.StateMinerInfo(ctx, addr, ts.Key()) if err != nil { return err } - aso, err := api.ChainReadObj(ctx, act.Head) - if err != nil { - return err - } - - var mst miner2.State - if err := mst.UnmarshalCBOR(bytes.NewReader(aso)); err != nil { - return err - } - - mi := mst.Info - fmt.Printf("Owner:\t%s\n", mi.Owner) fmt.Printf("Worker:\t%s\n", mi.Worker) fmt.Printf("PeerID:\t%s\n", mi.PeerId) diff --git a/node/impl/full/state.go b/node/impl/full/state.go index c4a9f67fd..fae2a7b67 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -77,12 +77,17 @@ func (a *StateAPI) StateMinerProvingSet(ctx context.Context, addr address.Addres return stmgr.GetProvingSetRaw(ctx, a.StateManager, mas) } -func (a *StateAPI) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) { +func (a *StateAPI) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (api.MinerInfo, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { - return miner.MinerInfo{}, xerrors.Errorf("loading tipset %s: %w", tsk, err) + return api.MinerInfo{}, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return stmgr.StateMinerInfo(ctx, a.StateManager, ts, actor) + + mi, err := stmgr.StateMinerInfo(ctx, a.StateManager, ts, actor) + if err != nil { + return api.MinerInfo{}, err + } + return api.NewApiMinerInfo(mi), nil } func (a *StateAPI) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) (*miner.Deadlines, error) { diff --git a/storage/miner.go b/storage/miner.go index 117c91a57..171c166aa 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -51,7 +51,7 @@ type storageMinerApi interface { StateMinerSectors(context.Context, address.Address, *abi.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error) StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) - StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) + StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error) StateMinerInitialPledgeCollateral(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (types.BigInt, error) StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) // TODO: removeme eventually From 1ccad2222752aa7cf4b7ee6fcf520a5da32d64db Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 9 Jun 2020 19:08:43 -0400 Subject: [PATCH 084/160] Move api.MinerInfo to api/types --- api/api_full.go | 33 --------------------------------- api/types.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 746e2b166..63d9a5e5d 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -299,39 +299,6 @@ type MinerPower struct { TotalPower power.Claim } -type MinerInfo struct { - Owner address.Address // Must be an ID-address. - Worker address.Address // Must be an ID-address. - NewWorker address.Address // Must be an ID-address. - WorkerChangeEpoch abi.ChainEpoch - PeerId peer.ID - Multiaddrs []abi.Multiaddrs - SealProofType abi.RegisteredProof - SectorSize abi.SectorSize - WindowPoStPartitionSectors uint64 -} - -func NewApiMinerInfo(info miner.MinerInfo) MinerInfo { - mi := MinerInfo{ - Owner: info.Owner, - Worker: info.Worker, - NewWorker: address.Undef, - WorkerChangeEpoch: -1, - PeerId: peer.ID(info.PeerId), - Multiaddrs: info.Multiaddrs, - SealProofType: info.SealProofType, - SectorSize: info.SectorSize, - WindowPoStPartitionSectors: info.WindowPoStPartitionSectors, - } - - if info.PendingWorkerKey != nil { - mi.NewWorker = info.PendingWorkerKey.NewWorker - mi.WorkerChangeEpoch = info.PendingWorkerKey.EffectiveAt - } - - return mi -} - type QueryOffer struct { Err string diff --git a/api/types.go b/api/types.go index fcc013441..f5a595353 100644 --- a/api/types.go +++ b/api/types.go @@ -2,6 +2,9 @@ package api import ( "encoding/json" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/libp2p/go-libp2p-core/peer" ma "github.com/multiformats/go-multiaddr" @@ -39,3 +42,36 @@ type PubsubScore struct { ID peer.ID Score float64 } + +type MinerInfo struct { + Owner address.Address // Must be an ID-address. + Worker address.Address // Must be an ID-address. + NewWorker address.Address // Must be an ID-address. + WorkerChangeEpoch abi.ChainEpoch + PeerId peer.ID + Multiaddrs []abi.Multiaddrs + SealProofType abi.RegisteredProof + SectorSize abi.SectorSize + WindowPoStPartitionSectors uint64 +} + +func NewApiMinerInfo(info miner.MinerInfo) MinerInfo { + mi := MinerInfo{ + Owner: info.Owner, + Worker: info.Worker, + NewWorker: address.Undef, + WorkerChangeEpoch: -1, + PeerId: peer.ID(info.PeerId), + Multiaddrs: info.Multiaddrs, + SealProofType: info.SealProofType, + SectorSize: info.SectorSize, + WindowPoStPartitionSectors: info.WindowPoStPartitionSectors, + } + + if info.PendingWorkerKey != nil { + mi.NewWorker = info.PendingWorkerKey.NewWorker + mi.WorkerChangeEpoch = info.PendingWorkerKey.EffectiveAt + } + + return mi +} From a00df0da900980891117107adb5a2a48fec3d45d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 8 Jun 2020 23:25:49 +0200 Subject: [PATCH 085/160] FFI v27 --- cmd/lotus-bench/main.go | 4 ++-- extern/filecoin-ffi | 2 +- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 7f6d0c5de..41af08352 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -412,7 +412,7 @@ var sealBenchCmd = &cli.Command{ fmt.Println(string(data)) } else { - fmt.Printf("----\nresults (v26) (%d)\n", sectorSize) + fmt.Printf("----\nresults (v27) (%d)\n", sectorSize) if robench == "" { fmt.Printf("seal: addPiece: %s (%s)\n", bo.SealingResults[0].AddPiece, bps(bo.SectorSize, bo.SealingResults[0].AddPiece)) // TODO: average across multiple sealings fmt.Printf("seal: preCommit phase 1: %s (%s)\n", bo.SealingResults[0].PreCommit1, bps(bo.SectorSize, bo.SealingResults[0].PreCommit1)) @@ -569,7 +569,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, mid } } - err := sb.UnsealPiece(context.TODO(), abi.SectorID{Miner: mid, Number: 1}, 0, abi.UnpaddedPieceSize(sectorSize), ticket, cids.Unsealed) + err := sb.UnsealPiece(context.TODO(), abi.SectorID{Miner: mid, Number: 1}, 0, abi.PaddedPieceSize(sectorSize).Unpadded(), ticket, cids.Unsealed) if err != nil { return nil, nil, err } diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 6109b6ad2..61c02f6be 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 6109b6ad2fa9968941c206161dd01ac059011d4e +Subproject commit 61c02f6bea8d69bb79c70daa1d62f26c486643aa diff --git a/go.mod b/go.mod index 39f5bff3d..03122bb73 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/sector-storage v0.0.0-20200608203246-bccaf697129c + github.com/filecoin-project/sector-storage v0.0.0-20200609231555-252c2b0c969d github.com/filecoin-project/specs-actors v0.5.6 github.com/filecoin-project/specs-storage v0.1.0 github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa diff --git a/go.sum b/go.sum index f86e293ef..99285c5c8 100644 --- a/go.sum +++ b/go.sum @@ -219,8 +219,8 @@ github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZO github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= github.com/filecoin-project/sector-storage v0.0.0-20200508203401-a74812ba12f3/go.mod h1:B+xzopr/oWZJz2hBL5Ekb7Obcum5ntmfbaAUlaaho28= -github.com/filecoin-project/sector-storage v0.0.0-20200608203246-bccaf697129c h1:xWl+lxNnPmRhHPvcIM10AJHHhdAEo8lrJjWdd7kpQzA= -github.com/filecoin-project/sector-storage v0.0.0-20200608203246-bccaf697129c/go.mod h1:SfaVNw/A6LwqqEt6wEMVBN5Grn3XC50j+yjmegHIe7Y= +github.com/filecoin-project/sector-storage v0.0.0-20200609231555-252c2b0c969d h1:q7KC8/yIirwVX7JCLgjEmo5LX77V8tYV0EzVCnCwrE8= +github.com/filecoin-project/sector-storage v0.0.0-20200609231555-252c2b0c969d/go.mod h1:SfaVNw/A6LwqqEt6wEMVBN5Grn3XC50j+yjmegHIe7Y= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= From e0a8e993bc9667187e2f5dd9b043aa2ea57fb4b2 Mon Sep 17 00:00:00 2001 From: yaohcn Date: Thu, 11 Jun 2020 16:35:46 +0800 Subject: [PATCH 086/160] bench prove nedd miner-addr flag --- cmd/lotus-bench/main.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 41af08352..7d7991636 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -598,6 +598,11 @@ var proveCmd = &cli.Command{ Name: "no-gpu", Usage: "disable gpu usage for the benchmark run", }, + &cli.StringFlag{ + Name: "miner-addr", + Usage: "pass miner address (only necessary if using existing sectorbuilder)", + Value: "t01000", + }, }, Action: func(c *cli.Context) error { if c.Bool("no-gpu") { @@ -659,7 +664,7 @@ var proveCmd = &cli.Command{ fmt.Printf("proof: %x\n", proof) - fmt.Printf("----\nresults (v23) (%d)\n", c2in.SectorSize) + fmt.Printf("----\nresults (v27) (%d)\n", c2in.SectorSize) dur := sealCommit2.Sub(start) fmt.Printf("seal: commit phase 2: %s (%s)\n", dur, bps(abi.SectorSize(c2in.SectorSize), dur)) From 2cd6347a134f0364de3a547360fb26e42ef3bbd7 Mon Sep 17 00:00:00 2001 From: Mike Greenberg Date: Thu, 4 Jun 2020 18:18:14 -0400 Subject: [PATCH 087/160] Express block validation, cpu/mem usage via OpenCensus --- chain/sync.go | 9 +++++++++ cmd/lotus/daemon.go | 8 ++++++++ metrics/metrics.go | 26 ++++++++++++++++---------- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index f8a73356f..960ee1ea1 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -528,8 +528,17 @@ func blockSanityChecks(h *types.BlockHeader) error { // ValidateBlock should match up with 'Semantical Validation' in validation.md in the spec func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) error { + validationStart := time.Now() + defer func() { + dur := time.Since(validationStart) + durMilli := dur.Seconds() * float64(1000) + stats.Record(ctx, metrics.BlockValidationDurationMilliseconds.M(durMilli)) + log.Infow("block validation", "took", dur, "height", b.Header.Height) + }() + ctx, span := trace.StartSpan(ctx, "validateBlock") defer span.End() + if build.InsecurePoStValidation { log.Warn("insecure test validation is enabled, if you see this outside of a test, it is a severe bug!") } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index fd9ceea85..912e65253 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -19,6 +19,7 @@ import ( "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" "github.com/urfave/cli/v2" + "go.opencensus.io/plugin/runmetrics" "go.opencensus.io/stats" "go.opencensus.io/stats/view" "go.opencensus.io/tag" @@ -114,6 +115,13 @@ var DaemonCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { + err := runmetrics.Enable(runmetrics.RunMetricOptions{ + EnableCPU: true, + EnableMemory: true, + }) + if err != nil { + return xerrors.Errorf("enabling runtime metrics: %w", err) + } if prof := cctx.String("pprof"); prof != "" { profile, err := os.Create(prof) if err != nil { diff --git a/metrics/metrics.go b/metrics/metrics.go index 340d57536..0ef63de4f 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -22,16 +22,17 @@ var ( // Measures var ( - LotusInfo = stats.Int64("info", "Arbitrary counter to tag lotus info to", stats.UnitDimensionless) - ChainNodeHeight = stats.Int64("chain/node_height", "Current Height of the node", stats.UnitDimensionless) - ChainNodeWorkerHeight = stats.Int64("chain/node_worker_height", "Current Height of workers on the node", stats.UnitDimensionless) - MessageReceived = stats.Int64("message/received", "Counter for total received messages", stats.UnitDimensionless) - MessageValidationFailure = stats.Int64("message/failure", "Counter for message validation failures", stats.UnitDimensionless) - MessageValidationSuccess = stats.Int64("message/success", "Counter for message validation successes", stats.UnitDimensionless) - BlockReceived = stats.Int64("block/received", "Counter for total received blocks", stats.UnitDimensionless) - BlockValidationFailure = stats.Int64("block/failure", "Counter for block validation failures", stats.UnitDimensionless) - BlockValidationSuccess = stats.Int64("block/success", "Counter for block validation successes", stats.UnitDimensionless) - PeerCount = stats.Int64("peer/count", "Current number of FIL peers", stats.UnitDimensionless) + LotusInfo = stats.Int64("info", "Arbitrary counter to tag lotus info to", stats.UnitDimensionless) + ChainNodeHeight = stats.Int64("chain/node_height", "Current Height of the node", stats.UnitDimensionless) + ChainNodeWorkerHeight = stats.Int64("chain/node_worker_height", "Current Height of workers on the node", stats.UnitDimensionless) + MessageReceived = stats.Int64("message/received", "Counter for total received messages", stats.UnitDimensionless) + MessageValidationFailure = stats.Int64("message/failure", "Counter for message validation failures", stats.UnitDimensionless) + MessageValidationSuccess = stats.Int64("message/success", "Counter for message validation successes", stats.UnitDimensionless) + BlockReceived = stats.Int64("block/received", "Counter for total received blocks", stats.UnitDimensionless) + BlockValidationFailure = stats.Int64("block/failure", "Counter for block validation failures", stats.UnitDimensionless) + BlockValidationSuccess = stats.Int64("block/success", "Counter for block validation successes", stats.UnitDimensionless) + BlockValidationDurationMilliseconds = stats.Float64("block/validation_ms", "Duration for Block Validation in ms", stats.UnitMilliseconds) + PeerCount = stats.Int64("peer/count", "Current number of FIL peers", stats.UnitDimensionless) ) var ( @@ -63,6 +64,10 @@ var ( Measure: BlockValidationSuccess, Aggregation: view.Count(), } + BlockValidationDurationView = &view.View{ + Measure: BlockValidationDurationMilliseconds, + Aggregation: view.Sum(), + } MessageReceivedView = &view.View{ Measure: MessageReceived, Aggregation: view.Count(), @@ -90,6 +95,7 @@ var DefaultViews = append([]*view.View{ BlockReceivedView, BlockValidationFailureView, BlockValidationSuccessView, + BlockValidationDurationView, MessageReceivedView, MessageValidationFailureView, MessageValidationSuccessView, From 512270593b77bd53224c67095762ffc8d565b85f Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 9 Jun 2020 12:49:31 -0700 Subject: [PATCH 088/160] clean up some of the more spammy logs --- chain/store/store.go | 5 ----- chain/sub/incoming.go | 5 +---- chain/sync_manager.go | 2 +- chain/vm/runtime.go | 4 ++-- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index 8f15b568f..7f552313f 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -18,7 +18,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/metrics" @@ -964,10 +963,6 @@ func (cs *ChainStore) GetTipsetByHeight(ctx context.Context, h abi.ChainEpoch, t return ts, nil } - if ts.Height()-h > build.ForkLengthThreshold { - log.Warnf("expensive call to GetTipsetByHeight, seeking %d levels", ts.Height()-h) - } - lbts, err := cs.cindex.GetTipsetByHeight(ctx, ts, h) if err != nil { return nil, err diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 8b5643de7..ba61dc5c5 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -60,8 +60,6 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha src := msg.GetFrom() go func() { - log.Infof("New block over pubsub: %s", blk.Cid()) - start := time.Now() log.Debug("about to fetch messages for block from pubsub") bmsgs, err := s.Bsync.FetchMessagesByCids(context.TODO(), blk.BlsMessages) @@ -145,8 +143,7 @@ func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub // track validation time begin := time.Now() defer func() { - end := time.Now() - log.Infof("block validation time: %s", end.Sub(begin)) + log.Debugf("block validation time: %s", time.Since(begin)) }() stats.Record(ctx, metrics.BlockReceived.M(1)) diff --git a/chain/sync_manager.go b/chain/sync_manager.go index e00063961..deb626fda 100644 --- a/chain/sync_manager.go +++ b/chain/sync_manager.go @@ -280,7 +280,7 @@ func (sm *SyncManager) syncScheduler() { } func (sm *SyncManager) scheduleIncoming(ts *types.TipSet) { - log.Info("scheduling incoming tipset sync: ", ts.Cids()) + log.Debug("scheduling incoming tipset sync: ", ts.Cids()) if sm.getBootstrapState() == BSStateSelected { sm.setBootstrapState(BSStateScheduled) sm.syncTargets <- ts diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index d9268afe4..6e425474c 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -139,7 +139,7 @@ func (rt *Runtime) shimCall(f func() interface{}) (rval []byte, aerr aerrors.Act defer func() { if r := recover(); r != nil { if ar, ok := r.(aerrors.ActorError); ok { - log.Errorf("VM.Call failure: %+v", ar) + log.Warnf("VM.Call failure: %+v", ar) aerr = ar return } @@ -309,7 +309,7 @@ func (rt *Runtime) Context() context.Context { } func (rt *Runtime) Abortf(code exitcode.ExitCode, msg string, args ...interface{}) { - log.Error("Abortf: ", fmt.Sprintf(msg, args...)) + log.Warnf("Abortf: ", fmt.Sprintf(msg, args...)) panic(aerrors.NewfSkip(2, code, msg, args...)) } From fe60ffb6ff016d7e26f55c76c3bf6c6835a37a16 Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 9 Jun 2020 11:04:17 -0700 Subject: [PATCH 089/160] display quantities of bytes in human-readable format - client commP - client local - client find - client query-ask - client list-deals - state sector-size --- cli/chain.go | 3 +-- cli/client.go | 10 +++++----- cli/state.go | 5 ++--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/cli/chain.go b/cli/chain.go index d76335263..01714d850 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -12,7 +12,6 @@ import ( "strings" "time" - "github.com/docker/go-units" "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/specs-actors/actors/abi" @@ -230,7 +229,7 @@ var chainStatObjCmd = &cli.Command{ } fmt.Printf("Links: %d\n", stats.Links) - fmt.Printf("Size: %s (%d)\n", units.BytesSize(float64(stats.Size)), stats.Size) + fmt.Printf("Size: %s (%d)\n", types.SizeStr(types.NewInt(stats.Size)), stats.Size) return nil }, } diff --git a/cli/client.go b/cli/client.go index d6b59db53..e512f9bd9 100644 --- a/cli/client.go +++ b/cli/client.go @@ -143,7 +143,7 @@ var clientCommPCmd = &cli.Command{ } fmt.Println("CID: ", encoder.Encode(ret.Root)) - fmt.Println("Piece size: ", ret.Size) + fmt.Println("Piece size: ", types.SizeStr(types.NewInt(uint64(ret.Size)))) return nil }, } @@ -203,7 +203,7 @@ var clientLocalCmd = &cli.Command{ } for _, v := range list { - fmt.Printf("%s %s %d %s\n", encoder.Encode(v.Key), v.FilePath, v.Size, v.Status) + fmt.Printf("%s %s %s %s\n", encoder.Encode(v.Key), v.FilePath, types.SizeStr(types.NewInt(v.Size)), v.Status) } return nil }, @@ -371,7 +371,7 @@ var clientFindCmd = &cli.Command{ fmt.Printf("ERR %s@%s: %s\n", offer.Miner, offer.MinerPeerID, offer.Err) continue } - fmt.Printf("RETRIEVAL %s@%s-%sfil-%db\n", offer.Miner, offer.MinerPeerID, types.FIL(offer.MinPrice), offer.Size) + fmt.Printf("RETRIEVAL %s@%s-%sfil-%s\n", offer.Miner, offer.MinerPeerID, types.FIL(offer.MinPrice), types.SizeStr(types.NewInt(offer.Size))) } return nil @@ -520,7 +520,7 @@ var clientQueryAskCmd = &cli.Command{ fmt.Printf("Ask: %s\n", maddr) fmt.Printf("Price per GiB: %s\n", types.FIL(ask.Ask.Price)) - fmt.Printf("Max Piece size: %d\n", ask.Ask.MaxPieceSize) + fmt.Printf("Max Piece size: %s\n", types.SizeStr(types.NewInt(uint64(ask.Ask.MaxPieceSize)))) size := cctx.Int64("size") if size == 0 { @@ -597,7 +597,7 @@ var clientListDeals = &cli.Command{ slashed = fmt.Sprintf("Y (epoch %d)", d.OnChainDealState.SlashEpoch) } - fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s\t%d\t%s\n", d.LocalDeal.ProposalCid, d.LocalDeal.DealID, d.LocalDeal.Provider, storagemarket.DealStates[d.LocalDeal.State], onChain, slashed, d.LocalDeal.PieceCID, d.LocalDeal.Size, d.LocalDeal.PricePerEpoch, d.LocalDeal.Duration, d.LocalDeal.Message) + fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n", d.LocalDeal.ProposalCid, d.LocalDeal.DealID, d.LocalDeal.Provider, storagemarket.DealStates[d.LocalDeal.State], onChain, slashed, d.LocalDeal.PieceCID, types.SizeStr(types.NewInt(d.LocalDeal.Size)), d.LocalDeal.PricePerEpoch, d.LocalDeal.Duration, d.LocalDeal.Message) } return w.Flush() }, diff --git a/cli/state.go b/cli/state.go index 6720ceacd..b741ddcd1 100644 --- a/cli/state.go +++ b/cli/state.go @@ -11,7 +11,6 @@ import ( "strings" "time" - "github.com/docker/go-units" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multihash" @@ -153,7 +152,7 @@ var stateMinerInfo = &cli.Command{ fmt.Printf("Owner:\t%s\n", mi.Owner) fmt.Printf("Worker:\t%s\n", mi.Worker) fmt.Printf("PeerID:\t%s\n", mi.PeerId) - fmt.Printf("SectorSize:\t%s (%d)\n", units.BytesSize(float64(mi.SectorSize)), mi.SectorSize) + fmt.Printf("SectorSize:\t%s (%d)\n", types.SizeStr(types.NewInt(uint64(mi.SectorSize))), mi.SectorSize) return nil }, @@ -707,7 +706,7 @@ var stateSectorSizeCmd = &cli.Command{ return err } - fmt.Printf("%d\n", mi.SectorSize) + fmt.Printf("%s (%d)\n", types.SizeStr(types.NewInt(uint64(mi.SectorSize))), mi.SectorSize) return nil }, } From dc112a5a00d3f34c2e671d543fad48d420e9e1e3 Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Tue, 9 Jun 2020 20:17:28 -0300 Subject: [PATCH 090/160] Lotus architecture notes (#1768) Co-authored-by: Aayush Rajasekaran --- chain/sync.go | 2 + chain/sync_manager.go | 1 + documentation/en/.library.json | 6 + documentation/en/architecture.md | 394 ++++++++++++++++++ .../en/dev/WIP-arch-complementary-notes.md | 153 +++++++ 5 files changed, 556 insertions(+) create mode 100644 documentation/en/architecture.md create mode 100644 documentation/en/dev/WIP-arch-complementary-notes.md diff --git a/chain/sync.go b/chain/sync.go index 960ee1ea1..03ccc3ab7 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -429,6 +429,8 @@ func (syncer *Syncer) Sync(ctx context.Context, maybeHead *types.TipSet) error { return xerrors.Errorf("collectChain failed: %w", err) } + // At this point we have accepted and synced to the new `maybeHead` + // (`StageSyncComplete`). if err := syncer.store.PutTipSet(ctx, maybeHead); err != nil { span.AddAttributes(trace.StringAttribute("put_error", err.Error())) span.SetStatus(trace.Status{ diff --git a/chain/sync_manager.go b/chain/sync_manager.go index deb626fda..e3fbdf4e1 100644 --- a/chain/sync_manager.go +++ b/chain/sync_manager.go @@ -35,6 +35,7 @@ type SyncManager struct { syncStates []*SyncerState + // Normally this handler is set to `(*Syncer).Sync()`. doSync func(context.Context, *types.TipSet) error stop chan struct{} diff --git a/documentation/en/.library.json b/documentation/en/.library.json index 018e6189b..f637c813f 100644 --- a/documentation/en/.library.json +++ b/documentation/en/.library.json @@ -71,6 +71,12 @@ ] }, { + "title": "Architecture", + "slug": "en+arch", + "github": "en/architecture.md", + "value": null, + "posts": [] + }, { "title": "Storage Mining", "slug": "en+mining", "github": "en/mining.md", diff --git a/documentation/en/architecture.md b/documentation/en/architecture.md new file mode 100644 index 000000000..970f2a13d --- /dev/null +++ b/documentation/en/architecture.md @@ -0,0 +1,394 @@ +# Lotus + +Lotus is an implementation of the [Filecoin Distributed Storage Network](https://filecoin.io/). +A Lotus node syncs blockchains that follow the +Filecoin protocol, validating the blocks and state transitions. +The specification for the Filecoin protocol can be found [here](https://filecoin-project.github.io/specs/). + +For information on how to setup and operate a Lotus node, +please follow the instructions [here](https://lotu.sh/en+getting-started). + +# Components + +At a high level, a Lotus node comprises the following components: + +FIXME: No mention of block production here, cross-reference with schomatis's miner doc +- The Syncer, which manages the process of syncing the blockchain +- The State Manager, which can compute the state at any given point in the chain +- The Virtual Machine (VM), which executes messages +- The Repository, where all data is stored +- P2P stuff (FIXME missing libp2p listed under other PL dependencies)? allows hello, blocksync, retrieval, storage +- API / CLI (FIXME missing, in scratchpad) +- Other Filecoin dependencies (specs actors, proofs, storage, etc., FIXME missing) +- Is the Builder worth its own component? +- Other PL dependencies (IPFS, libp2p, IPLD? FIXME, missing) +- External libraries used by Lotus and other deps (FIXME, missing) + +# Preliminaries + +We discuss some key Filecoin concepts here, aiming to explain them by contrasting them with analogous concepts +in other well-known blockchains like Ethereum. We only provide brief descriptions here; elaboration +can be found in the [spec](https://filecoin-project.github.io/specs/). + +### Tipsets + +Unlike in Ethereum, a block can have multiple parents in Filecoin. We thus refer to the parent set of a block, +instead of a single parent. +A [tipset](https://filecoin-project.github.io/specs/#systems__filecoin_blockchain__struct__tipset) +is any set of blocks that share the same parent set. + +There is no concept of "block difficulty" in Filecoin. Instead, +the weight of a tipset is simply the number of blocks in the chain that ends in that tipset. Note that a longer chain +can have less weight than a shorter chain with more blocks per tipset. + +We also allow for "null" tipsets, which include zero blocks. This allows miners to "skip" a round, and build on top +of an imaginary empty tipset if they want to. + +We call the heaviest tipset in a chain the "head" of the chain. + +### Actors and Messages + +An [Actor](https://filecoin-project.github.io/specs/#systems__filecoin_vm__actor) + is analogous to a smart contract in Ethereum. Filecoin does not allow users to define their own +actors, but comes with several [builtin actors](https://github.com/filecoin-project/specs-actors), +which can be thought of as pre-compiled contracts. + +A [Message](https://filecoin-project.github.io/specs/#systems__filecoin_vm__message) +is analogous to transactions in Ethereum. + +# Sync + +Sync refers to the process by which a Lotus node synchronizes to the heaviest chain being advertised by its peers. +At a high-level, Lotus syncs in a manner similar to most other blockchains; a Lotus node listens to the various +chains its peers claim to be at, picks the heaviest one, requests the blocks in the chosen chain, +and validates each block in that chain, running all state transitions along the way. + +The majority of the sync functionality happens in the [`Syncer`](https://github.com/filecoin-project/lotus/blob/master/chain/sync.go), +internally managed by a [`SyncManager`](https://github.com/filecoin-project/lotus/blob/master/chain/sync_manager.go). + +We now discuss the various stages of the sync process. + +## Sync setup + +When a Lotus node connects to a new peer, we exchange the head of our chain +with the new peer through [the `hello` protocol](https://github.com/filecoin-project/lotus/blob/master/node/hello/hello.go). +If the peer's head is heavier than ours, we try to sync to it. Note +that we do NOT update our chain head at this stage. + +## Fetching and Persisting Block Headers + +Note: The API refers to these stages as `StageHeaders` and `StagePersistHeaders`. + +We proceed in the sync process by requesting block headers from the peer, +moving back from their head, until we reach a tipset that we have in common +(such a common tipset must exist, thought it may simply be the genesis block). +The functionality can be found in `Syncer::collectHeaders()`. + +If the common tipset is our head, we treat the sync as a "fast-forward", else we must +drop part of our chain to connect to the peer's head (referred to as "forking"). + +FIXME: This next para might be best replaced with a link to the validation doc +Some of the possible causes of failure in this stage include: + +- The chain is linked to a block that we have previously marked as bad, +and stored in a [`BadBlockCache`](https://github.com/filecoin-project/lotus/blob/master/chain/badtscache.go). +- The beacon entries in a block are inconsistent (FIXME: more details about what is validated here wouldn't be bad). +- Switching to this new chain would involve a chain reorganization beyond the allowed threshold (SPECK-CHECK). + +## Fetching and Validating Blocks + +Note: The API refers to this stage as `StageMessages`. + +Having acquired the headers and found a common tipset, we then move forward, requesting the full blocks, including the messages. + +For each block, we first confirm the syntactic validity of the block (SPECK-CHECK), +which includes the syntactic validity of messages included +in the block. +We then apply the messages, running all the state transitions, and compare the state root we calculate with the provided state root. + + +FIXME: This next para might be best replaced with a link to the validation doc +Some of the possible causes of failure in this stage include: + +- a block is syntactically invalid (including potentially containing syntactically invalid messages) +- the computed state root after applying the block doesn't match the block's state root +- FIXME: Check what's covered by syntactic validity, and add anything important that isn't (like proof validity, future checks, etc.) + +The core functionality can be found in `Syncer::ValidateTipset()`, with `Syncer::checkBlockMessages()` performing +syntactic validation of messages. + +## Setting the head + +Note: The API refers to this stage as `StageSyncComplete`. + +If all validations pass we will now set that head as our heaviest tipset in +[`ChainStore`](https://github.com/filecoin-project/lotus/blob/master/chain/store/store.go). +We already have the full state, since we calculated +it during the sync process. + +FIXME (aayush) I don't fuilly understand the next 2 paragraphs, but it seems important. Confirm and polish. +Relevant issue in IPFS: https://github.com/ipfs/ipfs-docs/issues/264 + +It is important to note at this point that similar to the IPFS architecture of addressing by content and not by location/address (FIXME: check and link to IPFS docs) the "actual" chain stored in the node repo is *relative* to which CID we look for. We always have stored a series of Filecoin blocks pointing to other blocks, each a potential chain in itself by following its parent's reference, and its parent's parent, and so on up to the genesis block. (FIXME: We need a diagram here, one of the Filecoin blog entries might have something similar to what we are describing here.) It only depends on *where* (location) do we start to look for. The *only* address/location reference we hold of the chain, a relative reference, is the `heaviest` pointer. This is reflected by the fact that we don't store it in the `Blockstore` by a fixed, *absolute*, CID that reflects its contents, as this will change each time we sync to a new head (FIXME: link to the immutability IPFS doc that I need to write). + +FIXME: Create a further reading appendix, move this next para to it, along with other +extraneous content +This is one of the few items we store in `Datastore` by key, location, allowing its contents to change on every sync. This is reflected in the `(*ChainStore) writeHead()` function (called by `takeHeaviestTipSet()` above) where we reference the pointer by the explicit `chainHeadKey` address (the string `"head"`, not a hash embedded in a CID), and similarly in `(*ChainStore).Load()` when we start the node and create the `ChainStore`. Compare this to a Filecoin block or message which are immutable, stored in the `Blockstore` by CID, once created they never change. + +## Keeping up with the chain + +A Lotus node also listens for new blocks broadcast by its peers over the `gossipsub` channel (see FIXME for more). +If we have validated such a block's parent tipset, and adding it to our tipset at its height would lead to a heavier +head, then we validate and add this block. The validation described is identical to that invoked during the sync +process (indeed, it's the same codepath). + +# State + +In Filecoin, the chain state at any given point is a collection of data stored under a root CID +encapsulated in the [`StateTree`](https://github.com/filecoin-project/lotus/blob/master/chain/state/statetree.go), +and accessed through the +[`StateManager`](https://github.com/filecoin-project/lotus/blob/master/chain/stmgr/stmgr.go). +The state at the chain's head is thus easily tracked and updated in a state root CID. +(FIXME: Talk about CIDs somewhere, we might want to explain some of the modify/flush/update-root mechanism here.)) + +## Calculating a Tipset State + +Recall that a tipset is a set of blocks that have identical parents (that is, that are built on top of the same tipset). +The genesis tipset comprises the genesis block(s), and has some state corresponding to it. + +The methods `TipSetState()` and `computeTipSetState()` in +[`StateManager`](https://github.com/filecoin-project/lotus/blob/master/chain/stmgr/stmgr.go) + are responsible for computing +the state that results from applying a tipset. This involves applying all the messages included +in the tipset, and performing implicit operations like awarding block rewards. + +Any valid block built on top of a tipset `ts` should have its Parent State Root equal to the result of +calculating the tipset state of `ts`. Note that this means that all blocks in a tipset must have the same Parent +State Root (which is to be expected, since they have the same parent tipset) + +### Preparing to apply a tipset + +When `StateManager::computeTipsetState()` is called with a tipset, `ts`, +it retrieves the parent state root of the blocks in `ts`. It also creates a list of `BlockMessages`, which wraps the BLS +and SecP messages in a block along with the miner that produced the block. + +Control then flows to `StateManager::ApplyBlocks()`, which builds a VM to apply the messages given to it. The VM +is initialized with the parent state root of the blocks in `ts`. We apply the blocks in `ts` in order (see FIXME for +ordering of blocks in a tipset). + +### Applying a block + +For each block, we prepare to apply the ordered messages (first BLS, then SecP). Before applying a message, we check if +we have already applied a message with that CID within the scope of this method. If so, we simply skip that message; +this is how duplicate messages included in the same tipset are skipped (with only the miner of the "first" block to +include the message getting the reward). For the actual process of message application, see FIXME (need an +internal link here), for now we +simply assume that the outcome of the VM applying a message is either an error, or a +[`MessageReceipt`](https://github.com/filecoin-project/lotus/blob/master/chain/types/message_receipt.go) + and some +other information. + +We treat an error from the VM as a showstopper; there is no recovery, and no meaningful state can be computed for `ts`. +Given a successful receipt, we add the rewards and penalties to what the miner has earned so far. Once all the messages +included in a block have been applied (or skipped if they're a duplicate), we use an implicit message to call +the Reward Actor. This awards the miner their reward for having won a block, and also awards / penalizes them based +on the message rewards and penalties we tracked. + +We then proceed to apply the next block in `ts`, using the same VM. This means that the state changes that result +from applying a message are visible when applying all subsequent messages, even if they are included in a different block. + +### Finishing up + +Having applied all the blocks, we send one more implicit message, to the Cron Actor, which handles operations that +must be performed at the end of every epoch (see FIXME for more). The resulting state after calling the Cron Actor +is the computed state of the tipset. + +# Virtual Machine + +The Virtual Machine (VM) is responsible for executing messages. +The [Lotus Virtual Machine](https://github.com/filecoin-project/lotus/blob/master/chain/vm/vm.go) +invokes the appropriate methods in the builtin actors, and provides +a [`Runtime`](https://github.com/filecoin-project/specs-actors/blob/master/actors/runtime/runtime.go) +interface to the [builtin actors](https://github.com/filecoin-project/specs-actors) +that exposes their state, allows them to take certain actions, and meters +their gas usage. The VM also performs balance transfers, creates new account actors as needed, and tracks the gas reward, +penalty, return value, and exit code. + +## Applying a Message + +The primary entrypoint of the VM is the `ApplyMessage()` method. This method should not return an error +unless something goes unrecoverably wrong. + +The first thing this method does is assess if the message provided meets any of the penalty criteria. +If so, a penalty is issued, and the method returns. Next, the entire gas cost of the message is transferred to +a temporary gas holder account. It is from this gas holder that gas will be deducted; if it runs out of gas, the message +fails. Any unused gas in this holder will be refunded to the message's sender at the end of message execution. + +The VM then increments the sender's nonce, takes a snapshot of the state, and invokes `VM::send()`. + +The `send()` method creates a [`Runtime`](https://github.com/filecoin-project/lotus/blob/master/chain/vm/runtime.go) + for the subsequent message execution. +It then transfers the message's value to the recipient, creating a new account actor if needed. + +### Method Invocation + +We use reflection to translate a Filecoin message for the VM to an actual Go function, relying on the VM's +[`invoker`](https://github.com/filecoin-project/lotus/blob/master/chain/vm/invoker.go) structure. +Each actor has its own set of codes defined in `specs-actors/actors/builtin/methods.go`. +The `invoker` structure maps the builtin actors' CIDs + to a list of `invokeFunc` (one per exported method), which each take the `Runtime` (for state manipulation) + and the serialized input parameters. + +FIXME (aayush) Polish this next para. + +The basic layout (without reflection details) of `(*invoker).transform()` is as follows. From each actor registered in `NewInvoker()` we take its `Exports()` methods converting them to `invokeFunc`s. The actual method is wrapped in another function that takes care of decoding the serialized parameters and the runtime, this function is passed to `shimCall()` that will encapsulate the actors code being run inside a `defer` function to `recover()` from panics (we fail in the actors code with panics to unwrap the stack). The return values will then be (CBOR) marshaled and returned to the VM. + +### Returning from the VM + +Once method invocation is complete (including any subcalls), we return to `ApplyMessage()`, which receives +the serialized response and the [`ActorError`](https://github.com/filecoin-project/lotus/blob/master/chain/actors/aerrors/error.go). +The sender will be charged the appropriate amount of gas for the returned response, which gets put into the +[`MessageReceipt`](https://github.com/filecoin-project/lotus/blob/master/chain/types/message_receipt.go). + +The method then refunds any unused gas to the sender, sets up the gas reward for the miner, and +wraps all of this into an `ApplyRet`, which is returned. + +# Building a Lotus node + +When we launch a Lotus node with the command `./lotus daemon` +(see [here](https://github.com/filecoin-project/lotus/blob/master/cmd/lotus/daemon.go) for more), +the node is created through [dependency injection](https://godoc.org/go.uber.org/fx). +This relies on reflection, which makes some of the references hard to follow. +The node sets up all of the subsystems it needs to run, such as the repository, the network connections, thechain sync +service, etc. +This setup is orchestrated through calls to the `node.Override` function. +The structure of each call indicates the type of component it will set up +(many defined in [`node/modules/dtypes/`](https://github.com/filecoin-project/lotus/tree/master/node/modules/dtypes)), +and the function that will provide it. +The dependency is implicit in the argument of the provider function. + +As an example, consider the `modules.ChainStore()` function that provides the +[`ChainStore`](https://github.com/filecoin-project/lotus/blob/master/chain/store/store.go) structure. +It takes as one of its parameters the [`ChainBlockstore`](https://github.com/filecoin-project/lotus/blob/master/node/modules/dtypes/storage.go) +type, which becomes one of its dependencies. +For the node to be built successfully the `ChainBlockstore` will need to be provided before `ChainStore`, a requirement +that is made explicit in another `Override()` call that sets the provider of that type as the `ChainBlockstore()` function. + +## The Repository + +The repo is the directory where all of a node's information is stored. The node is entirely defined by its repo, which +makes it easy to port to another location. This one-to-one relationship means we can speak +of the node as the repo it is associated with, instead of the daemon process that runs from that repo. + +Only one daemon can run be running with an associated repo at a time. +A process signals that it is running a node associated with a particular repo, by creating and acquiring +a `repo.lock`. + +```sh +lsof ~/.lotus/repo.lock +# COMMAND PID +# lotus 52356 +``` +Trying to launch a second daemon hooked to the same repo leads to a `repo is already locked (lotus daemon already running)` +error. + +The `node.Repo()` function (`node/builder.go`) contains most of the dependencies (specified as `Override()` calls) +needed to properly set up the node's repo. We list the most salient ones here. + +### Datastore + +`Datastore` and `ChainBlockstore`: Data related to the node state is saved in the repo's `Datastore`, +an IPFS interface defined [here](github.com/ipfs/go-datastore/datastore.go). +Lotus creates this interface from a [Badger DB](https://github.com/dgraph-io/badger) in + [`FsRepo`](https://github.com/filecoin-project/lotus/blob/master/node/repo/fsrepo.go). +Every piece of data is fundamentally a key-value pair in the `datastore` directory of the repo. +There are several abstractions laid on top of it that appear through the code depending on *how* we access it, +but it is important to remember that we're always accessing it from the same place. + +FIXME: Maybe mention the `Batching` interface as the developer will stumble upon it before reaching the `Datastore` one. + +#### Blocks + +FIXME: IPFS blocks vs Filecoin blocks ideally happens before this / here + +The [`Blockstore` interface](`github.com/ipfs/go-ipfs-blockstore/blockstore.go`) structures the key-value pair +into the CID format for the key and the [`Block` interface](`github.com/ipfs/go-block-format/blocks.go`) for the value. +The `Block` value is just a raw string of bytes addressed by its hash, which is included in the CID key. + +`ChainBlockstore` creates a `Blockstore` in the repo under the `/blocks` namespace. +Every key stored there will have the `blocks` prefix so that it does not collide with other stores that use the same repo. + +FIXME: Link to IPFS documentation about DAG, CID, and related, especially we need a diagram that shows how do we wrap each datastore inside the next layer (datastore, batching, block store, gc, etc). + +#### Metadata + +`modules.Datastore()` creates a `dtypes.MetadataDS`, which is an alias for the basic `Datastore` interface. +Metadata is stored here under the `/metadata` prefix. +(FIXME: Explain *what* is metadata in contrast with the block store, namely we store the pointer to the heaviest chain, we might just link to that unwritten section here later.) + +FIXME: Explain the key store related calls (maybe remove, per Schomatis) + +### LockedRepo + +`LockedRepo()`: This method doesn't create or initialize any new structures, but rather registers an + `OnStop` [hook](https://godoc.org/go.uber.org/fx/internal/lifecycle#Hook) + that will close the locked repository associated with it on shutdown. + + +### Repo types / Node types + +FIXME: This section needs to be clarified / corrected...I don't fully understand the config differences (what do they have in common, if anything?) + +At the end of the `Repo()` function we see two mutually exclusive configuration calls based on the `RepoType` (`node/repo/fsrepo.go`). +```Go + ApplyIf(isType(repo.FullNode), ConfigFullNode(c)), + ApplyIf(isType(repo.StorageMiner), ConfigStorageMiner(c)), +``` +As we said, the repo fully identifies the node so a repo type is also a *node* type, in this case a full node or a storage miner. (FIXME: What is the difference between the two, does *full* imply miner?) In this case the `daemon` command will create a `FullNode`, this is specified in the command logic itself in `main.DaemonCmd()`, the `FsRepo` created (and passed to `node.Repo()`) will be initiated with that type (see `(*FsRepo).Init(t RepoType)`). + +## Online + +FIXME: Much of this might need to be subsumed into the p2p section + +The `node.Online()` configuration function (`node/builder.go`) initializes components that involve connecting to, +or interacting with, the Filecoin network. These connections are managed through the libp2p stack (FIXME link to this section when it exists). +We discuss some of the components found in the full node type (that is, included in the `ApplyIf(isType(repo.FullNode),` call). + +#### Chainstore + +`modules.ChainStore()` creates the [`store.ChainStore`](https://github.com/filecoin-project/lotus/blob/master/chain/store/store.go)) +that wraps the stores + previously instantiated in `Repo()`. It is the main point of entry for the node to all chain-related data + (FIXME: this is incorrect, we sometimes access its underlying block store directly, and probably shouldn't). + It also holds the crucial `heaviest` pointer, which indicates the current head of the chain. + + #### ChainExchange and ChainBlockservice +`ChainExchange()` and `ChainBlockservice()` establish a BitSwap connection (FIXME libp2p link) +to exchange chain information in the form of `blocks.Block`s stored in the repo. (See sync section for more details, the Filecoin blocks and messages are backed by these raw IPFS blocks that together form the different structures that define the state of the current/heaviest chain.) + +#### Incoming handlers +`HandleIncomingBlocks()` and `HandleIncomingMessages()` start the services in charge of processing new Filecoin blocks +and messages from the network (see `` for more information about the topics the node is subscribed to, FIXME: should that be part of the libp2p section or should we expand on gossipsub separately?). + +#### Hello +`RunHello()`: starts the services to both send (`(*Service).SayHello()`) and receive (`(*Service).HandleStream()`, `node/hello/hello.go`) +`hello` messages. When nodes establish a new connection with each other, they exchange these messages +to share chain-related information (namely their genesis block and their heaviest tipset). + +#### Syncer +`NewSyncer()` creates the `Syncer` structure and starts the services related to the chain sync process (FIXME link). + +### Ordering the dependencies + +We can establish the dependency relations by looking at the parameters that each function needs and by understanding +the architecture of the node and how the different components relate to each other (the chief purpose of this document). + +As an example, the sync mechanism depends on the node being able to exchange different IPFS blocks with the network, +so as to be able to request the "missing pieces" needed to construct the chain. This dependency is reflected by `NewSyncer()` +having a `blocksync.BlockSync` parameter, which in turn depends on `ChainBlockservice()` and `ChainExchange()`. +The chain exchange service further depends on the chain store to save and retrieve chain data, which is reflected +in `ChainExchange()` having `ChainGCBlockstore` as a parameter (which is just a wrapper around `ChainBlockstore` capable + of garbage collection). + +This block store is the same store underlying the chain store, which is an indirect dependency of `NewSyncer()` (through the `StateManager`). +(FIXME: This last line is flaky, we need to resolve the hierarchy better, we sometimes refer to the chain store and sometimes to its underlying block store. We need a diagram to visualize all the different components just mentioned otherwise it is too hard to follow. We probably even need to skip some of the connections mentioned.) \ No newline at end of file diff --git a/documentation/en/dev/WIP-arch-complementary-notes.md b/documentation/en/dev/WIP-arch-complementary-notes.md new file mode 100644 index 000000000..00bedb56a --- /dev/null +++ b/documentation/en/dev/WIP-arch-complementary-notes.md @@ -0,0 +1,153 @@ +# Genesis block + +Seems a good way to start exploring the VM state though the instantiation of its different actors like the storage power. + +Explain where do we load the genesis block, the CAR entries, and we set the root of the state. Follow the daemon command option, `chain.LoadGenesis()` saves all the blocks of the CAR file into the store provided by `ChainBlockstore` (this should already be explained in the previous section). The CAR root (MT root?) of those blocks is decoded into the `BlockHeader` that will be the Filecoin (genesis) block of the chain, but most of the information was stored in the raw data (non-Filecoin, what's the correct term?) blocks forwarded directly to the chain, the block header just has a pointer to it. + +`SetGenesis` block with name 0. `(ChainStore).SetGenesis()` stores it there. + +`MakeInitialStateTree` (`chain/gen/genesis/genesis.go`, used to construct the genesis block (`MakeGenesisBlock()`), constructs the state tree (`NewStateTree`) which is just a "pointer" (root node in the HAMT) to the different actors. It will be continuously used in `(*StateTree).SetActor()` an `types.Actor` structure under a certain `Address` (in the HAMT). (How does the `stateSnaps` work? It has no comments.) + +From this point we can follow different setup function like: + +* `SetupInitActor()`: see the `AddressMap`. + +* `SetupStoragePowerActor`: initial (zero) power state of the chain, most important attributes. + +* Account actors in the `template.Accounts`: `SetActor`. + +Which other actor type could be helpful at this point? + +# Basic concepts + +What should be clear at this point either from this document or the spec. + +## Addresses + +## Accounts + +# Sync Topics PubSub + +Gossip sub spec and some introduction. + +# Look at the constructor of a miner + +Follow the `lotus-storage-miner` command to see how a miner is created, from the command to the message to the storage power logic. + +# Directory structure so far, main structures seen, their relation + +List what are the main directories we should be looking at (e.g., `chain/`) and the most important structures (e.g., `StateTree`, `Runtime`, etc.) + +# Tests + +Run a few messages and observe state changes. What is the easiest test that also let's us "interact" with it (modify something and observe the difference). + +### Filecoin blocks vs IPFS blocks + +The term *block* has different meanings depending on the context, many times both meanings coexist at once in the code and it is important to distinguish them. (FIXME: link to IPFS blocks and related doc throughout this explanation). In terms of the lower IPFS layer, in charge of storing and retrieving data, both present at the repo or accessible through the network (e.g., through the BitSwap protocol discussed later), a block is a string of raw bytes identified by its hash, embedded and fully qualified in a CID identifier. IPFS blocks are the "building blocks" of almost any other piece of (chain) data described in the Filecoin protocol. + +In contrast, in the higher Filecoin (application) layer, a block is roughly (FIXME: link to spec definition, if we have any) a set of zero or more messages grouped together by a single miner which is itself grouped with other blocks (from other miners) in the same round to form a tipset. The Filecoin blockchain is a series of "chained" tipsets, each referencing its parent by its header's *CID*, that is, its header as seen as a single IPFS block, this is where both layers interact. + +Using now the full Go package qualifiers to avoid any ambiguity, the Filecoin block, `github.com/filecoin-project/lotus/chain/types.FullBlock`, is defined as, + +```Go +package types + +import "github.com/ipfs/go-cid" + +type FullBlock struct { + Header *BlockHeader + BlsMessages []*Message + SecpkMessages []*SignedMessage +} + +func (fb *FullBlock) Cid() cid.Cid { + return fb.Header.Cid() +} +``` + +It has, besides the Filecoin messages, a header with protocol related information (e.g., its `Height`) which is (like virtually any other piece of data in the Filecoin protocol) stored, retrieved and shared as an IPFS block with its corresponding CID, + +```Go +func (b *BlockHeader) Cid() cid.Cid { + sb, err := b.ToStorageBlock() + + return sb.Cid() +} + +func (b *BlockHeader) ToStorageBlock() (block.Block, error) { + data, err := b.Serialize() + + return github.com/ipfs/go-block-format.block.NewBlockWithCid(data) +} +``` + +These edited extracts from the `BlockHeader` show how it's treated as an IPFS block, `github.com/ipfs/go-block-format.block.BasicBlock`, to be both stored and referenced by its block storage CID. + +This duality permeates the code (and the Filecoin spec for that matter) but it is usually clear within the context to which block we are referring to. Normally the unqualified *block* is reserved for the Filecoin block and we won't usually refer to the IPFS one but only implicitly through the concept of its CID. With enough understanding of both stack's architecture the two definitions can coexist without much confusion as we will abstract away the IPFS layer and just use the CID as an identifier that we now its unique for two sequences of different *raw* byte strings. + +(FIXME: We use to do this presentation when talking about `gossipsub` topics and incoming blocks, and had to deal with, besides the block ambiguity, a similar confusion with the *message* term, used in libp2p to name anything that comes through the network, needing to present the extremely confusing hierarchy of a libp2p message containing a Filecoin block, identified by a IPFS block CID, containing Filecoin messages.) + +FIXME: Move the following tipset definition to sync or wherever is most needed, to avoid making this more confusing. + +Messages from the same round are collected into a block set (`chain/store/fts.go`): + +```Go +type FullTipSet struct { + Blocks []*types.FullBlock + tipset *types.TipSet + cids []cid.Cid +} +``` + +The "tipset" denomination might be a bit misleading as it doesn't refer *only* to the tip, the block set from the last round in the chain, but to *any* set of blocks, depending on the context the tipset is the actual tip or not. From its own perspective any block set is always the tip because it assumes nothing from following blocks. + +# CLI, API + +Explain how do we communicate with the node, both in terms of the CLI and the programmatic way (to create our own tools). + +## Client/server architecture + +In terms of the Filecoin network the node is a peer on a distributed hierarchy, but in terms of how we interact with the node we have client/server architecture. + +The node itself was initiated with the `daemon` command, it already started syncing to the chain by default. Along with that service it also started a [JSON-RPC](https://en.wikipedia.org/wiki/JSON-RPC) server to allow a client to interact with it. (FIXME: Check if this client is local or can be remote, link to external documentation of connection API.) + +We can connect to this server through the Lotus CLI. Virtually any other command other than `daemon` will run a client that will connect (by default) to the address specified in the `api` file in the repo associated with the node (by default in `~/.lotus`), e.g., + +```sh +cat ~/.lotus/api && echo +# /ip4/127.0.0.1/tcp/1234/http + +# With `lotus daemon` running in another terminal. +nc -v -z 127.0.0.1 1234 + +# Start daemon and turn off the logs to not clutter the command line. +bash -c "lotus daemon &" && + lotus wait-api && + lotus log set-level error # Or a env.var in the daemon command. + +nc -v -z 127.0.0.1 1234 +# Connection to 127.0.0.1 1234 port [tcp/*] succeeded! + +killall lotus +# FIXME: We need a lotus stop command: +# https://github.com/filecoin-project/lotus/issues/1827 +``` + +FIXME: Link to more in-depth documentation of the CLI architecture, maybe some IPFS documentation (since they share some common logic). + +## Node API + +The JSON-RPC server exposes the node API, the `FullNode` interface (defined in `api/api_full.go`). When we issue a command like `lotus sync status` to query the progress of the node sync we don't access the node's internals, those are decoupled in a separate daemon process, we call the `SyncState` function (of the `FullNode` API interface) through the RPC client started by our own command (see `NewFullNodeRPC` in `api/client/client.go` for more details). + +FIXME: Link to (and create) documentation about API fulfillment. + +Because we rely heavily on reflection for this part of the code the call chain is not easily visible by just following the references through the symbolic analysis of the IDE. If we start by the `lotus sync` command definition (in `cli/sync.go`), we eventually end up in the method interface `SyncState`, and when we look for its implementation we will find two functions: + +* `(*SyncAPI).SyncState()` (in `node/impl/full/sync.go`): this is the actual implementation of the API function that shows what the node (here acting as the RPC server) will execute when it receives the RPC request issued from the CLI acting as the client. + +* `(*FullNodeStruct).SyncState()`: this is an "empty placeholder" structure that will get later connected to the JSON-RPC client logic (see `NewMergeClient` in `lib/jsonrpc/client.go`, which is called by `NewFullNodeRPC`). (FIXME: check if this is accurate). The CLI (JSON-RPC client) will actually execute this function which will connect to the server and send the corresponding JSON request that will trigger the call of `(*SyncAPI).SyncState()` with the node implementation. + +This means that when we are tracking the logic of a CLI command we will eventually find this bifurcation and need to study the code of the server-side implementation in `node/impl/full` (mostly in the `common/` and `full/` directories). If we understand this architecture going directly to that part of the code abstracts away the JSON-RPC client/server logic and we can think that the CLI is actually running the node's logic. + +FIXME: Explain that "*the* node" is actually an API structure like `impl.FullNodeAPI` with the different API subcomponents like `full.SyncAPI`. We won't see a *single* node structure, each API (full node, minder, etc) will gather the necessary subcomponents it needs to service its calls. \ No newline at end of file From 6ba5c5d0e151f5351dbf444ff103db447f170e33 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 9 Jun 2020 18:19:33 -0400 Subject: [PATCH 091/160] Doc fix: use correct file name --- documentation/en/.library.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/en/.library.json b/documentation/en/.library.json index f637c813f..545eb4a5b 100644 --- a/documentation/en/.library.json +++ b/documentation/en/.library.json @@ -58,8 +58,8 @@ }, { "title": "Use Lotus with systemd", - "slug": "en+install-system-services", - "github": "en/install-system-services.md", + "slug": "en+install-systemd-services", + "github": "en/install-systemd-services.md", "value": null }, { From 27858506fff9df92f0fc5a51d31ed92a2774a5ae Mon Sep 17 00:00:00 2001 From: waynewyang Date: Wed, 10 Jun 2020 20:34:04 +0800 Subject: [PATCH 092/160] error check --- cli/chain.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cli/chain.go b/cli/chain.go index 01714d850..da88df657 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -889,6 +889,9 @@ var slashConsensusFault = &cli.Command{ BlockHeader1: bh1, BlockHeader2: bh2, }) + if err != nil { + return err + } if cctx.String("miner") == "" { return xerrors.Errorf("--miner flag is required") From 2ce3a5f887034e8d4ba8f5b2859dfc6ab19807a0 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 11 Jun 2020 00:09:48 +0200 Subject: [PATCH 093/160] docs: add a note about open files limit --- documentation/en/join-testnet.md | 2 ++ documentation/en/setup-troubleshooting.md | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/documentation/en/join-testnet.md b/documentation/en/join-testnet.md index bdd1b4a56..d01d7eafa 100644 --- a/documentation/en/join-testnet.md +++ b/documentation/en/join-testnet.md @@ -28,6 +28,8 @@ lotus net peers | wc -l In order to connect to the network, you need to be connected to at least 1 peer. If you’re seeing 0 peers, read our [troubleshooting notes](https://docs.lotu.sh/en+setup-troubleshooting). +Make sure that you have a reasonable "open files limit" set on your machine, such as 10000. If you're seeing a lower value, such as 256 (default on macOS), read our [troubleshooting notes](https://docs.lotu.sh/en+setup-troubleshooting) on how to update it prior to starting the Lotus daemon. + ## Chain sync While the daemon is running, the next requirement is to sync the chain. Run the command below to view the chain sync progress. To see current chain height, visit the [network stats page](https://stats.testnet.filecoin.io/). diff --git a/documentation/en/setup-troubleshooting.md b/documentation/en/setup-troubleshooting.md index 055a9c501..8a23544d9 100644 --- a/documentation/en/setup-troubleshooting.md +++ b/documentation/en/setup-troubleshooting.md @@ -29,4 +29,18 @@ ERROR hello hello/hello.go:81 other peer has different genesis! - repo is already locked ``` -- You already have another lotus daemon running. \ No newline at end of file +- You already have another lotus daemon running. + +## Config: Open files limit + +On most systems you can check the open files limit with: + +```sh +ulimit -n +``` + +You can also modify this number by using the `ulimit` command. It gives you the ability to control the resources available for the shell or process started by it. If the number is below 10000, you can change it with the following command prior to starting the Lotus daemon: + +```sh +ulimit -n 10000 +``` From 14ea847ef63eb1b05625c2ebb9d61443ec059429 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 10 Jun 2020 22:15:46 -0700 Subject: [PATCH 094/160] fix chain index seeking through long ranges of null rounds --- chain/store/index.go | 12 +++++-- chain/store/index_test.go | 76 +++++++++++++++++++++++++++++++++++++++ chain/types/mock/chain.go | 7 +++- 3 files changed, 91 insertions(+), 4 deletions(-) create mode 100644 chain/store/index_test.go diff --git a/chain/store/index.go b/chain/store/index.go index 15d5d7025..8747c79c6 100644 --- a/chain/store/index.go +++ b/chain/store/index.go @@ -97,9 +97,15 @@ func (ci *ChainIndex) fillCache(tsk types.TipSetKey) (*lbEntry, error) { rheight -= ci.skipLength - skipTarget, err := ci.walkBack(parent, rheight) - if err != nil { - return nil, err + var skipTarget *types.TipSet + if parent.Height() < rheight { + skipTarget = parent + + } else { + skipTarget, err = ci.walkBack(parent, rheight) + if err != nil { + return nil, xerrors.Errorf("fillCache walkback: %w", err) + } } lbe := &lbEntry{ diff --git a/chain/store/index_test.go b/chain/store/index_test.go new file mode 100644 index 000000000..6b97a8614 --- /dev/null +++ b/chain/store/index_test.go @@ -0,0 +1,76 @@ +package store_test + +import ( + "bytes" + "context" + "testing" + + "github.com/filecoin-project/lotus/chain/gen" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types/mock" + "github.com/filecoin-project/specs-actors/actors/abi" + datastore "github.com/ipfs/go-datastore" + syncds "github.com/ipfs/go-datastore/sync" + blockstore "github.com/ipfs/go-ipfs-blockstore" + "github.com/stretchr/testify/assert" +) + +func TestIndexSeeks(t *testing.T) { + cg, err := gen.NewGenerator() + if err != nil { + t.Fatal(err) + } + + gencar, err := cg.GenesisCar() + if err != nil { + t.Fatal(err) + } + + gen := cg.Genesis() + + ctx := context.TODO() + + nbs := blockstore.NewBlockstore(syncds.MutexWrap(datastore.NewMapDatastore())) + cs := store.NewChainStore(nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil) + + _, err = cs.Import(bytes.NewReader(gencar)) + if err != nil { + t.Fatal(err) + } + + cur := mock.TipSet(gen) + if err := cs.PutTipSet(ctx, mock.TipSet(gen)); err != nil { + t.Fatal(err) + } + cs.SetGenesis(gen) + + for i := 0; i < 100; i++ { + nextts := mock.TipSet(mock.MkBlock(cur, 1, 1)) + + if err := cs.PutTipSet(ctx, nextts); err != nil { + t.Fatal(err) + } + cur = nextts + } + + skip := mock.MkBlock(cur, 1, 1) + skip.Height += 50 + + skipts := mock.TipSet(skip) + + if err := cs.PutTipSet(ctx, skipts); err != nil { + t.Fatal(err) + } + + ts, err := cs.GetTipsetByHeight(ctx, skip.Height-10, skipts, false) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, abi.ChainEpoch(151), ts.Height()) + + ts2, err := cs.GetTipsetByHeight(ctx, 90, skipts, false) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, abi.ChainEpoch(90), ts2.Height()) +} diff --git a/chain/types/mock/chain.go b/chain/types/mock/chain.go index 5154ab115..00d0eecc9 100644 --- a/chain/types/mock/chain.go +++ b/chain/types/mock/chain.go @@ -49,6 +49,11 @@ func MkBlock(parents *types.TipSet, weightInc uint64, ticketNonce uint64) *types panic(err) } + pstateRoot := c + if parents != nil { + pstateRoot = parents.Blocks()[0].ParentStateRoot + } + var pcids []cid.Cid var height abi.ChainEpoch weight := types.NewInt(weightInc) @@ -72,7 +77,7 @@ func MkBlock(parents *types.TipSet, weightInc uint64, ticketNonce uint64) *types ParentWeight: weight, Messages: c, Height: height, - ParentStateRoot: c, + ParentStateRoot: pstateRoot, BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS, Data: []byte("boo! im a signature")}, } } From ac1048fcd7c3e1e616f7fedb4470bbd0ba159bae Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 10 Jun 2020 22:17:29 -0700 Subject: [PATCH 095/160] test a few more cases --- chain/store/index_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/chain/store/index_test.go b/chain/store/index_test.go index 6b97a8614..b1a85996f 100644 --- a/chain/store/index_test.go +++ b/chain/store/index_test.go @@ -68,9 +68,11 @@ func TestIndexSeeks(t *testing.T) { } assert.Equal(t, abi.ChainEpoch(151), ts.Height()) - ts2, err := cs.GetTipsetByHeight(ctx, 90, skipts, false) - if err != nil { - t.Fatal(err) + for i := 0; i <= 100; i++ { + ts3, err := cs.GetTipsetByHeight(ctx, abi.ChainEpoch(i), skipts, false) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, abi.ChainEpoch(i), ts3.Height()) } - assert.Equal(t, abi.ChainEpoch(90), ts2.Height()) } From c3f23ddce829ddb9bac444f019d2e47d5ba85ac7 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 11 Jun 2020 02:47:28 +0200 Subject: [PATCH 096/160] Refactor ExecutionResult to ExecutionTrace Signed-off-by: Jakub Sztandera --- api/api_full.go | 10 ++--- api/docgen/docgen.go | 2 +- chain/stmgr/call.go | 10 ++--- chain/stmgr/stmgr.go | 8 ++-- chain/types/execresult.go | 4 +- chain/vm/runtime.go | 37 +++++------------ chain/vm/vm.go | 86 +++++++++++++++++++++++---------------- cli/state.go | 14 +++---- go.mod | 1 + go.sum | 3 ++ node/impl/full/state.go | 10 ++--- 11 files changed, 95 insertions(+), 90 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 63d9a5e5d..e9370a0b8 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -350,11 +350,11 @@ type RetrievalOrder struct { } type InvocResult struct { - Msg *types.Message - MsgRct *types.MessageReceipt - InternalExecutions []*types.ExecutionResult - Error string - Duration time.Duration + Msg *types.Message + MsgRct *types.MessageReceipt + ExecutionTrace types.ExecutionTrace + Error string + Duration time.Duration } type MethodCall struct { diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index eb4ab5fd9..d0b5d8558 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -93,7 +93,7 @@ func init() { addExample(build.APIVersion) addExample(api.PCHInbound) addExample(time.Minute) - addExample(&types.ExecutionResult{ + addExample(&types.ExecutionTrace{ Msg: exampleValue(reflect.TypeOf(&types.Message{})).(*types.Message), MsgRct: exampleValue(reflect.TypeOf(&types.MessageReceipt{})).(*types.MessageReceipt), }) diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index b9635696f..32e502e95 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -62,11 +62,11 @@ func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate } return &api.InvocResult{ - Msg: msg, - MsgRct: &ret.MessageReceipt, - InternalExecutions: ret.InternalExecutions, - Error: errs, - Duration: ret.Duration, + Msg: msg, + MsgRct: &ret.MessageReceipt, + ExecutionTrace: ret.ExecutionTrace, + Error: errs, + Duration: ret.Duration, }, nil } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index d6aaebab4..d4c12dffe 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -121,10 +121,10 @@ func (sm *StateManager) ExecutionTrace(ctx context.Context, ts *types.TipSet) (c var trace []*api.InvocResult st, _, err := sm.computeTipSetState(ctx, ts.Blocks(), func(mcid cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { ir := &api.InvocResult{ - Msg: msg, - MsgRct: &ret.MessageReceipt, - InternalExecutions: ret.InternalExecutions, - Duration: ret.Duration, + Msg: msg, + MsgRct: &ret.MessageReceipt, + ExecutionTrace: ret.ExecutionTrace, + Duration: ret.Duration, } if ret.ActorErr != nil { ir.Error = ret.ActorErr.Error() diff --git a/chain/types/execresult.go b/chain/types/execresult.go index 56f2ef143..dd5c8fa3e 100644 --- a/chain/types/execresult.go +++ b/chain/types/execresult.go @@ -2,11 +2,11 @@ package types import "time" -type ExecutionResult struct { +type ExecutionTrace struct { Msg *Message MsgRct *MessageReceipt Error string Duration time.Duration - Subcalls []*ExecutionResult + Subcalls []ExecutionTrace } diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index d9268afe4..0067055d1 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -5,7 +5,6 @@ import ( "context" "encoding/binary" "fmt" - "time" "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi" @@ -51,10 +50,10 @@ type Runtime struct { origin address.Address originNonce uint64 - internalExecutions []*types.ExecutionResult - numActorsCreated uint64 - allowInternal bool - callerValidated bool + executionTrace types.ExecutionTrace + numActorsCreated uint64 + allowInternal bool + callerValidated bool } func (rt *Runtime) TotalFilCircSupply() abi.TokenAmount { @@ -368,7 +367,6 @@ func (rt *Runtime) Send(to address.Address, method abi.MethodNum, m vmr.CBORMars } func (rt *Runtime) internalSend(from, to address.Address, method abi.MethodNum, value types.BigInt, params []byte) ([]byte, aerrors.ActorError) { - start := time.Now() ctx, span := trace.StartSpan(rt.ctx, "vmc.Send") defer span.End() if span.IsRecordingEvents() { @@ -401,27 +399,10 @@ func (rt *Runtime) internalSend(from, to address.Address, method abi.MethodNum, } } - mr := types.MessageReceipt{ - ExitCode: aerrors.RetCode(errSend), - Return: ret, - GasUsed: 0, - } - - er := types.ExecutionResult{ - Msg: msg, - MsgRct: &mr, - Duration: time.Since(start), - } - - if errSend != nil { - er.Error = errSend.Error() - } - if subrt != nil { - er.Subcalls = subrt.internalExecutions rt.numActorsCreated = subrt.numActorsCreated } - rt.internalExecutions = append(rt.internalExecutions, &er) + rt.executionTrace.Subcalls = append(rt.executionTrace.Subcalls, subrt.executionTrace) //&er) return ret, errSend } @@ -504,13 +485,13 @@ func (rt *Runtime) stateCommit(oldh, newh cid.Cid) aerrors.ActorError { } func (rt *Runtime) ChargeGas(toUse int64) { - err := rt.chargeGasSafe(toUse) + err := rt.chargeGasInternal(toUse) if err != nil { panic(err) } } -func (rt *Runtime) chargeGasSafe(toUse int64) aerrors.ActorError { +func (rt *Runtime) chargeGasInternal(toUse int64) aerrors.ActorError { if rt.gasUsed+toUse > rt.gasAvailable { rt.gasUsed = rt.gasAvailable return aerrors.Newf(exitcode.SysErrOutOfGas, "not enough gas: used=%d, available=%d", rt.gasUsed, rt.gasAvailable) @@ -519,6 +500,10 @@ func (rt *Runtime) chargeGasSafe(toUse int64) aerrors.ActorError { return nil } +func (rt *Runtime) chargeGasSafe(toUse int64) aerrors.ActorError { + return rt.chargeGasInternal(toUse) +} + func (rt *Runtime) Pricelist() Pricelist { return rt.pricelist } diff --git a/chain/vm/vm.go b/chain/vm/vm.go index da38fcbf3..42d6d373e 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -102,6 +102,7 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, origin addres pricelist: PricelistByEpoch(vm.blockHeight), allowInternal: true, callerValidated: false, + executionTrace: types.ExecutionTrace{Msg: msg}, } rt.cst = &cbor.BasicIpldStore{ @@ -163,14 +164,16 @@ type Rand interface { type ApplyRet struct { types.MessageReceipt - ActorErr aerrors.ActorError - Penalty types.BigInt - InternalExecutions []*types.ExecutionResult - Duration time.Duration + ActorErr aerrors.ActorError + Penalty types.BigInt + ExecutionTrace types.ExecutionTrace + Duration time.Duration } func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, gasCharge int64) ([]byte, aerrors.ActorError, *Runtime) { + start := time.Now() + st := vm.cstate gasUsed := gasCharge @@ -191,35 +194,50 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, }() } - if aerr := rt.chargeGasSafe(rt.Pricelist().OnMethodInvocation(msg.Value, msg.Method)); aerr != nil { - return nil, aerrors.Wrap(aerr, "not enough gas for method invocation"), rt - } + ret, err := func() ([]byte, aerrors.ActorError) { + if aerr := rt.chargeGasSafe(rt.Pricelist().OnMethodInvocation(msg.Value, msg.Method)); aerr != nil { + return nil, aerrors.Wrap(aerr, "not enough gas for method invocation") + } - toActor, err := st.GetActor(msg.To) - if err != nil { - if xerrors.Is(err, init_.ErrAddressNotFound) { - a, err := TryCreateAccountActor(rt, msg.To) - if err != nil { - return nil, aerrors.Wrapf(err, "could not create account"), rt + toActor, err := st.GetActor(msg.To) + if err != nil { + if xerrors.Is(err, init_.ErrAddressNotFound) { + a, err := TryCreateAccountActor(rt, msg.To) + if err != nil { + return nil, aerrors.Wrapf(err, "could not create account") + } + toActor = a + } else { + return nil, aerrors.Escalate(err, "getting actor") } - toActor = a - } else { - return nil, aerrors.Escalate(err, "getting actor"), rt } - } - if types.BigCmp(msg.Value, types.NewInt(0)) != 0 { - if err := vm.transfer(msg.From, msg.To, msg.Value); err != nil { - return nil, aerrors.Wrap(err, "failed to transfer funds"), nil + if types.BigCmp(msg.Value, types.NewInt(0)) != 0 { + if err := vm.transfer(msg.From, msg.To, msg.Value); err != nil { + return nil, aerrors.Wrap(err, "failed to transfer funds") + } } + + if msg.Method != 0 { + var ret []byte + ret, err := vm.Invoke(toActor, rt, msg.Method, msg.Params) + return ret, err + } + return nil, nil + }() + + mr := types.MessageReceipt{ + ExitCode: aerrors.RetCode(err), + Return: ret, + GasUsed: rt.gasUsed, + } + rt.executionTrace.MsgRct = &mr + rt.executionTrace.Duration = time.Since(start) + if err != nil { + rt.executionTrace.Error = err.Error() } - if msg.Method != 0 { - ret, err := vm.Invoke(toActor, rt, msg.Method, msg.Params) - return ret, err, rt - } - - return nil, nil, rt + return ret, err, rt } func checkMessage(msg *types.Message) error { @@ -250,10 +268,10 @@ func (vm *VM) ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*Ap Return: ret, GasUsed: 0, }, - ActorErr: actorErr, - InternalExecutions: rt.internalExecutions, - Penalty: types.NewInt(0), - Duration: time.Since(start), + ActorErr: actorErr, + ExecutionTrace: rt.executionTrace, + Penalty: types.NewInt(0), + Duration: time.Since(start), }, actorErr } @@ -419,10 +437,10 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, Return: ret, GasUsed: gasUsed, }, - ActorErr: actorErr, - InternalExecutions: rt.internalExecutions, - Penalty: types.NewInt(0), - Duration: time.Since(start), + ActorErr: actorErr, + ExecutionTrace: rt.executionTrace, + Penalty: types.NewInt(0), + Duration: time.Since(start), }, nil } diff --git a/cli/state.go b/cli/state.go index 6720ceacd..5b13c43b9 100644 --- a/cli/state.go +++ b/cli/state.go @@ -931,14 +931,14 @@ var stateComputeStateCmd = &cli.Command{ if cctx.Bool("show-trace") { for _, ir := range stout.Trace { fmt.Printf("%s\t%s\t%s\t%d\t%x\t%d\t%x\n", ir.Msg.From, ir.Msg.To, ir.Msg.Value, ir.Msg.Method, ir.Msg.Params, ir.MsgRct.ExitCode, ir.MsgRct.Return) - printInternalExecutions("\t", ir.InternalExecutions) + printInternalExecutions("\t", ir.ExecutionTrace.Subcalls) } } return nil }, } -func printInternalExecutions(prefix string, trace []*types.ExecutionResult) { +func printInternalExecutions(prefix string, trace []types.ExecutionTrace) { for _, im := range trace { fmt.Printf("%s%s\t%s\t%s\t%d\t%x\t%d\t%x\n", prefix, im.Msg.From, im.Msg.To, im.Msg.Value, im.Msg.Method, im.Msg.Params, im.MsgRct.ExitCode, im.MsgRct.Return) printInternalExecutions(prefix+"\t", im.Subcalls) @@ -1028,11 +1028,9 @@ func computeStateHtml(ts *types.TipSet, o *api.ComputeStateOutput, getCode func( fmt.Printf(`
Error: 
%s
`, ir.Error) } - if len(ir.InternalExecutions) > 0 { - fmt.Println("
Internal executions:
") - if err := printInternalExecutionsHtml(cid.String(), ir.InternalExecutions, getCode); err != nil { - return err - } + fmt.Println("
Execution trace:
") + if err := printInternalExecutionsHtml(cid.String(), ir.ExecutionTrace.Subcalls, getCode); err != nil { + return err } fmt.Println("
") } @@ -1042,7 +1040,7 @@ func computeStateHtml(ts *types.TipSet, o *api.ComputeStateOutput, getCode func( return nil } -func printInternalExecutionsHtml(hashName string, trace []*types.ExecutionResult, getCode func(addr address.Address) (cid.Cid, error)) error { +func printInternalExecutionsHtml(hashName string, trace []types.ExecutionTrace, getCode func(addr address.Address) (cid.Cid, error)) error { for i, im := range trace { hashName := fmt.Sprintf("%s-r%d", hashName, i) diff --git a/go.mod b/go.mod index 21fea095f..5a3cf4a07 100644 --- a/go.mod +++ b/go.mod @@ -104,6 +104,7 @@ require ( github.com/multiformats/go-multibase v0.0.2 github.com/multiformats/go-multihash v0.0.13 github.com/opentracing/opentracing-go v1.1.0 + github.com/prometheus/common v0.9.1 github.com/stretchr/objx v0.2.0 // indirect github.com/stretchr/testify v1.5.1 github.com/syndtr/goleveldb v1.0.0 diff --git a/go.sum b/go.sum index 35395be66..e10972bb6 100644 --- a/go.sum +++ b/go.sum @@ -42,9 +42,11 @@ github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75/go.mod h1:uAXEEpARkRhCZfEvy/y0Jcc888f9tHCc1W7/UeEtreE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= @@ -1575,6 +1577,7 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/node/impl/full/state.go b/node/impl/full/state.go index fae2a7b67..1d68a1858 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -242,11 +242,11 @@ func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid. } return &api.InvocResult{ - Msg: m, - MsgRct: &r.MessageReceipt, - InternalExecutions: r.InternalExecutions, - Error: errstr, - Duration: r.Duration, + Msg: m, + MsgRct: &r.MessageReceipt, + ExecutionTrace: r.ExecutionTrace, + Error: errstr, + Duration: r.Duration, }, nil } From 500ddd65859c224dce2fb39a00bf9e151e1d4073 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 11 Jun 2020 16:07:04 +0200 Subject: [PATCH 097/160] go mod tidy Signed-off-by: Jakub Sztandera --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index 5a3cf4a07..21fea095f 100644 --- a/go.mod +++ b/go.mod @@ -104,7 +104,6 @@ require ( github.com/multiformats/go-multibase v0.0.2 github.com/multiformats/go-multihash v0.0.13 github.com/opentracing/opentracing-go v1.1.0 - github.com/prometheus/common v0.9.1 github.com/stretchr/objx v0.2.0 // indirect github.com/stretchr/testify v1.5.1 github.com/syndtr/goleveldb v1.0.0 From d4dbfdbe9191a2b17a660d97060522dbe749d708 Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 9 Jun 2020 15:04:11 -0700 Subject: [PATCH 098/160] rename Config to GetConfig --- node/builder.go | 2 +- node/repo/interface.go | 2 +- node/repo/memrepo.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/node/builder.go b/node/builder.go index f503bff5c..87b39ba41 100644 --- a/node/builder.go +++ b/node/builder.go @@ -412,7 +412,7 @@ func Repo(r repo.Repo) Option { if err != nil { return err } - c, err := lr.Config() + c, err := lr.GetConfig() if err != nil { return err } diff --git a/node/repo/interface.go b/node/repo/interface.go index d81d644c7..002373e0e 100644 --- a/node/repo/interface.go +++ b/node/repo/interface.go @@ -37,7 +37,7 @@ type LockedRepo interface { Datastore(namespace string) (datastore.Batching, error) // Returns config in this repo - Config() (interface{}, error) + GetConfig() (interface{}, error) GetStorage() (stores.StorageConfig, error) SetStorage(func(*stores.StorageConfig)) error diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index 17eeb0253..1b1c716c2 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -217,7 +217,7 @@ func (lmem *lockedMemRepo) Datastore(ns string) (datastore.Batching, error) { return namespace.Wrap(lmem.mem.datastore, datastore.NewKey(ns)), nil } -func (lmem *lockedMemRepo) Config() (interface{}, error) { +func (lmem *lockedMemRepo) GetConfig() (interface{}, error) { if err := lmem.checkToken(); err != nil { return nil, err } From 2d6b2e381148241e3def4ce1f7058fc04bb08fc6 Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 9 Jun 2020 16:30:30 -0700 Subject: [PATCH 099/160] fix a spelling error --- node/config/load.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/config/load.go b/node/config/load.go index 1590efedb..b0786643f 100644 --- a/node/config/load.go +++ b/node/config/load.go @@ -12,7 +12,7 @@ import ( ) // FromFile loads config from a specified file overriding defaults specified in -// the def parameter. If file does not exist or is empty defaults are asummed. +// the def parameter. If file does not exist or is empty defaults are assumed. func FromFile(path string, def interface{}) (interface{}, error) { file, err := os.Open(path) switch { From cf321f76678ab3850c4d1471c10ce3dfc9cedab7 Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 9 Jun 2020 16:37:18 -0700 Subject: [PATCH 100/160] add support for replacing config after node starts - TODO: does a "locked repo" need fine-grained (i.e. field-level) locking? --- node/builder.go | 2 +- node/repo/fsrepo.go | 24 ++++++++++++++++++++++++ node/repo/interface.go | 3 ++- node/repo/memrepo.go | 22 +++++++++++++++++++--- node/repo/repo_test.go | 18 ++++++++++++++++-- 5 files changed, 62 insertions(+), 7 deletions(-) diff --git a/node/builder.go b/node/builder.go index 87b39ba41..f503bff5c 100644 --- a/node/builder.go +++ b/node/builder.go @@ -412,7 +412,7 @@ func Repo(r repo.Repo) Option { if err != nil { return err } - c, err := lr.GetConfig() + c, err := lr.Config() if err != nil { return err } diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 6733b0868..a0530f0f7 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -11,6 +11,7 @@ import ( "strings" "sync" + "github.com/BurntSushi/toml" "github.com/ipfs/go-datastore" fslock "github.com/ipfs/go-fs-lock" logging "github.com/ipfs/go-log/v2" @@ -271,6 +272,29 @@ func (fsr *fsLockedRepo) Config() (interface{}, error) { return config.FromFile(fsr.join(fsConfig), defConfForType(fsr.repoType)) } +func (fsr *fsLockedRepo) SetConfig(cfg interface{}) error { + if err := fsr.stillValid(); err != nil { + return err + } + + tmp, err := ioutil.TempFile("", "lotus-config-temp") + if err != nil { + return err + } + + err = toml.NewEncoder(tmp).Encode(cfg) + if err != nil { + return err + } + + err = os.Rename(tmp.Name(), fsr.join(fsConfig)) + if err != nil { + return err + } + + return nil +} + func (fsr *fsLockedRepo) GetStorage() (stores.StorageConfig, error) { fsr.storageLk.Lock() defer fsr.storageLk.Unlock() diff --git a/node/repo/interface.go b/node/repo/interface.go index 002373e0e..560cd969c 100644 --- a/node/repo/interface.go +++ b/node/repo/interface.go @@ -37,7 +37,8 @@ type LockedRepo interface { Datastore(namespace string) (datastore.Batching, error) // Returns config in this repo - GetConfig() (interface{}, error) + Config() (interface{}, error) + SetConfig(interface{}) error GetStorage() (stores.StorageConfig, error) SetStorage(func(*stores.StorageConfig)) error diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index 1b1c716c2..d942b6bd9 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -30,8 +30,13 @@ type MemRepo struct { token *byte datastore datastore.Datastore - configF func(t RepoType) interface{} keystore map[string]types.KeyInfo + + // given a repo type, produce the default config + configF func(t RepoType) interface{} + + // holds the current config value + config interface{} } type lockedMemRepo struct { @@ -217,11 +222,22 @@ func (lmem *lockedMemRepo) Datastore(ns string) (datastore.Batching, error) { return namespace.Wrap(lmem.mem.datastore, datastore.NewKey(ns)), nil } -func (lmem *lockedMemRepo) GetConfig() (interface{}, error) { +func (lmem *lockedMemRepo) Config() (interface{}, error) { if err := lmem.checkToken(); err != nil { return nil, err } - return lmem.mem.configF(lmem.t), nil + + if lmem.mem.config == nil { + lmem.mem.config = lmem.mem.configF(lmem.t) + } + + return lmem.mem.config, nil +} + +func (lmem *lockedMemRepo) SetConfig(cfg interface{}) error { + lmem.mem.config = cfg + + return nil } func (lmem *lockedMemRepo) Storage() (stores.StorageConfig, error) { diff --git a/node/repo/repo_test.go b/node/repo/repo_test.go index 9c43e8f4b..5da4bcadc 100644 --- a/node/repo/repo_test.go +++ b/node/repo/repo_test.go @@ -9,6 +9,8 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/config" + + "github.com/stretchr/testify/require" ) func basicTest(t *testing.T, repo Repo) { @@ -47,10 +49,22 @@ func basicTest(t *testing.T, repo Repo) { assert.NoError(t, err, "setting multiaddr shouldn't error") assert.Equal(t, ma, apima, "returned API multiaddr should be the same") - cfg, err := lrepo.Config() - assert.Equal(t, config.DefaultFullNode(), cfg, "there should be a default config") + c1, err := lrepo.Config() + assert.Equal(t, config.DefaultFullNode(), c1, "there should be a default config") assert.NoError(t, err, "config should not error") + // mutate config and persist back to repo + cfg1 := c1.(*config.FullNode) + cfg1.Client.IpfsMAddr = "duvall" + err = lrepo.SetConfig(cfg1) + assert.NoError(t, err) + + // load config and verify changes + c2, err := lrepo.Config() + require.NoError(t, err) + cfg2 := c2.(*config.FullNode) + require.Equal(t, cfg2.Client.IpfsMAddr, "duvall") + err = lrepo.Close() assert.NoError(t, err, "should be able to close") From fb1d5197fa8a39cae762c43e2c72566d455794ff Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 10 Jun 2020 09:07:47 -0700 Subject: [PATCH 101/160] pass SetConfig a mutator func, as per PR feedback - fsLockedRepo.config gets a mutex - add missing checkToken call to lockedMemRepo#GetStorage and lockedMemRepo#SetStorage - LockedRepo#SetConfig accepts a mutating function as per @magik --- node/repo/fsrepo.go | 30 +++++++++++++++++++++++------- node/repo/interface.go | 2 +- node/repo/memrepo.go | 41 +++++++++++++++++++++++++++++++---------- node/repo/repo_test.go | 7 ++++--- 4 files changed, 59 insertions(+), 21 deletions(-) diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index a0530f0f7..b223731d9 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -230,6 +230,7 @@ type fsLockedRepo struct { dsOnce sync.Once storageLk sync.Mutex + configLk sync.Mutex } func (fsr *fsLockedRepo) Path() string { @@ -266,28 +267,43 @@ func (fsr *fsLockedRepo) stillValid() error { } func (fsr *fsLockedRepo) Config() (interface{}, error) { - if err := fsr.stillValid(); err != nil { - return nil, err - } + fsr.configLk.Lock() + defer fsr.configLk.Unlock() + + return fsr.loadConfigFromDisk() +} + +func (fsr *fsLockedRepo) loadConfigFromDisk() (interface{}, error) { return config.FromFile(fsr.join(fsConfig), defConfForType(fsr.repoType)) } -func (fsr *fsLockedRepo) SetConfig(cfg interface{}) error { +func (fsr *fsLockedRepo) SetConfig(c func(interface{})) error { if err := fsr.stillValid(); err != nil { return err } - tmp, err := ioutil.TempFile("", "lotus-config-temp") + fsr.configLk.Lock() + defer fsr.configLk.Unlock() + + cfg, err := fsr.loadConfigFromDisk() if err != nil { return err } - err = toml.NewEncoder(tmp).Encode(cfg) + // mutate in-memory representation of config + c(cfg) + + // buffer into which we write TOML bytes + buf := new(bytes.Buffer) + + // encode now-mutated config as TOML and write to buffer + err = toml.NewEncoder(buf).Encode(cfg) if err != nil { return err } - err = os.Rename(tmp.Name(), fsr.join(fsConfig)) + // write buffer of TOML bytes to config file + err = ioutil.WriteFile(fsr.join(fsConfig), buf.Bytes(), 0644) if err != nil { return err } diff --git a/node/repo/interface.go b/node/repo/interface.go index 560cd969c..5950f813f 100644 --- a/node/repo/interface.go +++ b/node/repo/interface.go @@ -38,7 +38,7 @@ type LockedRepo interface { // Returns config in this repo Config() (interface{}, error) - SetConfig(interface{}) error + SetConfig(func(interface{})) error GetStorage() (stores.StorageConfig, error) SetStorage(func(*stores.StorageConfig)) error diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index d942b6bd9..399b239c1 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -36,7 +36,10 @@ type MemRepo struct { configF func(t RepoType) interface{} // holds the current config value - config interface{} + config struct { + sync.Mutex + val interface{} + } } type lockedMemRepo struct { @@ -50,6 +53,10 @@ type lockedMemRepo struct { } func (lmem *lockedMemRepo) GetStorage() (stores.StorageConfig, error) { + if err := lmem.checkToken(); err != nil { + return stores.StorageConfig{}, err + } + if lmem.sc == nil { lmem.sc = &stores.StorageConfig{StoragePaths: []stores.LocalPath{ {Path: lmem.Path()}, @@ -60,6 +67,10 @@ func (lmem *lockedMemRepo) GetStorage() (stores.StorageConfig, error) { } func (lmem *lockedMemRepo) SetStorage(c func(*stores.StorageConfig)) error { + if err := lmem.checkToken(); err != nil { + return err + } + _, _ = lmem.GetStorage() c(lmem.sc) @@ -227,23 +238,33 @@ func (lmem *lockedMemRepo) Config() (interface{}, error) { return nil, err } - if lmem.mem.config == nil { - lmem.mem.config = lmem.mem.configF(lmem.t) + lmem.mem.config.Lock() + defer lmem.mem.config.Unlock() + + if lmem.mem.config.val == nil { + lmem.mem.config.val = lmem.mem.configF(lmem.t) } - return lmem.mem.config, nil + return lmem.mem.config.val, nil } -func (lmem *lockedMemRepo) SetConfig(cfg interface{}) error { - lmem.mem.config = cfg +func (lmem *lockedMemRepo) SetConfig(c func(interface{})) error { + if err := lmem.checkToken(); err != nil { + return err + } + + lmem.mem.config.Lock() + defer lmem.mem.config.Unlock() + + if lmem.mem.config.val == nil { + lmem.mem.config.val = lmem.mem.configF(lmem.t) + } + + c(lmem.mem.config.val) return nil } -func (lmem *lockedMemRepo) Storage() (stores.StorageConfig, error) { - panic("implement me") -} - func (lmem *lockedMemRepo) SetAPIEndpoint(ma multiaddr.Multiaddr) error { if err := lmem.checkToken(); err != nil { return err diff --git a/node/repo/repo_test.go b/node/repo/repo_test.go index 5da4bcadc..444fab267 100644 --- a/node/repo/repo_test.go +++ b/node/repo/repo_test.go @@ -54,9 +54,10 @@ func basicTest(t *testing.T, repo Repo) { assert.NoError(t, err, "config should not error") // mutate config and persist back to repo - cfg1 := c1.(*config.FullNode) - cfg1.Client.IpfsMAddr = "duvall" - err = lrepo.SetConfig(cfg1) + err = lrepo.SetConfig(func(c interface{}) { + cfg := c.(*config.FullNode) + cfg.Client.IpfsMAddr = "duvall" + }) assert.NoError(t, err) // load config and verify changes From 8354bd91db08764128f8d0d2abba576a5c3d9f9c Mon Sep 17 00:00:00 2001 From: waynewyang Date: Thu, 11 Jun 2020 23:52:44 +0800 Subject: [PATCH 102/160] fix:the condition of time-offset mining fault --- chain/vm/syscalls.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index 143807ab4..2349bc324 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -115,7 +115,7 @@ func (ss *syscallShim) VerifyConsensusFault(a, b, extra []byte) (*runtime.Consen // (b) time-offset mining fault // strictly speaking no need to compare heights based on double fork mining check above, // but at same height this would be a different fault. - if !types.CidArrsEqual(blockA.Parents, blockB.Parents) && blockA.Height != blockB.Height { + if types.CidArrsEqual(blockA.Parents, blockB.Parents) && blockA.Height != blockB.Height { consensusFault = &runtime.ConsensusFault{ Target: blockA.Miner, Epoch: blockB.Height, @@ -159,7 +159,7 @@ func (ss *syscallShim) VerifyConsensusFault(a, b, extra []byte) (*runtime.Consen } if sigErr := ss.VerifyBlockSig(&blockB); sigErr != nil { - return nil, xerrors.Errorf("cannot verify first block sig: %w", sigErr) + return nil, xerrors.Errorf("cannot verify second block sig: %w", sigErr) } return consensusFault, nil From d6b2519843ffe12657b8b9c2d7f5fd89cb57223c Mon Sep 17 00:00:00 2001 From: laser Date: Thu, 11 Jun 2020 11:29:59 -0700 Subject: [PATCH 103/160] config-driven IsAcceptingStorageDeals flag Makes incremental progress towards #1920. --- node/builder.go | 1 + node/config/def.go | 7 ++++++- node/modules/dtypes/miner.go | 4 ++++ node/modules/storageminer.go | 34 ++++++++++++++++++++++++++++++++-- 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/node/builder.go b/node/builder.go index f503bff5c..adbb25efe 100644 --- a/node/builder.go +++ b/node/builder.go @@ -313,6 +313,7 @@ func Online() Option { Override(HandleDealsKey, modules.HandleDeals), Override(new(gen.WinningPoStProver), storage.NewWinningPoStProver), Override(new(*miner.Miner), modules.SetupBlockProducer), + Override(new(dtypes.IsAcceptingStorageDealsFunc), modules.NewIsAcceptingStorageDealsFunc), ), ) } diff --git a/node/config/def.go b/node/config/def.go index 9ca97bb47..a1b03b899 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -27,7 +27,12 @@ type FullNode struct { type StorageMiner struct { Common - Storage sectorstorage.SealerConfig + StorageDeals StorageDealConfig + Storage sectorstorage.SealerConfig +} + +type StorageDealConfig struct { + IsAcceptingStorageDeals bool } // API contains configs for API endpoint diff --git a/node/modules/dtypes/miner.go b/node/modules/dtypes/miner.go index c872fdf69..9a3fcc2cd 100644 --- a/node/modules/dtypes/miner.go +++ b/node/modules/dtypes/miner.go @@ -7,3 +7,7 @@ import ( type MinerAddress address.Address type MinerID abi.ActorID + +// IsAcceptingStorageDealsFunc is a function which reads from miner config to +// determine if the user has disabled storage deals (or not). +type IsAcceptingStorageDealsFunc func() (bool, error) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 1d358628b..4fd1443d1 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -35,6 +35,7 @@ import ( paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/go-storedcounter" + "github.com/filecoin-project/lotus/node/config" sectorstorage "github.com/filecoin-project/sector-storage" "github.com/filecoin-project/sector-storage/ffiwrapper" "github.com/filecoin-project/sector-storage/stores" @@ -304,14 +305,27 @@ func NewStorageAsk(ctx helpers.MetricsCtx, fapi lapi.FullNode, ds dtypes.Metadat return storedAsk, nil } -func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Config, storedAsk *storedask.StoredAsk, h host.Host, ds dtypes.MetadataDS, ibs dtypes.StagingBlockstore, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode) (storagemarket.StorageProvider, error) { +func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Config, storedAsk *storedask.StoredAsk, h host.Host, ds dtypes.MetadataDS, ibs dtypes.StagingBlockstore, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode, isAcceptingStorageDealsFunc dtypes.IsAcceptingStorageDealsFunc) (storagemarket.StorageProvider, error) { net := smnet.NewFromLibp2pHost(h) store, err := piecefilestore.NewLocalFileStore(piecefilestore.OsPath(r.Path())) if err != nil { return nil, err } - p, err := storageimpl.NewProvider(net, namespace.Wrap(ds, datastore.NewKey("/deals/provider")), ibs, store, pieceStore, dataTransfer, spn, address.Address(minerAddress), ffiConfig.SealProofType, storedAsk) + opt := storageimpl.CustomDealDecisionLogic(func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) { + willEntertainProposals, err := isAcceptingStorageDealsFunc() + if err != nil { + return false, "IsAcceptingStorageDealsFunc error", err + } + + if !willEntertainProposals { + return false, "user has disabled storage deals", nil + } + + return true, "", nil + }) + + p, err := storageimpl.NewProvider(net, namespace.Wrap(ds, datastore.NewKey("/deals/provider")), ibs, store, pieceStore, dataTransfer, spn, address.Address(minerAddress), ffiConfig.SealProofType, storedAsk, opt) if err != nil { return p, err } @@ -361,3 +375,19 @@ func StorageAuth(ctx helpers.MetricsCtx, ca lapi.Common) (sectorstorage.StorageA headers.Add("Authorization", "Bearer "+string(token)) return sectorstorage.StorageAuth(headers), nil } + +func NewIsAcceptingStorageDealsFunc(r repo.LockedRepo) (dtypes.IsAcceptingStorageDealsFunc, error) { + return func() (bool, error) { + raw, err := r.Config() + if err != nil { + return false, err + } + + cfg, ok := raw.(*config.StorageMiner) + if !ok { + return false, xerrors.New("expected address of config.StorageMiner") + } + + return cfg.StorageDeals.IsAcceptingStorageDeals, nil + }, nil +} From 6acc9a62f83450eb51e22c5062b82730d482f41b Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 11 Jun 2020 20:37:14 +0200 Subject: [PATCH 104/160] Split gas internally into compute gas and storage gas Signed-off-by: Jakub Sztandera --- chain/vm/gas.go | 53 ++++++++++++++++++++++++++++++------------ chain/vm/gas_v0.go | 56 +++++++++++++++++++++++---------------------- chain/vm/runtime.go | 11 +++++---- chain/vm/vm.go | 4 ++-- 4 files changed, 75 insertions(+), 49 deletions(-) diff --git a/chain/vm/gas.go b/chain/vm/gas.go index f1cf89845..c030621f6 100644 --- a/chain/vm/gas.go +++ b/chain/vm/gas.go @@ -12,34 +12,57 @@ import ( "github.com/ipfs/go-cid" ) +const ( + GasStorageMulti = 1 + GasComputeMulti = 1 +) + +type GasCharge struct { + Name string + ComputeGas int64 + StorageGas int64 +} + +func (g GasCharge) Total() int64 { + return g.ComputeGas*GasComputeMulti + g.StorageGas*GasStorageMulti +} + +func newGasCharge(name string, computeGas int64, storageGas int64) GasCharge { + return GasCharge{ + Name: name, + ComputeGas: computeGas, + StorageGas: storageGas, + } +} + // Pricelist provides prices for operations in the VM. // // Note: this interface should be APPEND ONLY since last chain checkpoint type Pricelist interface { // OnChainMessage returns the gas used for storing a message of a given size in the chain. - OnChainMessage(msgSize int) int64 + OnChainMessage(msgSize int) GasCharge // OnChainReturnValue returns the gas used for storing the response of a message in the chain. - OnChainReturnValue(dataSize int) int64 + OnChainReturnValue(dataSize int) GasCharge // OnMethodInvocation returns the gas used when invoking a method. - OnMethodInvocation(value abi.TokenAmount, methodNum abi.MethodNum) int64 + OnMethodInvocation(value abi.TokenAmount, methodNum abi.MethodNum) GasCharge // OnIpldGet returns the gas used for storing an object - OnIpldGet(dataSize int) int64 + OnIpldGet(dataSize int) GasCharge // OnIpldPut returns the gas used for storing an object - OnIpldPut(dataSize int) int64 + OnIpldPut(dataSize int) GasCharge // OnCreateActor returns the gas used for creating an actor - OnCreateActor() int64 + OnCreateActor() GasCharge // OnDeleteActor returns the gas used for deleting an actor - OnDeleteActor() int64 + OnDeleteActor() GasCharge - OnVerifySignature(sigType crypto.SigType, planTextSize int) (int64, error) - OnHashing(dataSize int) int64 - OnComputeUnsealedSectorCid(proofType abi.RegisteredProof, pieces []abi.PieceInfo) int64 - OnVerifySeal(info abi.SealVerifyInfo) int64 - OnVerifyPost(info abi.WindowPoStVerifyInfo) int64 - OnVerifyConsensusFault() int64 + OnVerifySignature(sigType crypto.SigType, planTextSize int) (GasCharge, error) + OnHashing(dataSize int) GasCharge + OnComputeUnsealedSectorCid(proofType abi.RegisteredProof, pieces []abi.PieceInfo) GasCharge + OnVerifySeal(info abi.SealVerifyInfo) GasCharge + OnVerifyPost(info abi.WindowPoStVerifyInfo) GasCharge + OnVerifyConsensusFault() GasCharge } var prices = map[abi.ChainEpoch]Pricelist{ @@ -93,7 +116,7 @@ func PricelistByEpoch(epoch abi.ChainEpoch) Pricelist { type pricedSyscalls struct { under vmr.Syscalls pl Pricelist - chargeGas func(int64) + chargeGas func(GasCharge) } // Verifies that a signature is valid for an address and plaintext. @@ -146,6 +169,6 @@ func (ps pricedSyscalls) VerifyConsensusFault(h1 []byte, h2 []byte, extra []byte } func (ps pricedSyscalls) BatchVerifySeals(inp map[address.Address][]abi.SealVerifyInfo) (map[address.Address][]bool, error) { - ps.chargeGas(0) // TODO: this is only called by the cron actor. Should we even charge gas? + ps.chargeGas(newGasCharge("BatchVerifySeals", 0, 0)) // TODO: this is only called by the cron actor. Should we even charge gas? return ps.under.BatchVerifySeals(inp) } diff --git a/chain/vm/gas_v0.go b/chain/vm/gas_v0.go index a20ac927d..0b6196377 100644 --- a/chain/vm/gas_v0.go +++ b/chain/vm/gas_v0.go @@ -2,6 +2,7 @@ package vm import ( "fmt" + "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/crypto" @@ -84,17 +85,17 @@ type pricelistV0 struct { var _ Pricelist = (*pricelistV0)(nil) // OnChainMessage returns the gas used for storing a message of a given size in the chain. -func (pl *pricelistV0) OnChainMessage(msgSize int) int64 { - return pl.onChainMessageBase + pl.onChainMessagePerByte*int64(msgSize) +func (pl *pricelistV0) OnChainMessage(msgSize int) GasCharge { + return newGasCharge("OnChainMessage", 0, pl.onChainMessageBase+pl.onChainMessagePerByte*int64(msgSize)) } // OnChainReturnValue returns the gas used for storing the response of a message in the chain. -func (pl *pricelistV0) OnChainReturnValue(dataSize int) int64 { - return int64(dataSize) * pl.onChainReturnValuePerByte +func (pl *pricelistV0) OnChainReturnValue(dataSize int) GasCharge { + return newGasCharge("OnChainReturnValue", 0, int64(dataSize)*pl.onChainReturnValuePerByte) } // OnMethodInvocation returns the gas used when invoking a method. -func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.MethodNum) int64 { +func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.MethodNum) GasCharge { ret := pl.sendBase if value != abi.NewTokenAmount(0) { ret += pl.sendTransferFunds @@ -102,62 +103,63 @@ func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.M if methodNum != builtin.MethodSend { ret += pl.sendInvokeMethod } - return ret + return newGasCharge("OnMethodInvocation", ret, 0) } // OnIpldGet returns the gas used for storing an object -func (pl *pricelistV0) OnIpldGet(dataSize int) int64 { - return pl.ipldGetBase + int64(dataSize)*pl.ipldGetPerByte +func (pl *pricelistV0) OnIpldGet(dataSize int) GasCharge { + return newGasCharge("OnIpldGet", pl.ipldGetBase+int64(dataSize)*pl.ipldGetPerByte, 0) } // OnIpldPut returns the gas used for storing an object -func (pl *pricelistV0) OnIpldPut(dataSize int) int64 { - return pl.ipldPutBase + int64(dataSize)*pl.ipldPutPerByte +func (pl *pricelistV0) OnIpldPut(dataSize int) GasCharge { + return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte) } // OnCreateActor returns the gas used for creating an actor -func (pl *pricelistV0) OnCreateActor() int64 { - return pl.createActorBase + pl.createActorExtra +func (pl *pricelistV0) OnCreateActor() GasCharge { + return newGasCharge("OnCreateActor", pl.createActorBase, pl.createActorExtra) } // OnDeleteActor returns the gas used for deleting an actor -func (pl *pricelistV0) OnDeleteActor() int64 { - return pl.deleteActor +func (pl *pricelistV0) OnDeleteActor() GasCharge { + return newGasCharge("OnDeleteActor", 0, pl.deleteActor) } // OnVerifySignature -func (pl *pricelistV0) OnVerifySignature(sigType crypto.SigType, planTextSize int) (int64, error) { +func (pl *pricelistV0) OnVerifySignature(sigType crypto.SigType, planTextSize int) (GasCharge, error) { costFn, ok := pl.verifySignature[sigType] if !ok { - return 0, fmt.Errorf("cost function for signature type %d not supported", sigType) + return GasCharge{}, fmt.Errorf("cost function for signature type %d not supported", sigType) } - return costFn(int64(planTextSize)), nil + sigName, _ := sigType.Name() + return newGasCharge("OnVerifySignature/"+sigName, costFn(int64(planTextSize)), 0), nil } // OnHashing -func (pl *pricelistV0) OnHashing(dataSize int) int64 { - return pl.hashingBase + int64(dataSize)*pl.hashingPerByte +func (pl *pricelistV0) OnHashing(dataSize int) GasCharge { + return newGasCharge("OnHashing", pl.hashingBase+int64(dataSize)*pl.hashingPerByte, 0) } // OnComputeUnsealedSectorCid -func (pl *pricelistV0) OnComputeUnsealedSectorCid(proofType abi.RegisteredProof, pieces []abi.PieceInfo) int64 { +func (pl *pricelistV0) OnComputeUnsealedSectorCid(proofType abi.RegisteredProof, pieces []abi.PieceInfo) GasCharge { // TODO: this needs more cost tunning, check with @lotus - return pl.computeUnsealedSectorCidBase + return newGasCharge("OnComputeUnsealedSectorCid", pl.computeUnsealedSectorCidBase, 0) } // OnVerifySeal -func (pl *pricelistV0) OnVerifySeal(info abi.SealVerifyInfo) int64 { +func (pl *pricelistV0) OnVerifySeal(info abi.SealVerifyInfo) GasCharge { // TODO: this needs more cost tunning, check with @lotus - return pl.verifySealBase + return newGasCharge("OnVerifySeal", pl.verifySealBase, 0) } // OnVerifyPost -func (pl *pricelistV0) OnVerifyPost(info abi.WindowPoStVerifyInfo) int64 { +func (pl *pricelistV0) OnVerifyPost(info abi.WindowPoStVerifyInfo) GasCharge { // TODO: this needs more cost tunning, check with @lotus - return pl.verifyPostBase + return newGasCharge("OnVerifyPost", pl.verifyPostBase, 0) } // OnVerifyConsensusFault -func (pl *pricelistV0) OnVerifyConsensusFault() int64 { - return pl.verifyConsensusFault +func (pl *pricelistV0) OnVerifyConsensusFault() GasCharge { + return newGasCharge("OnVerifyConsensusFault", pl.verifyConsensusFault, 0) } diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 70583e47f..ae3a60616 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -484,14 +484,15 @@ func (rt *Runtime) stateCommit(oldh, newh cid.Cid) aerrors.ActorError { return nil } -func (rt *Runtime) ChargeGas(toUse int64) { - err := rt.chargeGasInternal(toUse) +func (rt *Runtime) ChargeGas(gas GasCharge) { + err := rt.chargeGasInternal(gas) if err != nil { panic(err) } } -func (rt *Runtime) chargeGasInternal(toUse int64) aerrors.ActorError { +func (rt *Runtime) chargeGasInternal(gas GasCharge) aerrors.ActorError { + toUse := gas.Total() if rt.gasUsed+toUse > rt.gasAvailable { rt.gasUsed = rt.gasAvailable return aerrors.Newf(exitcode.SysErrOutOfGas, "not enough gas: used=%d, available=%d", rt.gasUsed, rt.gasAvailable) @@ -500,8 +501,8 @@ func (rt *Runtime) chargeGasInternal(toUse int64) aerrors.ActorError { return nil } -func (rt *Runtime) chargeGasSafe(toUse int64) aerrors.ActorError { - return rt.chargeGasInternal(toUse) +func (rt *Runtime) chargeGasSafe(gas GasCharge) aerrors.ActorError { + return rt.chargeGasInternal(gas) } func (rt *Runtime) Pricelist() Pricelist { diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 42d6d373e..11db0d419 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -62,7 +62,7 @@ func ResolveToKeyAddr(state types.StateTree, cst cbor.IpldStore, addr address.Ad var _ cbor.IpldBlockstore = (*gasChargingBlocks)(nil) type gasChargingBlocks struct { - chargeGas func(int64) + chargeGas func(GasCharge) pricelist Pricelist under cbor.IpldBlockstore } @@ -294,7 +294,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, pl := PricelistByEpoch(vm.blockHeight) - msgGasCost := pl.OnChainMessage(cmsg.ChainLength()) + msgGasCost := pl.OnChainMessage(cmsg.ChainLength()).Total() // this should never happen, but is currently still exercised by some tests if msgGasCost > msg.GasLimit { return &ApplyRet{ From 0200e3def59c0f4bd7a942694503b9368d5b647c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 11 Jun 2020 20:33:15 +0200 Subject: [PATCH 105/160] wdpost: Submit recoveries for sectors within a deadline --- storage/wdpost_run.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 8cd9fdc5f..3a55356ba 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -111,7 +111,7 @@ func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check *abi.BitFi return &sbf, nil } -func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, deadline uint64, ts *types.TipSet) error { +func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, deadline uint64, deadlineSectors *abi.BitField, ts *types.TipSet) error { faults, err := s.api.StateMinerFaults(ctx, s.actor, ts.Key()) if err != nil { return xerrors.Errorf("getting on-chain faults: %w", err) @@ -136,6 +136,11 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, deadline return xerrors.Errorf("subtracting recovered set from fault set: %w", err) } + unrecovered, err = bitfield.IntersectBitField(unrecovered, deadlineSectors) + if err != nil { + return xerrors.Errorf("intersect unrecovered set with deadlineSectors: %w", err) + } + uc, err := unrecovered.Count() if err != nil { return xerrors.Errorf("counting unrecovered sectors: %w", err) @@ -326,7 +331,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo // late to declare them for this deadline declDeadline := (di.Index + 1) % miner.WPoStPeriodDeadlines - if err := s.checkNextRecoveries(ctx, declDeadline, ts); err != nil { + if err := s.checkNextRecoveries(ctx, declDeadline, deadlines.Due[declDeadline], ts); err != nil { // TODO: This is potentially quite bad, but not even trying to post when this fails is objectively worse log.Errorf("checking sector recoveries: %v", err) } From c458b4712a806639ba9dcdb3c64c0133d19f31a2 Mon Sep 17 00:00:00 2001 From: laser Date: Thu, 11 Jun 2020 12:06:38 -0700 Subject: [PATCH 106/160] better rejection messages --- node/modules/storageminer.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 4fd1443d1..0b3f12586 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -315,11 +315,12 @@ func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Con opt := storageimpl.CustomDealDecisionLogic(func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) { willEntertainProposals, err := isAcceptingStorageDealsFunc() if err != nil { - return false, "IsAcceptingStorageDealsFunc error", err + return false, "miner error", err } if !willEntertainProposals { - return false, "user has disabled storage deals", nil + log.Warnf("storage deal acceptance disabled; rejecting storage deal proposal from client: %s", deal.Client.String()) + return false, "miner is not accepting storage deals", nil } return true, "", nil From 5421d0d1c58093ba5fb2b6e20ce522dce1627994 Mon Sep 17 00:00:00 2001 From: laser Date: Thu, 11 Jun 2020 12:15:28 -0700 Subject: [PATCH 107/160] better field and struct names + set default appropriately Storage miners, by default, should be configured to accept storage deals. --- node/config/def.go | 10 +++++++--- node/modules/storageminer.go | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/node/config/def.go b/node/config/def.go index a1b03b899..3debfd64e 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -27,11 +27,11 @@ type FullNode struct { type StorageMiner struct { Common - StorageDeals StorageDealConfig - Storage sectorstorage.SealerConfig + Dealmaking DealmakingConfig + Storage sectorstorage.SealerConfig } -type StorageDealConfig struct { +type DealmakingConfig struct { IsAcceptingStorageDeals bool } @@ -114,6 +114,10 @@ func DefaultStorageMiner() *StorageMiner { AllowCommit: true, AllowUnseal: true, }, + + Dealmaking: DealmakingConfig{ + IsAcceptingStorageDeals: true, + }, } cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" cfg.Common.API.RemoteListenAddress = "127.0.0.1:2345" diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 0b3f12586..3719f621a 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -389,6 +389,6 @@ func NewIsAcceptingStorageDealsFunc(r repo.LockedRepo) (dtypes.IsAcceptingStorag return false, xerrors.New("expected address of config.StorageMiner") } - return cfg.StorageDeals.IsAcceptingStorageDeals, nil + return cfg.Dealmaking.IsAcceptingStorageDeals, nil }, nil } From 67110ce739a11f9e6e35598ea156bd7381a90938 Mon Sep 17 00:00:00 2001 From: laser Date: Thu, 11 Jun 2020 12:20:11 -0700 Subject: [PATCH 108/160] stub enable/disable storage deal commands --- cmd/lotus-storage-miner/main.go | 2 ++ cmd/lotus-storage-miner/market.go | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index ba7722e4e..9ade0a41b 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -33,6 +33,8 @@ func main() { setPriceCmd, workersCmd, provingCmd, + enableCmd, + disableCmd, } jaeger := tracing.SetupJaegerTracing("lotus") defer func() { diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 68b626360..cc0b84298 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -10,6 +10,24 @@ import ( "github.com/urfave/cli/v2" ) +var enableCmd = &cli.Command{ + Name: "enable", + Usage: "Configure the miner to consider storage deal proposals", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + panic("enable storage deals") + }, +} + +var disableCmd = &cli.Command{ + Name: "disable", + Usage: "Configure the miner to reject all storage deal proposals", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + panic("disable storage deals") + }, +} + var setPriceCmd = &cli.Command{ Name: "set-price", Usage: "Set price that miner will accept storage deals at (FIL / GiB / Epoch)", From c90d35869bd74ab1992971a9f1ed62419313838b Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 11 Jun 2020 21:59:39 +0200 Subject: [PATCH 109/160] Add gas tracing Signed-off-by: Jakub Sztandera --- chain/sync.go | 2 +- chain/types/execresult.go | 55 +++++++++++++++++++++++++++++++++++---- chain/vm/runtime.go | 38 +++++++++++++++++++++++---- chain/vm/vm.go | 40 +++++++++++++++++++--------- cli/state.go | 15 +++++++++++ miner/miner.go | 2 +- 6 files changed, 127 insertions(+), 25 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index 03ccc3ab7..f4b32f2bc 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -857,7 +857,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock // Phase 1: syntactic validation, as defined in the spec minGas := vm.PricelistByEpoch(baseTs.Height()).OnChainMessage(msg.ChainLength()) - if err := m.ValidForBlockInclusion(minGas); err != nil { + if err := m.ValidForBlockInclusion(minGas.Total()); err != nil { return err } diff --git a/chain/types/execresult.go b/chain/types/execresult.go index dd5c8fa3e..41372b832 100644 --- a/chain/types/execresult.go +++ b/chain/types/execresult.go @@ -1,12 +1,57 @@ package types -import "time" +import ( + "encoding/json" + "fmt" + "runtime" + "strings" + "time" +) type ExecutionTrace struct { - Msg *Message - MsgRct *MessageReceipt - Error string - Duration time.Duration + Msg *Message + MsgRct *MessageReceipt + Error string + Duration time.Duration + GasCharges []*GasTrace Subcalls []ExecutionTrace } + +type GasTrace struct { + Name string + Location string + TotalGas int64 + ComputeGas int64 + StorageGas int64 + + TimeTaken time.Duration + + Callers []uintptr `json:"-"` +} + +func (gt *GasTrace) MarshalJSON() ([]byte, error) { + type GasTraceCopy GasTrace + if gt.Location == "" { + if len(gt.Callers) != 0 { + frames := runtime.CallersFrames(gt.Callers) + for { + frame, more := frames.Next() + fn := strings.Split(frame.Function, "/") + + split := strings.Split(frame.File, "/") + file := strings.Join(split[len(split)-2:], "/") + gt.Location += fmt.Sprintf("%s@%s:%d", fn[len(fn)-1], file, frame.Line) + if !more { + break + } + gt.Location += "|" + } + } else { + gt.Location = "n/a" + } + } + + cpy := (*GasTraceCopy)(gt) + return json.Marshal(cpy) +} diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index ae3a60616..902ef94bb 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -5,6 +5,8 @@ import ( "context" "encoding/binary" "fmt" + gruntime "runtime" + "time" "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi" @@ -50,10 +52,12 @@ type Runtime struct { origin address.Address originNonce uint64 - executionTrace types.ExecutionTrace - numActorsCreated uint64 - allowInternal bool - callerValidated bool + executionTrace types.ExecutionTrace + numActorsCreated uint64 + allowInternal bool + callerValidated bool + lastGasChargeTime time.Time + lastGasCharge *types.GasTrace } func (rt *Runtime) TotalFilCircSupply() abi.TokenAmount { @@ -392,7 +396,7 @@ func (rt *Runtime) internalSend(from, to address.Address, method abi.MethodNum, } defer st.ClearSnapshot() - ret, errSend, subrt := rt.vm.send(ctx, msg, rt, 0) + ret, errSend, subrt := rt.vm.send(ctx, msg, rt, nil) if errSend != nil { if errRevert := st.Revert(); errRevert != nil { return nil, aerrors.Escalate(errRevert, "failed to revert state tree after failed subcall") @@ -490,9 +494,33 @@ func (rt *Runtime) ChargeGas(gas GasCharge) { panic(err) } } +func (rt *Runtime) finilizeGasTracing() { + if rt.lastGasCharge != nil { + rt.lastGasCharge.TimeTaken = time.Since(rt.lastGasChargeTime) + } +} func (rt *Runtime) chargeGasInternal(gas GasCharge) aerrors.ActorError { toUse := gas.Total() + var callers [3]uintptr + cout := gruntime.Callers(3, callers[:]) + + now := time.Now() + if rt.lastGasCharge != nil { + rt.lastGasCharge.TimeTaken = now.Sub(rt.lastGasChargeTime) + } + + gasTrace := types.GasTrace{ + Name: gas.Name, + TotalGas: toUse, + ComputeGas: gas.ComputeGas, + StorageGas: gas.StorageGas, + Callers: callers[:cout], + } + rt.executionTrace.GasCharges = append(rt.executionTrace.GasCharges, &gasTrace) + rt.lastGasChargeTime = now + rt.lastGasCharge = &gasTrace + if rt.gasUsed+toUse > rt.gasAvailable { rt.gasUsed = rt.gasAvailable return aerrors.Newf(exitcode.SysErrOutOfGas, "not enough gas: used=%d, available=%d", rt.gasUsed, rt.gasAvailable) diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 11db0d419..75bbdfedb 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -96,13 +96,14 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, origin addres originNonce: originNonce, height: vm.blockHeight, - gasUsed: usedGas, - gasAvailable: msg.GasLimit, - numActorsCreated: nac, - pricelist: PricelistByEpoch(vm.blockHeight), - allowInternal: true, - callerValidated: false, - executionTrace: types.ExecutionTrace{Msg: msg}, + gasUsed: usedGas, + gasAvailable: msg.GasLimit, + numActorsCreated: nac, + pricelist: PricelistByEpoch(vm.blockHeight), + allowInternal: true, + callerValidated: false, + executionTrace: types.ExecutionTrace{Msg: msg}, + lastGasChargeTime: time.Now(), } rt.cst = &cbor.BasicIpldStore{ @@ -171,17 +172,17 @@ type ApplyRet struct { } func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, - gasCharge int64) ([]byte, aerrors.ActorError, *Runtime) { + gasCharge *GasCharge) ([]byte, aerrors.ActorError, *Runtime) { start := time.Now() st := vm.cstate - gasUsed := gasCharge origin := msg.From on := msg.Nonce var nac uint64 = 0 + var gasUsed int64 if parent != nil { - gasUsed = parent.gasUsed + gasUsed + gasUsed = parent.gasUsed origin = parent.origin on = parent.originNonce nac = parent.numActorsCreated @@ -189,11 +190,22 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, rt := vm.makeRuntime(ctx, msg, origin, on, gasUsed, nac) if parent != nil { + rt.lastGasChargeTime = parent.lastGasChargeTime + rt.lastGasCharge = parent.lastGasCharge defer func() { parent.gasUsed = rt.gasUsed + parent.lastGasChargeTime = rt.lastGasChargeTime + parent.lastGasCharge = rt.lastGasCharge }() } + if gasCharge != nil { + if err := rt.chargeGasSafe(*gasCharge); err != nil { + // this should never happen + return nil, aerrors.Wrap(err, "not enough gas for initial message charge, this should not happen"), rt + } + } + ret, err := func() ([]byte, aerrors.ActorError) { if aerr := rt.chargeGasSafe(rt.Pricelist().OnMethodInvocation(msg.Value, msg.Method)); aerr != nil { return nil, aerrors.Wrap(aerr, "not enough gas for method invocation") @@ -261,7 +273,7 @@ func checkMessage(msg *types.Message) error { func (vm *VM) ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*ApplyRet, error) { start := time.Now() - ret, actorErr, rt := vm.send(ctx, msg, nil, 0) + ret, actorErr, rt := vm.send(ctx, msg, nil, nil) return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: aerrors.RetCode(actorErr), @@ -294,7 +306,8 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, pl := PricelistByEpoch(vm.blockHeight) - msgGasCost := pl.OnChainMessage(cmsg.ChainLength()).Total() + msgGas := pl.OnChainMessage(cmsg.ChainLength()) + msgGasCost := msgGas.Total() // this should never happen, but is currently still exercised by some tests if msgGasCost > msg.GasLimit { return &ApplyRet{ @@ -377,7 +390,8 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, } defer st.ClearSnapshot() - ret, actorErr, rt := vm.send(ctx, msg, nil, msgGasCost) + ret, actorErr, rt := vm.send(ctx, msg, nil, &msgGas) + rt.finilizeGasTracing() if aerrors.IsFatal(actorErr) { return nil, xerrors.Errorf("[from=%s,to=%s,n=%d,m=%d,h=%d] fatal error: %w", msg.From, msg.To, msg.Nonce, msg.Method, vm.blockHeight, actorErr) } diff --git a/cli/state.go b/cli/state.go index 1b6f676f2..c65eb1713 100644 --- a/cli/state.go +++ b/cli/state.go @@ -975,6 +975,7 @@ func computeStateHtml(ts *types.TipSet, o *api.ComputeStateOutput, getCode func( } .slow-true-false { color: #660; } .slow-true-true { color: #f80; } + table { font-size: 12px; } @@ -1026,6 +1027,13 @@ func computeStateHtml(ts *types.TipSet, o *api.ComputeStateOutput, getCode func( if ir.MsgRct.ExitCode != 0 { fmt.Printf(`
Error: 
%s
`, ir.Error) } + fmt.Printf("\n
Gas Trace" + + "") + for _, gc := range ir.ExecutionTrace.GasCharges { + fmt.Printf("", + gc.Name, gc.TotalGas, gc.ComputeGas, gc.StorageGas, gc.TimeTaken, gc.Location) + } + fmt.Printf("
NameTotal/Compute/StorageTime TakenLocation
%s%d/%d/%d%s%s
\n") fmt.Println("
Execution trace:
") if err := printInternalExecutionsHtml(cid.String(), ir.ExecutionTrace.Subcalls, getCode); err != nil { @@ -1082,6 +1090,13 @@ func printInternalExecutionsHtml(hashName string, trace []types.ExecutionTrace, if im.MsgRct.ExitCode != 0 { fmt.Printf(`
Error: 
%s
`, im.Error) } + fmt.Printf("\n
Gas Trace" + + "") + for _, gc := range im.GasCharges { + fmt.Printf("", + gc.Name, gc.TotalGas, gc.ComputeGas, gc.StorageGas, gc.TimeTaken, gc.Location) + } + fmt.Printf("
NameTotal/Compute/StorageTime TakenLocation
%s%d/%d/%d%s%s
\n") if len(im.Subcalls) > 0 { fmt.Println("
Subcalls:
") if err := printInternalExecutionsHtml(hashName, im.Subcalls, getCode); err != nil { diff --git a/miner/miner.go b/miner/miner.go index f2059ef7f..23f40338d 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -485,7 +485,7 @@ func SelectMessages(ctx context.Context, al ActorLookup, ts *types.TipSet, msgs vmstart := time.Now() minGas := vm.PricelistByEpoch(ts.Height()).OnChainMessage(msg.ChainLength()) // TODO: really should be doing just msg.ChainLength() but the sync side of this code doesnt seem to have access to that - if err := msg.VMMessage().ValidForBlockInclusion(minGas); err != nil { + if err := msg.VMMessage().ValidForBlockInclusion(minGas.Total()); err != nil { log.Warnf("invalid message in message pool: %s", err) continue } From 7587e6c08bc8048802616212dfc14bb8c0d396fe Mon Sep 17 00:00:00 2001 From: laser Date: Thu, 11 Jun 2020 12:59:50 -0700 Subject: [PATCH 110/160] get and set storage deal acceptance through CLI --- api/api_storage.go | 1 + api/apistruct/struct.go | 9 +++++++-- cmd/lotus-storage-miner/main.go | 2 -- cmd/lotus-storage-miner/market.go | 18 ++++++++++++++++-- node/builder.go | 1 + node/impl/storminer.go | 7 +++++++ node/modules/dtypes/miner.go | 4 ++++ node/modules/storageminer.go | 20 ++++++++++++++++++++ 8 files changed, 56 insertions(+), 6 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index 7d3c78b48..e414ed353 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -54,6 +54,7 @@ type StorageMiner interface { DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) + DealsSetIsAcceptingStorageDeals(ctx context.Context, b bool) error StorageAddLocal(ctx context.Context, path string) error } diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 31cd6badd..23164a993 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -220,8 +220,9 @@ type StorageMinerStruct struct { StorageLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error `perm:"admin"` StorageTryLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) (bool, error) `perm:"admin"` - DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` - DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` + DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` + DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` + DealsSetIsAcceptingStorageDeals func(ctx context.Context, b bool) error `perm:"admin"` StorageAddLocal func(ctx context.Context, path string) error `perm:"admin"` } @@ -852,6 +853,10 @@ func (c *StorageMinerStruct) DealsList(ctx context.Context) ([]storagemarket.Sto return c.Internal.DealsList(ctx) } +func (c *StorageMinerStruct) DealsSetIsAcceptingStorageDeals(ctx context.Context, b bool) error { + return c.Internal.DealsSetIsAcceptingStorageDeals(ctx, b) +} + func (c *StorageMinerStruct) StorageAddLocal(ctx context.Context, path string) error { return c.Internal.StorageAddLocal(ctx, path) } diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index 9ade0a41b..ba7722e4e 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -33,8 +33,6 @@ func main() { setPriceCmd, workersCmd, provingCmd, - enableCmd, - disableCmd, } jaeger := tracing.SetupJaegerTracing("lotus") defer func() { diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index cc0b84298..4b9f903c8 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -15,7 +15,13 @@ var enableCmd = &cli.Command{ Usage: "Configure the miner to consider storage deal proposals", Flags: []cli.Flag{}, Action: func(cctx *cli.Context) error { - panic("enable storage deals") + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + return api.DealsSetIsAcceptingStorageDeals(lcli.DaemonContext(cctx), true) }, } @@ -24,7 +30,13 @@ var disableCmd = &cli.Command{ Usage: "Configure the miner to reject all storage deal proposals", Flags: []cli.Flag{}, Action: func(cctx *cli.Context) error { - panic("disable storage deals") + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + return api.DealsSetIsAcceptingStorageDeals(lcli.DaemonContext(cctx), false) }, } @@ -60,6 +72,8 @@ var dealsCmd = &cli.Command{ Subcommands: []*cli.Command{ dealsImportDataCmd, dealsListCmd, + enableCmd, + disableCmd, }, } diff --git a/node/builder.go b/node/builder.go index adbb25efe..bf42eabd6 100644 --- a/node/builder.go +++ b/node/builder.go @@ -314,6 +314,7 @@ func Online() Option { Override(new(gen.WinningPoStProver), storage.NewWinningPoStProver), Override(new(*miner.Miner), modules.SetupBlockProducer), Override(new(dtypes.IsAcceptingStorageDealsFunc), modules.NewIsAcceptingStorageDealsFunc), + Override(new(dtypes.SetAcceptingStorageDealsFunc), modules.NewSetAcceptingStorageDealsFunc), ), ) } diff --git a/node/impl/storminer.go b/node/impl/storminer.go index e48c9fcad..b88541886 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -25,6 +25,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/impl/common" + "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -41,6 +42,8 @@ type StorageMinerAPI struct { Full api.FullNode StorageMgr *sectorstorage.Manager `optional:"true"` *stores.Index + + SetAcceptingStorageDealsFunc dtypes.SetAcceptingStorageDealsFunc } func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) { @@ -206,6 +209,10 @@ func (sm *StorageMinerAPI) DealsList(ctx context.Context) ([]storagemarket.Stora return sm.StorageProvider.ListDeals(ctx) } +func (sm *StorageMinerAPI) DealsSetIsAcceptingStorageDeals(ctx context.Context, b bool) error { + return sm.SetAcceptingStorageDealsFunc(b) +} + func (sm *StorageMinerAPI) DealsImportData(ctx context.Context, deal cid.Cid, fname string) error { fi, err := os.Open(fname) if err != nil { diff --git a/node/modules/dtypes/miner.go b/node/modules/dtypes/miner.go index 9a3fcc2cd..d14a8775e 100644 --- a/node/modules/dtypes/miner.go +++ b/node/modules/dtypes/miner.go @@ -11,3 +11,7 @@ type MinerID abi.ActorID // IsAcceptingStorageDealsFunc is a function which reads from miner config to // determine if the user has disabled storage deals (or not). type IsAcceptingStorageDealsFunc func() (bool, error) + +// SetAcceptingStorageDealsFunc is a function which is used to disable or enable +// storage deal acceptance. +type SetAcceptingStorageDealsFunc func(bool) error diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 3719f621a..9847f7cc2 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -2,6 +2,7 @@ package modules import ( "context" + "errors" "net/http" "github.com/ipfs/go-bitswap" @@ -17,6 +18,7 @@ import ( "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/routing" "go.uber.org/fx" + "go.uber.org/multierr" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -392,3 +394,21 @@ func NewIsAcceptingStorageDealsFunc(r repo.LockedRepo) (dtypes.IsAcceptingStorag return cfg.Dealmaking.IsAcceptingStorageDeals, nil }, nil } + +func NewSetAcceptingStorageDealsFunc(r repo.LockedRepo) (dtypes.SetAcceptingStorageDealsFunc, error) { + return func(b bool) error { + var typeErr error + + setConfigErr := r.SetConfig(func(raw interface{}) { + cfg, ok := raw.(*config.StorageMiner) + if !ok { + typeErr = errors.New("expected storage miner config") + return + } + + cfg.Dealmaking.IsAcceptingStorageDeals = b + }) + + return multierr.Combine(typeErr, setConfigErr) + }, nil +} From 36b327b57b320d48db272812d186252c4d27695a Mon Sep 17 00:00:00 2001 From: laser Date: Thu, 11 Jun 2020 13:18:18 -0700 Subject: [PATCH 111/160] various symbol renames --- api/api_storage.go | 2 +- api/apistruct/struct.go | 10 +++++----- cmd/lotus-storage-miner/market.go | 4 ++-- node/builder.go | 5 +++-- node/config/def.go | 4 ++-- node/impl/storminer.go | 6 +++--- node/modules/dtypes/miner.go | 6 +++--- node/modules/storageminer.go | 14 +++++++------- 8 files changed, 26 insertions(+), 25 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index e414ed353..04ff8311c 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -54,7 +54,7 @@ type StorageMiner interface { DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) - DealsSetIsAcceptingStorageDeals(ctx context.Context, b bool) error + DealsSetAcceptingStorageDeals(context.Context, bool) error StorageAddLocal(ctx context.Context, path string) error } diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 23164a993..199ad2357 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -220,9 +220,9 @@ type StorageMinerStruct struct { StorageLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error `perm:"admin"` StorageTryLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) (bool, error) `perm:"admin"` - DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` - DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` - DealsSetIsAcceptingStorageDeals func(ctx context.Context, b bool) error `perm:"admin"` + DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` + DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` + DealsSetAcceptingStorageDeals func(context.Context, bool) error `perm:"admin"` StorageAddLocal func(ctx context.Context, path string) error `perm:"admin"` } @@ -853,8 +853,8 @@ func (c *StorageMinerStruct) DealsList(ctx context.Context) ([]storagemarket.Sto return c.Internal.DealsList(ctx) } -func (c *StorageMinerStruct) DealsSetIsAcceptingStorageDeals(ctx context.Context, b bool) error { - return c.Internal.DealsSetIsAcceptingStorageDeals(ctx, b) +func (c *StorageMinerStruct) DealsSetAcceptingStorageDeals(ctx context.Context, b bool) error { + return c.Internal.DealsSetAcceptingStorageDeals(ctx, b) } func (c *StorageMinerStruct) StorageAddLocal(ctx context.Context, path string) error { diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 4b9f903c8..c2f2555fa 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -21,7 +21,7 @@ var enableCmd = &cli.Command{ } defer closer() - return api.DealsSetIsAcceptingStorageDeals(lcli.DaemonContext(cctx), true) + return api.DealsSetAcceptingStorageDeals(lcli.DaemonContext(cctx), true) }, } @@ -36,7 +36,7 @@ var disableCmd = &cli.Command{ } defer closer() - return api.DealsSetIsAcceptingStorageDeals(lcli.DaemonContext(cctx), false) + return api.DealsSetAcceptingStorageDeals(lcli.DaemonContext(cctx), false) }, } diff --git a/node/builder.go b/node/builder.go index bf42eabd6..e628e999b 100644 --- a/node/builder.go +++ b/node/builder.go @@ -313,8 +313,9 @@ func Online() Option { Override(HandleDealsKey, modules.HandleDeals), Override(new(gen.WinningPoStProver), storage.NewWinningPoStProver), Override(new(*miner.Miner), modules.SetupBlockProducer), - Override(new(dtypes.IsAcceptingStorageDealsFunc), modules.NewIsAcceptingStorageDealsFunc), - Override(new(dtypes.SetAcceptingStorageDealsFunc), modules.NewSetAcceptingStorageDealsFunc), + + Override(new(dtypes.AcceptingStorageDealsConfigFunc), modules.NewAcceptingStorageDealsConfigFunc), + Override(new(dtypes.SetAcceptingStorageDealsConfigFunc), modules.NewSetAcceptingStorageDealsConfigFunc), ), ) } diff --git a/node/config/def.go b/node/config/def.go index 3debfd64e..651e99aed 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -32,7 +32,7 @@ type StorageMiner struct { } type DealmakingConfig struct { - IsAcceptingStorageDeals bool + AcceptingStorageDeals bool } // API contains configs for API endpoint @@ -116,7 +116,7 @@ func DefaultStorageMiner() *StorageMiner { }, Dealmaking: DealmakingConfig{ - IsAcceptingStorageDeals: true, + AcceptingStorageDeals: true, }, } cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" diff --git a/node/impl/storminer.go b/node/impl/storminer.go index b88541886..de80eb4cd 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -43,7 +43,7 @@ type StorageMinerAPI struct { StorageMgr *sectorstorage.Manager `optional:"true"` *stores.Index - SetAcceptingStorageDealsFunc dtypes.SetAcceptingStorageDealsFunc + SetAcceptingStorageDealsConfigFunc dtypes.SetAcceptingStorageDealsConfigFunc } func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) { @@ -209,8 +209,8 @@ func (sm *StorageMinerAPI) DealsList(ctx context.Context) ([]storagemarket.Stora return sm.StorageProvider.ListDeals(ctx) } -func (sm *StorageMinerAPI) DealsSetIsAcceptingStorageDeals(ctx context.Context, b bool) error { - return sm.SetAcceptingStorageDealsFunc(b) +func (sm *StorageMinerAPI) DealsSetAcceptingStorageDeals(ctx context.Context, b bool) error { + return sm.SetAcceptingStorageDealsConfigFunc(b) } func (sm *StorageMinerAPI) DealsImportData(ctx context.Context, deal cid.Cid, fname string) error { diff --git a/node/modules/dtypes/miner.go b/node/modules/dtypes/miner.go index d14a8775e..5c761d3e5 100644 --- a/node/modules/dtypes/miner.go +++ b/node/modules/dtypes/miner.go @@ -8,10 +8,10 @@ import ( type MinerAddress address.Address type MinerID abi.ActorID -// IsAcceptingStorageDealsFunc is a function which reads from miner config to +// AcceptingStorageDealsFunc is a function which reads from miner config to // determine if the user has disabled storage deals (or not). -type IsAcceptingStorageDealsFunc func() (bool, error) +type AcceptingStorageDealsConfigFunc func() (bool, error) // SetAcceptingStorageDealsFunc is a function which is used to disable or enable // storage deal acceptance. -type SetAcceptingStorageDealsFunc func(bool) error +type SetAcceptingStorageDealsConfigFunc func(bool) error diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 9847f7cc2..440aa8593 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -307,7 +307,7 @@ func NewStorageAsk(ctx helpers.MetricsCtx, fapi lapi.FullNode, ds dtypes.Metadat return storedAsk, nil } -func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Config, storedAsk *storedask.StoredAsk, h host.Host, ds dtypes.MetadataDS, ibs dtypes.StagingBlockstore, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode, isAcceptingStorageDealsFunc dtypes.IsAcceptingStorageDealsFunc) (storagemarket.StorageProvider, error) { +func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Config, storedAsk *storedask.StoredAsk, h host.Host, ds dtypes.MetadataDS, ibs dtypes.StagingBlockstore, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode, isAcceptingFunc dtypes.AcceptingStorageDealsConfigFunc) (storagemarket.StorageProvider, error) { net := smnet.NewFromLibp2pHost(h) store, err := piecefilestore.NewLocalFileStore(piecefilestore.OsPath(r.Path())) if err != nil { @@ -315,12 +315,12 @@ func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Con } opt := storageimpl.CustomDealDecisionLogic(func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) { - willEntertainProposals, err := isAcceptingStorageDealsFunc() + b, err := isAcceptingFunc() if err != nil { return false, "miner error", err } - if !willEntertainProposals { + if !b { log.Warnf("storage deal acceptance disabled; rejecting storage deal proposal from client: %s", deal.Client.String()) return false, "miner is not accepting storage deals", nil } @@ -379,7 +379,7 @@ func StorageAuth(ctx helpers.MetricsCtx, ca lapi.Common) (sectorstorage.StorageA return sectorstorage.StorageAuth(headers), nil } -func NewIsAcceptingStorageDealsFunc(r repo.LockedRepo) (dtypes.IsAcceptingStorageDealsFunc, error) { +func NewAcceptingStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.AcceptingStorageDealsConfigFunc, error) { return func() (bool, error) { raw, err := r.Config() if err != nil { @@ -391,11 +391,11 @@ func NewIsAcceptingStorageDealsFunc(r repo.LockedRepo) (dtypes.IsAcceptingStorag return false, xerrors.New("expected address of config.StorageMiner") } - return cfg.Dealmaking.IsAcceptingStorageDeals, nil + return cfg.Dealmaking.AcceptingStorageDeals, nil }, nil } -func NewSetAcceptingStorageDealsFunc(r repo.LockedRepo) (dtypes.SetAcceptingStorageDealsFunc, error) { +func NewSetAcceptingStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.SetAcceptingStorageDealsConfigFunc, error) { return func(b bool) error { var typeErr error @@ -406,7 +406,7 @@ func NewSetAcceptingStorageDealsFunc(r repo.LockedRepo) (dtypes.SetAcceptingStor return } - cfg.Dealmaking.IsAcceptingStorageDeals = b + cfg.Dealmaking.AcceptingStorageDeals = b }) return multierr.Combine(typeErr, setConfigErr) From 071ddf6563f37ef1d0a361102ed95f8411153c8a Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 11 Jun 2020 22:25:24 +0200 Subject: [PATCH 112/160] Add aggregate number, add number of bytes for storage calls Signed-off-by: Jakub Sztandera --- chain/vm/gas_v0.go | 4 ++-- cli/state.go | 24 +++++++++++++++++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/chain/vm/gas_v0.go b/chain/vm/gas_v0.go index 0b6196377..c631beaee 100644 --- a/chain/vm/gas_v0.go +++ b/chain/vm/gas_v0.go @@ -108,12 +108,12 @@ func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.M // OnIpldGet returns the gas used for storing an object func (pl *pricelistV0) OnIpldGet(dataSize int) GasCharge { - return newGasCharge("OnIpldGet", pl.ipldGetBase+int64(dataSize)*pl.ipldGetPerByte, 0) + return newGasCharge(fmt.Sprintf("OnIpldGet:%db", dataSize), pl.ipldGetBase+int64(dataSize)*pl.ipldGetPerByte, 0) } // OnIpldPut returns the gas used for storing an object func (pl *pricelistV0) OnIpldPut(dataSize int) GasCharge { - return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte) + return newGasCharge(fmt.Sprintf("OnIpldPut:%db", dataSize), pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte) } // OnCreateActor returns the gas used for creating an actor diff --git a/cli/state.go b/cli/state.go index c65eb1713..acc648c23 100644 --- a/cli/state.go +++ b/cli/state.go @@ -975,7 +975,11 @@ func computeStateHtml(ts *types.TipSet, o *api.ComputeStateOutput, getCode func( } .slow-true-false { color: #660; } .slow-true-true { color: #f80; } - table { font-size: 12px; } + table { + font-size: 12px; + border-collapse: collapse; + } + tr.sum { border-top: 1px solid black; } @@ -1029,10 +1033,20 @@ func computeStateHtml(ts *types.TipSet, o *api.ComputeStateOutput, getCode func( } fmt.Printf("\n
Gas Trace" + "") + + var sumTotal, sumCompute, sumStorage int64 + var sumTime time.Duration for _, gc := range ir.ExecutionTrace.GasCharges { fmt.Printf("", gc.Name, gc.TotalGas, gc.ComputeGas, gc.StorageGas, gc.TimeTaken, gc.Location) + sumTotal += gc.TotalGas + sumCompute += gc.ComputeGas + sumStorage += gc.StorageGas + sumTime += gc.TimeTaken } + fmt.Printf("", + "Sum", sumTotal, sumCompute, sumStorage, sumTime, "") + fmt.Printf("
NameTotal/Compute/StorageTime TakenLocation
%s%d/%d/%d%s%s
%s%d/%d/%d%s%s
\n") fmt.Println("
Execution trace:
") @@ -1092,10 +1106,18 @@ func printInternalExecutionsHtml(hashName string, trace []types.ExecutionTrace, } fmt.Printf("\n
Gas Trace" + "") + var sumTotal, sumCompute, sumStorage int64 + var sumTime time.Duration for _, gc := range im.GasCharges { fmt.Printf("", gc.Name, gc.TotalGas, gc.ComputeGas, gc.StorageGas, gc.TimeTaken, gc.Location) + sumTotal += gc.TotalGas + sumCompute += gc.ComputeGas + sumStorage += gc.StorageGas + sumTime += gc.TimeTaken } + fmt.Printf("", + "Sum", sumTotal, sumCompute, sumStorage, sumTime, "") fmt.Printf("
NameTotal/Compute/StorageTime TakenLocation
%s%d/%d/%d%s%s
%s%d/%d/%d%s%s
\n") if len(im.Subcalls) > 0 { fmt.Println("
Subcalls:
") From c7e3a5cff4e3c3e0cec5027dde55610c054670bb Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 11 Jun 2020 17:11:38 -0700 Subject: [PATCH 113/160] fix null incrementing --- miner/miner.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/miner/miner.go b/miner/miner.go index 23f40338d..bdeed8ac5 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -188,6 +188,8 @@ func (m *Miner) mine(ctx context.Context) { log.Errorf("failed to submit newly mined block: %s", err) } } else { + base.NullRounds++ + // Wait until the next epoch, plus the propagation delay, so a new tipset // has enough time to form. // @@ -261,7 +263,6 @@ func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*types.BlockMsg, return nil, xerrors.Errorf("failed to get mining base info: %w", err) } if mbi == nil { - base.NullRounds++ return nil, nil } @@ -278,7 +279,6 @@ func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*types.BlockMsg, } if !hasPower { // slashed or just have no power yet - base.NullRounds++ return nil, nil } @@ -302,7 +302,6 @@ func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*types.BlockMsg, } if winner == nil { - base.NullRounds++ return nil, nil } From bd25d6db06e5c7060a2884282c1ddbcd9a8cf959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 12 Jun 2020 13:24:38 +0200 Subject: [PATCH 114/160] compute-state html: Convert to templates --- cli/state.go | 285 ++++++++++++++++++++++++--------------------------- 1 file changed, 134 insertions(+), 151 deletions(-) diff --git a/cli/state.go b/cli/state.go index acc648c23..47a1624c4 100644 --- a/cli/state.go +++ b/cli/state.go @@ -5,10 +5,12 @@ import ( "context" "encoding/json" "fmt" + "os" "reflect" "sort" "strconv" "strings" + "text/template" "time" "github.com/ipfs/go-cid" @@ -31,6 +33,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + "github.com/filecoin-project/specs-actors/actors/runtime/exitcode" "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" @@ -40,7 +43,7 @@ import ( ) type methodMeta struct { - name string + Name string params reflect.Type ret reflect.Type @@ -68,7 +71,7 @@ func init() { nf := rt.NumField() methods[c] = append(methods[c], methodMeta{ - name: "Send", + Name: "Send", params: reflect.TypeOf(new(adt.EmptyValue)), ret: reflect.TypeOf(new(adt.EmptyValue)), }) @@ -78,7 +81,7 @@ func init() { export := reflect.TypeOf(exports[i+1]) methods[c] = append(methods[c], methodMeta{ - name: rt.Field(i).Name, + Name: rt.Field(i).Name, params: export.In(1), ret: export.Out(0), }) @@ -923,7 +926,7 @@ var stateComputeStateCmd = &cli.Command{ return c.Code, nil } - return computeStateHtml(ts, stout, getCode) + return computeStateHtmlT(ts, stout, getCode) } fmt.Println("computed state cid: ", stout.Root) @@ -944,16 +947,8 @@ func printInternalExecutions(prefix string, trace []types.ExecutionTrace) { } } -func codeStr(c cid.Cid) string { - cmh, err := multihash.Decode(c.Hash()) - if err != nil { - panic(err) - } - return string(cmh.Digest) -} - -func computeStateHtml(ts *types.TipSet, o *api.ComputeStateOutput, getCode func(addr address.Address) (cid.Cid, error)) error { - fmt.Printf(` +var compStateTemplate = ` + -
Tipset: %s
-
Height: %d
-
State CID: %s
-
Calls
`, ts.Key(), ts.Height(), o.Root) +
Tipset: {{.TipSet.Key}}
+
Epoch: {{.TipSet.Height}}
+
State CID: {{.Comp.Root}}
+
Calls
+ {{range .Comp.Trace}} + {{template "message" (Call .ExecutionTrace false .Msg.Cid.String)}} + {{end}} + + +` - for _, ir := range o.Trace { - toCode, err := getCode(ir.Msg.To) - if err != nil { - return xerrors.Errorf("getting code for %s: %w", toCode, err) - } +var compStateMsg = ` +
+ {{$code := GetCode .Msg.To}} + {{$h := "h2"}}{{if .Subcall}}{{$h = "h4"}}{{end}} - params, err := jsonParams(toCode, ir.Msg.Method, ir.Msg.Params) - if err != nil { - return xerrors.Errorf("decoding params: %w", err) - } + +
{{.Msg.From}} -> {{.Msg.To}} ({{ToFil .Msg.Value}} FIL), M{{.Msg.Method}}
+ {{if not .Subcall}}
Msg CID: {{.Msg.Cid}}
{{end}} + {{if gt (len .Msg.Params) 0}} + 
{{JsonParams ($code) (.Msg.Method) (.Msg.Params) | html}}
+ {{end}} +
Took {{.Duration}}, Exit: {{.MsgRct.ExitCode}}{{if gt (len .MsgRct.Return) 0}}, Return{{end}}
+ + {{if gt (len .MsgRct.Return) 0}} + 
{{JsonReturn ($code) (.Msg.Method) (.MsgRct.Return) | html}}
+ {{end}} - if len(ir.Msg.Params) != 0 { - params = `
` + params + `
` - } else { - params = "" - } + {{if ne .MsgRct.ExitCode 0}} +
Error: 
{{.Error}}
+ {{end}} - ret, err := jsonReturn(toCode, ir.Msg.Method, ir.MsgRct.Return) - if err != nil { - return xerrors.Errorf("decoding return value: %w", err) - } +
+ Gas Trace + + + {{range .GasCharges}} + + {{end}} + {{with SumGas .GasCharges}} + + {{end}} +
NameTotal/Compute/StorageTime TakenLocation
{{.Name}}{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}}{{.Location}}
Sum{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}}
+
+ {{if gt (len .Subcalls) 0}} +
Subcalls:
+ {{$hash := .Hash}} + {{range .Subcalls}} + {{template "message" (Call . true (printf "%s-%s" $hash .Msg.Cid.String))}} + {{end}} + {{end}} +
` - if len(ir.MsgRct.Return) == 0 { - ret = "
" - } else { - ret = `, Return
` + ret + `
` - } - - slow := ir.Duration > 10*time.Millisecond - veryslow := ir.Duration > 50*time.Millisecond - - cid := ir.Msg.Cid() - - fmt.Printf(`
- -
%s -> %s (%s FIL), M%d
-
Msg CID: %s
-%s -
Took %s, Exit: %d%s -`, cid, cid, codeStr(toCode), methods[toCode][ir.Msg.Method].name, ir.Msg.From, ir.Msg.To, types.FIL(ir.Msg.Value), ir.Msg.Method, cid, params, slow, veryslow, ir.Duration, ir.MsgRct.ExitCode, ir.MsgRct.ExitCode, ret) - if ir.MsgRct.ExitCode != 0 { - fmt.Printf(`
Error: 
%s
`, ir.Error) - } - fmt.Printf("\n
Gas Trace" + - "") - - var sumTotal, sumCompute, sumStorage int64 - var sumTime time.Duration - for _, gc := range ir.ExecutionTrace.GasCharges { - fmt.Printf("", - gc.Name, gc.TotalGas, gc.ComputeGas, gc.StorageGas, gc.TimeTaken, gc.Location) - sumTotal += gc.TotalGas - sumCompute += gc.ComputeGas - sumStorage += gc.StorageGas - sumTime += gc.TimeTaken - } - fmt.Printf("", - "Sum", sumTotal, sumCompute, sumStorage, sumTime, "") - - fmt.Printf("
NameTotal/Compute/StorageTime TakenLocation
%s%d/%d/%d%s%s
%s%d/%d/%d%s%s
\n") - - fmt.Println("
Execution trace:
") - if err := printInternalExecutionsHtml(cid.String(), ir.ExecutionTrace.Subcalls, getCode); err != nil { - return err - } - fmt.Println("
") - } - - fmt.Printf(` -`) - return nil +type compStateHtmlIn struct { + TipSet *types.TipSet + Comp *api.ComputeStateOutput } -func printInternalExecutionsHtml(hashName string, trace []types.ExecutionTrace, getCode func(addr address.Address) (cid.Cid, error)) error { - for i, im := range trace { - hashName := fmt.Sprintf("%s-r%d", hashName, i) - - toCode, err := getCode(im.Msg.To) - if err != nil { - return xerrors.Errorf("getting code for %s: %w", toCode, err) - } - - params, err := jsonParams(toCode, im.Msg.Method, im.Msg.Params) - if err != nil { - return xerrors.Errorf("decoding params: %w", err) - } - - if len(im.Msg.Params) != 0 { - params = `
` + params + `
` - } else { - params = "" - } - - ret, err := jsonReturn(toCode, im.Msg.Method, im.MsgRct.Return) - if err != nil { - return xerrors.Errorf("decoding return value: %w", err) - } - - if len(im.MsgRct.Return) == 0 { - ret = "
" - } else { - ret = `, Return
` + ret + `
` - } - - slow := im.Duration > 10*time.Millisecond - veryslow := im.Duration > 50*time.Millisecond - - fmt.Printf(`
- -
%s -> %s (%s FIL), M%d
-%s -
Took %s, Exit: %d%s -`, hashName, hashName, codeStr(toCode), methods[toCode][im.Msg.Method].name, im.Msg.From, im.Msg.To, types.FIL(im.Msg.Value), im.Msg.Method, params, slow, veryslow, im.Duration, im.MsgRct.ExitCode, im.MsgRct.ExitCode, ret) - if im.MsgRct.ExitCode != 0 { - fmt.Printf(`
Error: 
%s
`, im.Error) - } - fmt.Printf("\n
Gas Trace" + - "") - var sumTotal, sumCompute, sumStorage int64 - var sumTime time.Duration - for _, gc := range im.GasCharges { - fmt.Printf("", - gc.Name, gc.TotalGas, gc.ComputeGas, gc.StorageGas, gc.TimeTaken, gc.Location) - sumTotal += gc.TotalGas - sumCompute += gc.ComputeGas - sumStorage += gc.StorageGas - sumTime += gc.TimeTaken - } - fmt.Printf("", - "Sum", sumTotal, sumCompute, sumStorage, sumTime, "") - fmt.Printf("
NameTotal/Compute/StorageTime TakenLocation
%s%d/%d/%d%s%s
%s%d/%d/%d%s%s
\n") - if len(im.Subcalls) > 0 { - fmt.Println("
Subcalls:
") - if err := printInternalExecutionsHtml(hashName, im.Subcalls, getCode); err != nil { - return err - } - } - fmt.Println("
") +func computeStateHtmlT(ts *types.TipSet, o *api.ComputeStateOutput, getCode func(addr address.Address) (cid.Cid, error)) error { + t, err := template.New("compute_state").Funcs(map[string]interface{}{ + "GetCode": getCode, + "GetMethod": getMethod, + "ToFil": toFil, + "JsonParams": jsonParams, + "JsonReturn": jsonReturn, + "IsSlow": isSlow, + "IsVerySlow": isVerySlow, + "IntExit": func(i exitcode.ExitCode) int64 { return int64(i) }, + "SumGas": sumGas, + "CodeStr": codeStr, + "Call": call, + }).Parse(compStateTemplate) + if err != nil { + return err + } + t, err = t.New("message").Parse(compStateMsg) + if err != nil { + return err } - return nil + return t.ExecuteTemplate(os.Stdout, "compute_state", &compStateHtmlIn{ + TipSet: ts, + Comp: o, + }) +} + +type callMeta struct { + types.ExecutionTrace + Subcall bool + Hash string +} + +func call(e types.ExecutionTrace, subcall bool, hash string) callMeta { + return callMeta{ + ExecutionTrace: e, + Subcall: subcall, + Hash: hash, + } +} + +func codeStr(c cid.Cid) string { + cmh, err := multihash.Decode(c.Hash()) + if err != nil { + panic(err) + } + return string(cmh.Digest) +} + +func getMethod(code cid.Cid, method abi.MethodNum) string { + return methods[code][method].Name +} + +func toFil(f types.BigInt) types.FIL { + return types.FIL(f) +} + +func isSlow(t time.Duration) bool { + return t > 10*time.Millisecond +} + +func isVerySlow(t time.Duration) bool { + return t > 50*time.Millisecond +} + +func sumGas(changes []*types.GasTrace) types.GasTrace { + var out types.GasTrace + for _, gc := range changes { + out.TotalGas += gc.TotalGas + out.ComputeGas += gc.ComputeGas + out.StorageGas += gc.StorageGas + out.TimeTaken += gc.TimeTaken + } + + return out } func jsonParams(code cid.Cid, method abi.MethodNum, params []byte) (string, error) { From 813940762c58e26393e343f7f1f7b40fdfa03748 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 12 Jun 2020 14:46:50 +0200 Subject: [PATCH 115/160] Use html/template Signed-off-by: Jakub Sztandera --- cli/state.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/cli/state.go b/cli/state.go index 47a1624c4..3d92f9000 100644 --- a/cli/state.go +++ b/cli/state.go @@ -5,12 +5,12 @@ import ( "context" "encoding/json" "fmt" + "html/template" "os" "reflect" "sort" "strconv" "strings" - "text/template" "time" "github.com/ipfs/go-cid" @@ -992,9 +992,22 @@ var compStateTemplate = ` var compStateMsg = `
{{$code := GetCode .Msg.To}} - {{$h := "h2"}}{{if .Subcall}}{{$h = "h4"}}{{end}} + -
{{.Msg.From}} -> {{.Msg.To}} ({{ToFil .Msg.Value}} FIL), M{{.Msg.Method}}
{{if not .Subcall}}
Msg CID: {{.Msg.Cid}}
{{end}} {{if gt (len .Msg.Params) 0}} @@ -1049,6 +1062,9 @@ func computeStateHtmlT(ts *types.TipSet, o *api.ComputeStateOutput, getCode func "SumGas": sumGas, "CodeStr": codeStr, "Call": call, + "htmlSafeAttr": func(html string) template.HTMLAttr { + return template.HTMLAttr(html) + }, }).Parse(compStateTemplate) if err != nil { return err From 00385acc1d2561e49ec395086f7780c8e66706ab Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 12 Jun 2020 15:26:58 +0200 Subject: [PATCH 116/160] Remove htmlSafeAttr Signed-off-by: Jakub Sztandera --- cli/state.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cli/state.go b/cli/state.go index 3d92f9000..e2ec3b6d1 100644 --- a/cli/state.go +++ b/cli/state.go @@ -1062,9 +1062,6 @@ func computeStateHtmlT(ts *types.TipSet, o *api.ComputeStateOutput, getCode func "SumGas": sumGas, "CodeStr": codeStr, "Call": call, - "htmlSafeAttr": func(html string) template.HTMLAttr { - return template.HTMLAttr(html) - }, }).Parse(compStateTemplate) if err != nil { return err From 82eb7786d464d963c9e2be23672608e09d07d918 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 12 Jun 2020 16:25:55 +0200 Subject: [PATCH 117/160] Cleanup lint warnings Signed-off-by: Jakub Sztandera --- cli/state.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/state.go b/cli/state.go index e2ec3b6d1..96114864f 100644 --- a/cli/state.go +++ b/cli/state.go @@ -926,7 +926,7 @@ var stateComputeStateCmd = &cli.Command{ return c.Code, nil } - return computeStateHtmlT(ts, stout, getCode) + return computeStateHTMLTempl(ts, stout, getCode) } fmt.Println("computed state cid: ", stout.Root) @@ -1044,12 +1044,12 @@ var compStateMsg = ` {{end}}
` -type compStateHtmlIn struct { +type compStateHTMLIn struct { TipSet *types.TipSet Comp *api.ComputeStateOutput } -func computeStateHtmlT(ts *types.TipSet, o *api.ComputeStateOutput, getCode func(addr address.Address) (cid.Cid, error)) error { +func computeStateHTMLTempl(ts *types.TipSet, o *api.ComputeStateOutput, getCode func(addr address.Address) (cid.Cid, error)) error { t, err := template.New("compute_state").Funcs(map[string]interface{}{ "GetCode": getCode, "GetMethod": getMethod, @@ -1071,7 +1071,7 @@ func computeStateHtmlT(ts *types.TipSet, o *api.ComputeStateOutput, getCode func return err } - return t.ExecuteTemplate(os.Stdout, "compute_state", &compStateHtmlIn{ + return t.ExecuteTemplate(os.Stdout, "compute_state", &compStateHTMLIn{ TipSet: ts, Comp: o, }) From cd69e57a33513089aaf142cd0a4fbab0c95cefdf Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 12 Jun 2020 18:49:29 +0200 Subject: [PATCH 118/160] Expose more callers, ellipsis unimportant ones Signed-off-by: Jakub Sztandera --- chain/types/execresult.go | 68 +++++++++++++++++++++++++++++++++------ chain/vm/runtime.go | 31 ++++++++++++------ chain/vm/vm.go | 4 +-- cli/state.go | 48 ++++++++++++++++++++++++--- 4 files changed, 125 insertions(+), 26 deletions(-) diff --git a/chain/types/execresult.go b/chain/types/execresult.go index 41372b832..57e14dcf9 100644 --- a/chain/types/execresult.go +++ b/chain/types/execresult.go @@ -19,36 +19,84 @@ type ExecutionTrace struct { } type GasTrace struct { - Name string - Location string + Name string + + Location []Loc TotalGas int64 ComputeGas int64 StorageGas int64 TimeTaken time.Duration + Extra interface{} Callers []uintptr `json:"-"` } +type Loc struct { + File string + Line int + Function string +} + +func (l Loc) Show() bool { + ignorePrefix := []string{ + "reflect.", + "github.com/filecoin-project/lotus/chain/vm.(*Invoker).transform", + "github.com/filecoin-project/go-amt-ipld/", + } + for _, pre := range ignorePrefix { + if strings.HasPrefix(l.Function, pre) { + return false + } + } + return true +} +func (l Loc) String() string { + file := strings.Split(l.File, "/") + + fn := strings.Split(l.Function, "/") + var fnpkg string + if len(fn) > 2 { + fnpkg = strings.Join(fn[len(fn)-2:], "/") + } else { + fnpkg = l.Function + } + + return fmt.Sprintf("%s@%s:%d", fnpkg, file[len(file)-1], l.Line) +} + +func (l Loc) Important() bool { + if strings.HasPrefix(l.Function, "github.com/filecoin-project/specs-actors/actors/builtin") { + return true + } + return false +} + func (gt *GasTrace) MarshalJSON() ([]byte, error) { type GasTraceCopy GasTrace - if gt.Location == "" { + if len(gt.Location) == 0 { if len(gt.Callers) != 0 { frames := runtime.CallersFrames(gt.Callers) for { frame, more := frames.Next() - fn := strings.Split(frame.Function, "/") + if frame.Function == "github.com/filecoin-project/lotus/chain/vm.(*VM).ApplyMessage" { + break + } + l := Loc{ + File: frame.File, + Line: frame.Line, + Function: frame.Function, + } + //fn := strings.Split(frame.Function, "/") - split := strings.Split(frame.File, "/") - file := strings.Join(split[len(split)-2:], "/") - gt.Location += fmt.Sprintf("%s@%s:%d", fn[len(fn)-1], file, frame.Line) + //split := strings.Split(frame.File, "/") + //file := strings.Join(split[len(split)-2:], "/") + //gt.Location += fmt.Sprintf("%s@%s:%d", fn[len(fn)-1], file, frame.Line) + gt.Location = append(gt.Location, l) if !more { break } - gt.Location += "|" } - } else { - gt.Location = "n/a" } } diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 902ef94bb..67f7adc03 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -488,22 +488,33 @@ func (rt *Runtime) stateCommit(oldh, newh cid.Cid) aerrors.ActorError { return nil } -func (rt *Runtime) ChargeGas(gas GasCharge) { - err := rt.chargeGasInternal(gas) - if err != nil { - panic(err) - } -} func (rt *Runtime) finilizeGasTracing() { if rt.lastGasCharge != nil { rt.lastGasCharge.TimeTaken = time.Since(rt.lastGasChargeTime) } } -func (rt *Runtime) chargeGasInternal(gas GasCharge) aerrors.ActorError { +func (rt *Runtime) ChargeGas(gas GasCharge) { + err := rt.chargeGasInternal(gas, 1) + if err != nil { + panic(err) + } +} + +func (rt *Runtime) chargeGasFunc(skip int) func(GasCharge) { + return func(gas GasCharge) { + err := rt.chargeGasInternal(gas, 1+skip) + if err != nil { + panic(err) + } + } + +} + +func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError { toUse := gas.Total() - var callers [3]uintptr - cout := gruntime.Callers(3, callers[:]) + var callers [10]uintptr + cout := gruntime.Callers(2+skip, callers[:]) now := time.Now() if rt.lastGasCharge != nil { @@ -530,7 +541,7 @@ func (rt *Runtime) chargeGasInternal(gas GasCharge) aerrors.ActorError { } func (rt *Runtime) chargeGasSafe(gas GasCharge) aerrors.ActorError { - return rt.chargeGasInternal(gas) + return rt.chargeGasInternal(gas, 1) } func (rt *Runtime) Pricelist() Pricelist { diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 75bbdfedb..acae3cde0 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -107,12 +107,12 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, origin addres } rt.cst = &cbor.BasicIpldStore{ - Blocks: &gasChargingBlocks{rt.ChargeGas, rt.pricelist, vm.cst.Blocks}, + Blocks: &gasChargingBlocks{rt.chargeGasFunc(2), rt.pricelist, vm.cst.Blocks}, Atlas: vm.cst.Atlas, } rt.sys = pricedSyscalls{ under: vm.Syscalls, - chargeGas: rt.ChargeGas, + chargeGas: rt.chargeGasFunc(1), pl: rt.pricelist, } diff --git a/cli/state.go b/cli/state.go index 96114864f..f2185c4ab 100644 --- a/cli/state.go +++ b/cli/state.go @@ -972,9 +972,35 @@ var compStateTemplate = ` .slow-true-true { color: #f80; } table { font-size: 12px; - border-collapse: collapse; - } - tr.sum { border-top: 1px solid black; } + border-collapse: collapse; + } + tr { + border-top: 1px solid black; + border-bottom: 1px solid black; + } + tr.sum { border-top: 2px solid black; } + tr:first-child { border-top: none; } + tr:last-child { border-bottom: none; } + + + .ellipsis-content, + .ellipsis-toggle input { + display: none; + } + .ellipsis-toggle { + cursor: pointer; + } + /** + Checked State + **/ + + .ellipsis-toggle input:checked + .ellipsis { + display: none; + } + .ellipsis-toggle input:checked ~ .ellipsis-content { + display: inline; + background-color: #ddd; + } @@ -1028,7 +1054,21 @@ var compStateMsg = ` {{range .GasCharges}} - + + {{end}} {{with SumGas .GasCharges}} From 8ad2b3c3d946428bd10fb8f0cca2851601752838 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 12 Jun 2020 18:58:55 +0200 Subject: [PATCH 119/160] Removed commented out code Signed-off-by: Jakub Sztandera --- chain/types/execresult.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/chain/types/execresult.go b/chain/types/execresult.go index 57e14dcf9..7ffa8c219 100644 --- a/chain/types/execresult.go +++ b/chain/types/execresult.go @@ -87,11 +87,6 @@ func (gt *GasTrace) MarshalJSON() ([]byte, error) { Line: frame.Line, Function: frame.Function, } - //fn := strings.Split(frame.Function, "/") - - //split := strings.Split(frame.File, "/") - //file := strings.Join(split[len(split)-2:], "/") - //gt.Location += fmt.Sprintf("%s@%s:%d", fn[len(fn)-1], file, frame.Line) gt.Location = append(gt.Location, l) if !more { break From 4e9293ba04946ab3b3ef7bb8b11af55c07b17878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 12 Jun 2020 19:16:54 +0100 Subject: [PATCH 120/160] fix a potential race with chain reorgs notifees. --- chain/store/store.go | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index dba5995de..f0fb2a611 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -48,6 +48,9 @@ var log = logging.Logger("chainstore") var chainHeadKey = dstore.NewKey("head") +// ReorgNotifee represents a callback that gets called upon reorgs. +type ReorgNotifee func(rev, app []*types.TipSet) error + type ChainStore struct { bs bstore.Blockstore ds dstore.Datastore @@ -63,8 +66,8 @@ type ChainStore struct { cindex *ChainIndex - reorgCh chan<- reorg - headChangeNotifs []func(rev, app []*types.TipSet) error + reorgCh chan<- reorg + reorgNotifeeCh chan ReorgNotifee mmCache *lru.ARCCache tsCache *lru.ARCCache @@ -89,8 +92,6 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls runtime.Sys cs.cindex = ci - cs.reorgCh = cs.reorgWorker(context.TODO()) - hcnf := func(rev, app []*types.TipSet) error { cs.pubLk.Lock() defer cs.pubLk.Unlock() @@ -122,7 +123,8 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls runtime.Sys return nil } - cs.headChangeNotifs = append(cs.headChangeNotifs, hcnf, hcmetric) + cs.reorgNotifeeCh = make(chan ReorgNotifee) + cs.reorgCh = cs.reorgWorker(context.TODO(), []ReorgNotifee{hcnf, hcmetric}) return cs } @@ -211,8 +213,8 @@ func (cs *ChainStore) SubHeadChanges(ctx context.Context) chan []*api.HeadChange return out } -func (cs *ChainStore) SubscribeHeadChanges(f func(rev, app []*types.TipSet) error) { - cs.headChangeNotifs = append(cs.headChangeNotifs, f) +func (cs *ChainStore) SubscribeHeadChanges(f ReorgNotifee) { + cs.reorgNotifeeCh <- f } func (cs *ChainStore) SetGenesis(b *types.BlockHeader) error { @@ -273,13 +275,19 @@ type reorg struct { new *types.TipSet } -func (cs *ChainStore) reorgWorker(ctx context.Context) chan<- reorg { +func (cs *ChainStore) reorgWorker(ctx context.Context, initialNotifees []ReorgNotifee) chan<- reorg { out := make(chan reorg, 32) + notifees := make([]ReorgNotifee, len(initialNotifees)) + copy(notifees, initialNotifees) + go func() { defer log.Warn("reorgWorker quit") for { select { + case n := <-cs.reorgNotifeeCh: + notifees = append(notifees, n) + case r := <-out: revert, apply, err := cs.ReorgOps(r.old, r.new) if err != nil { @@ -293,7 +301,7 @@ func (cs *ChainStore) reorgWorker(ctx context.Context) chan<- reorg { apply[i], apply[opp] = apply[opp], apply[i] } - for _, hcf := range cs.headChangeNotifs { + for _, hcf := range notifees { if err := hcf(revert, apply); err != nil { log.Error("head change func errored (BAD): ", err) } From 91ea03907709e923ac9499144c40d6e44be37cab Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 12 Jun 2020 20:46:04 +0200 Subject: [PATCH 121/160] Change ellipsis, more accurate timing Signed-off-by: Jakub Sztandera --- chain/vm/runtime.go | 4 +- chain/vm/vm.go | 27 +++++++------- cli/state.go | 89 ++++++++++++++++++++++++++++++++------------- 3 files changed, 81 insertions(+), 39 deletions(-) diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 67f7adc03..be34f075c 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -371,6 +371,8 @@ func (rt *Runtime) Send(to address.Address, method abi.MethodNum, m vmr.CBORMars } func (rt *Runtime) internalSend(from, to address.Address, method abi.MethodNum, value types.BigInt, params []byte) ([]byte, aerrors.ActorError) { + + start := time.Now() ctx, span := trace.StartSpan(rt.ctx, "vmc.Send") defer span.End() if span.IsRecordingEvents() { @@ -396,7 +398,7 @@ func (rt *Runtime) internalSend(from, to address.Address, method abi.MethodNum, } defer st.ClearSnapshot() - ret, errSend, subrt := rt.vm.send(ctx, msg, rt, nil) + ret, errSend, subrt := rt.vm.send(ctx, msg, rt, nil, start) if errSend != nil { if errRevert := st.Revert(); errRevert != nil { return nil, aerrors.Escalate(errRevert, "failed to revert state tree after failed subcall") diff --git a/chain/vm/vm.go b/chain/vm/vm.go index acae3cde0..9a4c9991d 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -96,14 +96,13 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, origin addres originNonce: originNonce, height: vm.blockHeight, - gasUsed: usedGas, - gasAvailable: msg.GasLimit, - numActorsCreated: nac, - pricelist: PricelistByEpoch(vm.blockHeight), - allowInternal: true, - callerValidated: false, - executionTrace: types.ExecutionTrace{Msg: msg}, - lastGasChargeTime: time.Now(), + gasUsed: usedGas, + gasAvailable: msg.GasLimit, + numActorsCreated: nac, + pricelist: PricelistByEpoch(vm.blockHeight), + allowInternal: true, + callerValidated: false, + executionTrace: types.ExecutionTrace{Msg: msg}, } rt.cst = &cbor.BasicIpldStore{ @@ -172,8 +171,7 @@ type ApplyRet struct { } func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, - gasCharge *GasCharge) ([]byte, aerrors.ActorError, *Runtime) { - start := time.Now() + gasCharge *GasCharge, start time.Time) ([]byte, aerrors.ActorError, *Runtime) { st := vm.cstate @@ -189,6 +187,7 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, } rt := vm.makeRuntime(ctx, msg, origin, on, gasUsed, nac) + rt.lastGasChargeTime = start if parent != nil { rt.lastGasChargeTime = parent.lastGasChargeTime rt.lastGasCharge = parent.lastGasCharge @@ -273,7 +272,8 @@ func checkMessage(msg *types.Message) error { func (vm *VM) ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*ApplyRet, error) { start := time.Now() - ret, actorErr, rt := vm.send(ctx, msg, nil, nil) + ret, actorErr, rt := vm.send(ctx, msg, nil, nil, start) + rt.finilizeGasTracing() return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: aerrors.RetCode(actorErr), @@ -390,8 +390,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, } defer st.ClearSnapshot() - ret, actorErr, rt := vm.send(ctx, msg, nil, &msgGas) - rt.finilizeGasTracing() + ret, actorErr, rt := vm.send(ctx, msg, nil, &msgGas, start) if aerrors.IsFatal(actorErr) { return nil, xerrors.Errorf("[from=%s,to=%s,n=%d,m=%d,h=%d] fatal error: %w", msg.From, msg.To, msg.Nonce, msg.Method, vm.blockHeight, actorErr) } @@ -445,6 +444,8 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, return nil, xerrors.Errorf("gas handling math is wrong") } + rt.finilizeGasTracing() + return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: errcode, diff --git a/cli/state.go b/cli/state.go index f2185c4ab..d3d95675c 100644 --- a/cli/state.go +++ b/cli/state.go @@ -1001,6 +1001,12 @@ var compStateTemplate = ` display: inline; background-color: #ddd; } + hr { + border: none; + height: 1px; + background-color: black; + margin: 0; + } @@ -1049,32 +1055,54 @@ var compStateMsg = `
Error: 
{{.Error}}
{{end}} -
- Gas Trace -
NameTotal/Compute/StorageTime TakenLocation
{{.Name}}{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}}{{.Location}}
{{.Name}}{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}} + {{ range $index, $ele := .Location }} + {{- if $index }}|​{{end -}} + {{- if .Show -}} + {{- if .Important }}{{end -}} + {{- . -}} + {{- if .Important }}{{end -}} + {{- else -}} + + {{- end -}} + {{end}} +
Sum{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}}
- - {{range .GasCharges}} - - + {{end}} + {{with SumGas .GasCharges}} + + {{end}} +
NameTotal/Compute/StorageTime TakenLocation
{{.Name}}{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}} - {{ range $index, $ele := .Location }} - {{- if $index }}|​{{end -}} - {{- if .Show -}} - {{- if .Important }}{{end -}} - {{- . -}} - {{- if .Important }}{{end -}} - {{- else -}} - - {{- end -}} +
+Gas Trace + + + {{range .GasCharges}} + + - {{end}} - {{with SumGas .GasCharges}} - - {{end}} -
NameTotal/Compute/StorageTime TakenLocation
{{.Name}}{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}} + {{ $fImp := FirstImportant .Location }} + {{ if $fImp }} +
+ {{ $fImp }}
+ {{ $elipOn := false }} + {{ range $index, $ele := .Location -}} + {{- if $index }}
{{end -}} + {{- if .Show -}} + {{ if $elipOn }} + {{ $elipOn = false }} + + {{end}} + + {{- if .Important }}{{end -}} + {{- . -}} + {{if .Important }}{{end}} + {{else}} + {{ if not $elipOn }} + {{ $elipOn = true }} + {{end}} -
Sum{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}}
-
+ + {{end}} +
Sum{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}}
+ + + {{if gt (len .Subcalls) 0}}
Subcalls:
{{$hash := .Hash}} @@ -1102,6 +1130,17 @@ func computeStateHTMLTempl(ts *types.TipSet, o *api.ComputeStateOutput, getCode "SumGas": sumGas, "CodeStr": codeStr, "Call": call, + "FirstImportant": func(locs []types.Loc) *types.Loc { + if len(locs) != 0 { + for _, l := range locs { + if l.Important() { + return &l + } + } + return &locs[0] + } + return nil + }, }).Parse(compStateTemplate) if err != nil { return err From 464f6a6b08f3267fcf3b5d5aea2bf190c6578c18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 15 Jun 2020 14:49:18 +0200 Subject: [PATCH 122/160] Update specs-actors to v0.6 --- extern/filecoin-ffi | 2 +- go.mod | 4 ++-- go.sum | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 61c02f6be..1bff7f456 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 61c02f6bea8d69bb79c70daa1d62f26c486643aa +Subproject commit 1bff7f4563370ada590a605b5459b91e1662ebaa diff --git a/go.mod b/go.mod index 21fea095f..5cfe55a8d 100644 --- a/go.mod +++ b/go.mod @@ -29,8 +29,8 @@ require ( github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/sector-storage v0.0.0-20200609231555-252c2b0c969d - github.com/filecoin-project/specs-actors v0.5.6 + github.com/filecoin-project/sector-storage v0.0.0-20200615123301-7d09fa88b4b0 + github.com/filecoin-project/specs-actors v0.6.0 github.com/filecoin-project/specs-storage v0.1.0 github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index e10972bb6..5366cf6b6 100644 --- a/go.sum +++ b/go.sum @@ -223,6 +223,8 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/ github.com/filecoin-project/sector-storage v0.0.0-20200508203401-a74812ba12f3/go.mod h1:B+xzopr/oWZJz2hBL5Ekb7Obcum5ntmfbaAUlaaho28= github.com/filecoin-project/sector-storage v0.0.0-20200609231555-252c2b0c969d h1:q7KC8/yIirwVX7JCLgjEmo5LX77V8tYV0EzVCnCwrE8= github.com/filecoin-project/sector-storage v0.0.0-20200609231555-252c2b0c969d/go.mod h1:SfaVNw/A6LwqqEt6wEMVBN5Grn3XC50j+yjmegHIe7Y= +github.com/filecoin-project/sector-storage v0.0.0-20200615123301-7d09fa88b4b0 h1:DpdlpS3G2XN+F68rxOTAXsGL3cV7UkFr0TxdmKSiaJg= +github.com/filecoin-project/sector-storage v0.0.0-20200615123301-7d09fa88b4b0/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= @@ -231,6 +233,8 @@ github.com/filecoin-project/specs-actors v0.5.5 h1:bDsowem6dLRc9B7g3sgFYvHgfWTH4 github.com/filecoin-project/specs-actors v0.5.5/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-actors v0.5.6 h1:WlhtoXwFoKlP1b06NI4NJaxC4m9EXNV+qFVl43/xRN8= github.com/filecoin-project/specs-actors v0.5.6/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= +github.com/filecoin-project/specs-actors v0.6.0 h1:IepUsmDGY60QliENVTkBTAkwqGWw9kNbbHOcU/9oiC0= +github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 h1:T3f/zkuvgtgqcXrb0NO3BicuveGOxxUAMPa/Yif2kuE= github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102/go.mod h1:xJ1/xl9+8zZeSSSFmDC3Wr6uusCTxyYPI0VeNVSFmPE= github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sgOVdJbvFjOnD5w94= From a23a87a17fedaed8f3ff77a17f02e55aca5c2515 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 15 Jun 2020 16:18:05 +0200 Subject: [PATCH 123/160] Add virtual gas Signed-off-by: Jakub Sztandera --- chain/types/execresult.go | 11 +++++++---- chain/vm/gas.go | 9 +++++++++ chain/vm/runtime.go | 16 ++++++++++------ cli/state.go | 25 ++++++++++++++++++++++--- 4 files changed, 48 insertions(+), 13 deletions(-) diff --git a/chain/types/execresult.go b/chain/types/execresult.go index 7ffa8c219..b0e1f760d 100644 --- a/chain/types/execresult.go +++ b/chain/types/execresult.go @@ -21,10 +21,13 @@ type ExecutionTrace struct { type GasTrace struct { Name string - Location []Loc - TotalGas int64 - ComputeGas int64 - StorageGas int64 + Location []Loc + TotalGas int64 + ComputeGas int64 + StorageGas int64 + TotalVirtualGas int64 + VirtualComputeGas int64 + VirtualStorageGas int64 TimeTaken time.Duration Extra interface{} diff --git a/chain/vm/gas.go b/chain/vm/gas.go index c030621f6..dc9966e57 100644 --- a/chain/vm/gas.go +++ b/chain/vm/gas.go @@ -21,11 +21,20 @@ type GasCharge struct { Name string ComputeGas int64 StorageGas int64 + + VirtualCompute int64 + VirtualStorage int64 } func (g GasCharge) Total() int64 { return g.ComputeGas*GasComputeMulti + g.StorageGas*GasStorageMulti } +func (g GasCharge) WithVirtual(compute, storage int64) GasCharge { + out := g + out.VirtualCompute = compute + out.VirtualStorage = storage + return out +} func newGasCharge(name string, computeGas int64, storageGas int64) GasCharge { return GasCharge{ diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index be34f075c..9b83593ab 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -524,11 +524,14 @@ func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError } gasTrace := types.GasTrace{ - Name: gas.Name, - TotalGas: toUse, - ComputeGas: gas.ComputeGas, - StorageGas: gas.StorageGas, - Callers: callers[:cout], + Name: gas.Name, + TotalGas: toUse, + ComputeGas: gas.ComputeGas, + StorageGas: gas.StorageGas, + TotalVirtualGas: gas.VirtualCompute*GasComputeMulti + gas.VirtualStorage*GasStorageMulti, + VirtualComputeGas: gas.VirtualCompute, + VirtualStorageGas: gas.VirtualStorage, + Callers: callers[:cout], } rt.executionTrace.GasCharges = append(rt.executionTrace.GasCharges, &gasTrace) rt.lastGasChargeTime = now @@ -536,7 +539,8 @@ func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError if rt.gasUsed+toUse > rt.gasAvailable { rt.gasUsed = rt.gasAvailable - return aerrors.Newf(exitcode.SysErrOutOfGas, "not enough gas: used=%d, available=%d", rt.gasUsed, rt.gasAvailable) + return aerrors.Newf(exitcode.SysErrOutOfGas, "not enough gas: used=%d, available=%d", + rt.gasUsed, rt.gasAvailable) } rt.gasUsed += toUse return nil diff --git a/cli/state.go b/cli/state.go index bb5dd4277..753b0b017 100644 --- a/cli/state.go +++ b/cli/state.go @@ -971,6 +971,7 @@ var compStateTemplate = ` } .slow-true-false { color: #660; } .slow-true-true { color: #f80; } + .deemp { color: #444; } table { font-size: 12px; border-collapse: collapse; @@ -1060,8 +1061,20 @@ var compStateMsg = ` Gas Trace + {{define "virt" -}} + {{- if . -}} + +({{.}}) + {{- end -}} + {{- end}} + + {{define "gasC" -}} + + {{- end}} + {{range .GasCharges}} - + + {{template "gasC" .}} + {{end}} {{with SumGas .GasCharges}} - + + {{template "gasC" .}} + + {{end}}
NameTotal/Compute/StorageTime TakenLocation
{{.TotalGas}}{{template "virt" .TotalVirtualGas }}/{{.ComputeGas}}{{template "virt" .VirtualComputeGas}}/{{.StorageGas}}{{template "virt" .VirtualStorageGas}}
{{.Name}}{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}}
{{.Name}}{{.TimeTaken}} {{ $fImp := FirstImportant .Location }} {{ if $fImp }} @@ -1098,7 +1111,10 @@ var compStateMsg = `
Sum{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}}
Sum{{.TimeTaken}}
@@ -1201,7 +1217,10 @@ func sumGas(changes []*types.GasTrace) types.GasTrace { out.TotalGas += gc.TotalGas out.ComputeGas += gc.ComputeGas out.StorageGas += gc.StorageGas - out.TimeTaken += gc.TimeTaken + + out.TotalVirtualGas += gc.TotalVirtualGas + out.VirtualComputeGas += gc.VirtualComputeGas + out.VirtualStorageGas += gc.VirtualStorageGas } return out From 2a78ebabd9117de39ac344cea12bd7845b528558 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 17 Apr 2020 20:16:36 +0200 Subject: [PATCH 124/160] Use HashVerify Signed-off-by: Jakub Sztandera --- chain/sync.go | 18 ++++-------------- lib/sigs/bls/init.go | 4 ++-- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index f4b32f2bc..bfbca0ec1 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -8,7 +8,6 @@ import ( "os" "sort" "strings" - "sync" "time" "github.com/Gurpartap/async" @@ -949,23 +948,14 @@ func (syncer *Syncer) verifyBlsAggregate(ctx context.Context, sig *crypto.Signat trace.Int64Attribute("msgCount", int64(len(msgs))), ) - var wg sync.WaitGroup - - digests := make([]bls.Digest, len(msgs)) - for i := 0; i < 10; i++ { - wg.Add(1) - go func(w int) { - defer wg.Done() - for j := 0; (j*10)+w < len(msgs); j++ { - digests[j*10+w] = bls.Hash(bls.Message(msgs[j*10+w].Bytes())) - } - }(i) + bmsgs := make([]bls.Message, len(msgs)) + for i, m := range msgs { + bmsgs[i] = m.Bytes() } - wg.Wait() var bsig bls.Signature copy(bsig[:], sig.Data) - if !bls.Verify(&bsig, digests, pubks) { + if !bls.HashVerify(&bsig, bmsgs, pubks) { return xerrors.New("bls aggregate signature failed to verify") } diff --git a/lib/sigs/bls/init.go b/lib/sigs/bls/init.go index fe916c446..66a5ade81 100644 --- a/lib/sigs/bls/init.go +++ b/lib/sigs/bls/init.go @@ -33,16 +33,16 @@ func (blsSigner) Sign(p []byte, msg []byte) ([]byte, error) { } func (blsSigner) Verify(sig []byte, a address.Address, msg []byte) error { - digests := []ffi.Digest{ffi.Hash(ffi.Message(msg))} var pubk ffi.PublicKey copy(pubk[:], a.Payload()) pubkeys := []ffi.PublicKey{pubk} + digests := []ffi.Message{msg} var s ffi.Signature copy(s[:], sig) - if !ffi.Verify(&s, digests, pubkeys) { + if !ffi.HashVerify(&s, digests, pubkeys) { return fmt.Errorf("bls signature failed to verify") } From 907364ce670c394ae3308e63728005f591468836 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 15 Jun 2020 18:30:49 +0200 Subject: [PATCH 125/160] Update deps, fix more tests --- api/apistruct/struct.go | 26 +++++++++++++------------- api/docgen/docgen.go | 2 +- api/types.go | 2 +- build/params_testnet.go | 8 ++++---- chain/gen/gen.go | 10 +++++----- chain/gen/gen_test.go | 4 ++-- chain/gen/genesis/miners.go | 12 ++++++------ chain/stmgr/forks_test.go | 4 ++-- chain/stmgr/stmgr.go | 7 +++---- chain/stmgr/utils.go | 6 +++--- chain/store/store_test.go | 4 ++-- chain/sync_test.go | 4 ++-- chain/types/blockheader_test.go | 9 ++++----- chain/vm/gas.go | 4 ++-- chain/vm/gas_v0.go | 2 +- chain/vm/syscalls.go | 2 +- cmd/lotus-bench/main.go | 23 ++++++++++++++--------- cmd/lotus-seed/seed/seed.go | 11 +++-------- cmd/lotus-shed/proofs.go | 2 +- extern/filecoin-ffi | 2 +- genesis/types.go | 2 +- go.mod | 10 ++++++---- go.sum | 10 ++++++++++ lotuspond/spawn.go | 6 +++--- node/impl/full/state.go | 2 +- node/node_test.go | 6 +++--- storage/adapter_storage_miner.go | 2 +- storage/miner.go | 2 +- storage/wdpost_run.go | 6 +++--- storage/wdpost_sched.go | 2 +- 30 files changed, 101 insertions(+), 91 deletions(-) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 31cd6badd..c607c13a4 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -207,18 +207,18 @@ type StorageMinerStruct struct { WorkerConnect func(context.Context, string) error `perm:"admin"` // TODO: worker perm WorkerStats func(context.Context) (map[uint64]storiface.WorkerStats, error) `perm:"admin"` - StorageList func(context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"` - StorageLocal func(context.Context) (map[stores.ID]string, error) `perm:"admin"` - StorageStat func(context.Context, stores.ID) (stores.FsStat, error) `perm:"admin"` - StorageAttach func(context.Context, stores.StorageInfo, stores.FsStat) error `perm:"admin"` - StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType, bool) error `perm:"admin"` - StorageDropSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType) error `perm:"admin"` - StorageFindSector func(context.Context, abi.SectorID, stores.SectorFileType, bool) ([]stores.SectorStorageInfo, error) `perm:"admin"` - StorageInfo func(context.Context, stores.ID) (stores.StorageInfo, error) `perm:"admin"` - StorageBestAlloc func(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredProof, sealing stores.PathType) ([]stores.StorageInfo, error) `perm:"admin"` - StorageReportHealth func(ctx context.Context, id stores.ID, report stores.HealthReport) error `perm:"admin"` - StorageLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error `perm:"admin"` - StorageTryLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) (bool, error) `perm:"admin"` + StorageList func(context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"` + StorageLocal func(context.Context) (map[stores.ID]string, error) `perm:"admin"` + StorageStat func(context.Context, stores.ID) (stores.FsStat, error) `perm:"admin"` + StorageAttach func(context.Context, stores.StorageInfo, stores.FsStat) error `perm:"admin"` + StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType, bool) error `perm:"admin"` + StorageDropSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType) error `perm:"admin"` + StorageFindSector func(context.Context, abi.SectorID, stores.SectorFileType, bool) ([]stores.SectorStorageInfo, error) `perm:"admin"` + StorageInfo func(context.Context, stores.ID) (stores.StorageInfo, error) `perm:"admin"` + StorageBestAlloc func(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredSealProof, sealing stores.PathType) ([]stores.StorageInfo, error) `perm:"admin"` + StorageReportHealth func(ctx context.Context, id stores.ID, report stores.HealthReport) error `perm:"admin"` + StorageLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error `perm:"admin"` + StorageTryLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) (bool, error) `perm:"admin"` DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` @@ -812,7 +812,7 @@ func (c *StorageMinerStruct) StorageInfo(ctx context.Context, id stores.ID) (sto return c.Internal.StorageInfo(ctx, id) } -func (c *StorageMinerStruct) StorageBestAlloc(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredProof, pt stores.PathType) ([]stores.StorageInfo, error) { +func (c *StorageMinerStruct) StorageBestAlloc(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredSealProof, pt stores.PathType) ([]stores.StorageInfo, error) { return c.Internal.StorageBestAlloc(ctx, allocate, spt, pt) } diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index d0b5d8558..4cd982aa7 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -72,7 +72,7 @@ func init() { addExample(pid) addExample(bitfield.NewFromSet([]uint64{5})) - addExample(abi.RegisteredProof_StackedDRG32GiBPoSt) + addExample(abi.RegisteredSealProof_StackedDrg32GiBV1) addExample(abi.ChainEpoch(10101)) addExample(crypto.SigTypeBLS) addExample(int64(9)) diff --git a/api/types.go b/api/types.go index f5a595353..29bd7401c 100644 --- a/api/types.go +++ b/api/types.go @@ -50,7 +50,7 @@ type MinerInfo struct { WorkerChangeEpoch abi.ChainEpoch PeerId peer.ID Multiaddrs []abi.Multiaddrs - SealProofType abi.RegisteredProof + SealProofType abi.RegisteredSealProof SectorSize abi.SectorSize WindowPoStPartitionSectors uint64 } diff --git a/build/params_testnet.go b/build/params_testnet.go index 917b8059e..c222862eb 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -13,10 +13,10 @@ import ( func init() { power.ConsensusMinerMinPower = big.NewInt(1024 << 20) - miner.SupportedProofTypes = map[abi.RegisteredProof]struct{}{ - abi.RegisteredProof_StackedDRG512MiBSeal: {}, - abi.RegisteredProof_StackedDRG32GiBSeal: {}, - abi.RegisteredProof_StackedDRG64GiBSeal: {}, + miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg512MiBV1: {}, + abi.RegisteredSealProof_StackedDrg32GiBV1: {}, + abi.RegisteredSealProof_StackedDrg64GiBV1: {}, } } diff --git a/chain/gen/gen.go b/chain/gen/gen.go index d58bbc28e..368240a04 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -89,8 +89,8 @@ func (m mybs) Get(c cid.Cid) (block.Block, error) { } func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { - saminer.SupportedProofTypes = map[abi.RegisteredProof]struct{}{ - abi.RegisteredProof_StackedDRG2KiBSeal: {}, + saminer.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } mr := repo.NewMemory(nil) @@ -141,7 +141,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { return nil, err } - genm1, k1, err := seed.PreSeal(maddr1, abi.RegisteredProof_StackedDRG2KiBPoSt, 0, numSectors, m1temp, []byte("some randomness"), nil) + genm1, k1, err := seed.PreSeal(maddr1, abi.RegisteredSealProof_StackedDrg2KiBV1, 0, numSectors, m1temp, []byte("some randomness"), nil) if err != nil { return nil, err } @@ -153,7 +153,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { return nil, err } - genm2, k2, err := seed.PreSeal(maddr2, abi.RegisteredProof_StackedDRG2KiBPoSt, 0, numSectors, m2temp, []byte("some randomness"), nil) + genm2, k2, err := seed.PreSeal(maddr2, abi.RegisteredSealProof_StackedDrg2KiBV1, 0, numSectors, m2temp, []byte("some randomness"), nil) if err != nil { return nil, err } @@ -611,6 +611,6 @@ func (m genFakeVerifier) VerifyWindowPoSt(ctx context.Context, info abi.WindowPo panic("not supported") } -func (m genFakeVerifier) GenerateWinningPoStSectorChallenge(ctx context.Context, proof abi.RegisteredProof, id abi.ActorID, randomness abi.PoStRandomness, u uint64) ([]uint64, error) { +func (m genFakeVerifier) GenerateWinningPoStSectorChallenge(ctx context.Context, proof abi.RegisteredPoStProof, id abi.ActorID, randomness abi.PoStRandomness, u uint64) ([]uint64, error) { panic("not supported") } diff --git a/chain/gen/gen_test.go b/chain/gen/gen_test.go index 69c8587ef..7a4d73031 100644 --- a/chain/gen/gen_test.go +++ b/chain/gen/gen_test.go @@ -14,8 +14,8 @@ import ( ) func init() { - miner.SupportedProofTypes = map[abi.RegisteredProof]struct{}{ - abi.RegisteredProof_StackedDRG2KiBSeal: {}, + miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } power.ConsensusMinerMinPower = big.NewInt(2048) verifreg.MinVerifiedDealSize = big.NewInt(256) diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index 685fc58cc..acc63c31c 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -204,12 +204,12 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid { newSectorInfo := &miner.SectorOnChainInfo{ Info: miner.SectorPreCommitInfo{ - RegisteredProof: preseal.ProofType, - SectorNumber: preseal.SectorID, - SealedCID: preseal.CommR, - SealRandEpoch: 0, - DealIDs: []abi.DealID{dealIDs[pi]}, - Expiration: preseal.Deal.EndEpoch, + SealProof: preseal.ProofType, + SectorNumber: preseal.SectorID, + SealedCID: preseal.CommR, + SealRandEpoch: 0, + DealIDs: []abi.DealID{dealIDs[pi]}, + Expiration: preseal.Deal.EndEpoch, }, ActivationEpoch: 0, DealWeight: dealWeight.DealWeight, diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index a3b01cd84..2fbcbbc99 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -37,8 +37,8 @@ import ( ) func init() { - miner.SupportedProofTypes = map[abi.RegisteredProof]struct{}{ - abi.RegisteredProof_StackedDRG2KiBSeal: {}, + miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } power.ConsensusMinerMinPower = big.NewInt(2048) verifreg.MinVerifiedDealSize = big.NewInt(256) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index d4c12dffe..917b5ca26 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -184,10 +184,9 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, pstate cid.Cid, bms []B var err error params, err := actors.SerializeParams(&reward.AwardBlockRewardParams{ - Miner: b.Miner, - Penalty: penalty, - GasReward: gasReward, - TicketCount: 1, // TODO: no longer need ticket count here. + Miner: b.Miner, + Penalty: penalty, + GasReward: gasReward, }) if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("failed to serialize award params: %w", err) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 308c30823..cf424b4eb 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -206,9 +206,9 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S out := make([]abi.SectorInfo, len(ids)) for i, n := range ids { out[i] = abi.SectorInfo{ - RegisteredProof: wpt, - SectorNumber: sectorSet[n].ID, - SealedCID: sectorSet[n].Info.Info.SealedCID, + SealProof: spt, + SectorNumber: sectorSet[n].ID, + SealedCID: sectorSet[n].Info.Info.SealedCID, } } diff --git a/chain/store/store_test.go b/chain/store/store_test.go index 41b875916..939c85d20 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -22,8 +22,8 @@ import ( ) func init() { - miner.SupportedProofTypes = map[abi.RegisteredProof]struct{}{ - abi.RegisteredProof_StackedDRG2KiBSeal: {}, + miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } power.ConsensusMinerMinPower = big.NewInt(2048) verifreg.MinVerifiedDealSize = big.NewInt(256) diff --git a/chain/sync_test.go b/chain/sync_test.go index e1c846d4e..92c0f72d7 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -34,8 +34,8 @@ import ( func init() { build.InsecurePoStValidation = true os.Setenv("TRUST_PARAMS", "1") - miner.SupportedProofTypes = map[abi.RegisteredProof]struct{}{ - abi.RegisteredProof_StackedDRG2KiBSeal: {}, + miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } power.ConsensusMinerMinPower = big.NewInt(2048) verifreg.MinVerifiedDealSize = big.NewInt(256) diff --git a/chain/types/blockheader_test.go b/chain/types/blockheader_test.go index 387fe86c2..a1ece308a 100644 --- a/chain/types/blockheader_test.go +++ b/chain/types/blockheader_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" "fmt" + "github.com/stretchr/testify/require" "reflect" "testing" @@ -79,7 +80,7 @@ func TestInteropBH(t *testing.T) { } posts := []abi.PoStProof{ - {abi.RegisteredProof_StackedDRG2KiBWinningPoSt, []byte{0x07}}, + {abi.RegisteredPoStProof_StackedDrgWinning2KiBV1, []byte{0x07}}, } bh := &BlockHeader{ @@ -116,10 +117,8 @@ func TestInteropBH(t *testing.T) { } // acquired from go-filecoin - gfc := "8f5501d04cb15021bf6bd003073d79e2238d4e61f1ad22814301020381420a0b818205410c818209410781d82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619cc430003e802d82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619ccd82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619ccd82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619cc410001f603" - if gfc != hex.EncodeToString(bhsb) { - t.Fatal("not equal!") - } + gfc := "8f5501d04cb15021bf6bd003073d79e2238d4e61f1ad22814301020381420a0b818205410c818200410781d82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619cc430003e802d82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619ccd82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619ccd82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619cc410001f603" + require.Equal(t, gfc, hex.EncodeToString(bhsb)) } func BenchmarkBlockHeaderMarshal(b *testing.B) { diff --git a/chain/vm/gas.go b/chain/vm/gas.go index c030621f6..66d74fb70 100644 --- a/chain/vm/gas.go +++ b/chain/vm/gas.go @@ -59,7 +59,7 @@ type Pricelist interface { OnVerifySignature(sigType crypto.SigType, planTextSize int) (GasCharge, error) OnHashing(dataSize int) GasCharge - OnComputeUnsealedSectorCid(proofType abi.RegisteredProof, pieces []abi.PieceInfo) GasCharge + OnComputeUnsealedSectorCid(proofType abi.RegisteredSealProof, pieces []abi.PieceInfo) GasCharge OnVerifySeal(info abi.SealVerifyInfo) GasCharge OnVerifyPost(info abi.WindowPoStVerifyInfo) GasCharge OnVerifyConsensusFault() GasCharge @@ -136,7 +136,7 @@ func (ps pricedSyscalls) HashBlake2b(data []byte) [32]byte { } // Computes an unsealed sector CID (CommD) from its constituent piece CIDs (CommPs) and sizes. -func (ps pricedSyscalls) ComputeUnsealedSectorCID(reg abi.RegisteredProof, pieces []abi.PieceInfo) (cid.Cid, error) { +func (ps pricedSyscalls) ComputeUnsealedSectorCID(reg abi.RegisteredSealProof, pieces []abi.PieceInfo) (cid.Cid, error) { ps.chargeGas(ps.pl.OnComputeUnsealedSectorCid(reg, pieces)) return ps.under.ComputeUnsealedSectorCID(reg, pieces) } diff --git a/chain/vm/gas_v0.go b/chain/vm/gas_v0.go index c631beaee..b1b2c5c9d 100644 --- a/chain/vm/gas_v0.go +++ b/chain/vm/gas_v0.go @@ -142,7 +142,7 @@ func (pl *pricelistV0) OnHashing(dataSize int) GasCharge { } // OnComputeUnsealedSectorCid -func (pl *pricelistV0) OnComputeUnsealedSectorCid(proofType abi.RegisteredProof, pieces []abi.PieceInfo) GasCharge { +func (pl *pricelistV0) OnComputeUnsealedSectorCid(proofType abi.RegisteredSealProof, pieces []abi.PieceInfo) GasCharge { // TODO: this needs more cost tunning, check with @lotus return newGasCharge("OnComputeUnsealedSectorCid", pl.computeUnsealedSectorCidBase, 0) } diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index 2349bc324..a5910121d 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -43,7 +43,7 @@ type syscallShim struct { verifier ffiwrapper.Verifier } -func (ss *syscallShim) ComputeUnsealedSectorCID(st abi.RegisteredProof, pieces []abi.PieceInfo) (cid.Cid, error) { +func (ss *syscallShim) ComputeUnsealedSectorCID(st abi.RegisteredSealProof, pieces []abi.PieceInfo) (cid.Cid, error) { var sum abi.PaddedPieceSize for _, p := range pieces { sum += p.Size diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 7d7991636..57f5241df 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -74,7 +74,7 @@ func main() { log.Info("Starting lotus-bench") - miner.SupportedProofTypes[abi.RegisteredProof_StackedDRG2KiBSeal] = struct{}{} + miner.SupportedProofTypes[abi.RegisteredSealProof_StackedDrg2KiBV1] = struct{}{} app := &cli.App{ Name: "lotus-bench", @@ -270,9 +270,9 @@ var sealBenchCmd = &cli.Command{ for _, s := range genm.Sectors { sealedSectors = append(sealedSectors, abi.SectorInfo{ - SealedCID: s.CommR, - SectorNumber: s.SectorID, - RegisteredProof: s.ProofType, + SealedCID: s.CommR, + SectorNumber: s.SectorID, + SealProof: s.ProofType, }) } } @@ -284,7 +284,12 @@ var sealBenchCmd = &cli.Command{ if !c.Bool("skip-commit2") { log.Info("generating winning post candidates") - fcandidates, err := ffiwrapper.ProofVerifier.GenerateWinningPoStSectorChallenge(context.TODO(), spt, mid, challenge[:], uint64(len(sealedSectors))) + wpt, err := spt.RegisteredWindowPoStProof() + if err != nil { + return err + } + + fcandidates, err := ffiwrapper.ProofVerifier.GenerateWinningPoStSectorChallenge(context.TODO(), wpt, mid, challenge[:], uint64(len(sealedSectors))) if err != nil { return err } @@ -485,9 +490,9 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, mid precommit2 := time.Now() sealedSectors = append(sealedSectors, abi.SectorInfo{ - RegisteredProof: sb.SealProofType(), - SectorNumber: i, - SealedCID: cids.Sealed, + SealProof: sb.SealProofType(), + SectorNumber: i, + SealedCID: cids.Sealed, }) seed := lapi.SealSeed{ @@ -536,7 +541,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, mid svi := abi.SealVerifyInfo{ SectorID: abi.SectorID{Miner: mid, Number: i}, SealedCID: cids.Sealed, - RegisteredProof: sb.SealProofType(), + SealProof: sb.SealProofType(), Proof: proof, DealIDs: nil, Randomness: ticket, diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index 06cb011b4..ffb651c00 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -33,12 +33,7 @@ import ( var log = logging.Logger("preseal") -func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNumber, sectors int, sbroot string, preimage []byte, key *types.KeyInfo) (*genesis.Miner, *types.KeyInfo, error) { - spt, err := pt.RegisteredSealProof() - if err != nil { - return nil, nil, err - } - +func PreSeal(maddr address.Address, spt abi.RegisteredSealProof, offset abi.SectorNumber, sectors int, sbroot string, preimage []byte, key *types.KeyInfo) (*genesis.Miner, *types.KeyInfo, error) { mid, err := address.IDFromAddress(maddr) if err != nil { return nil, nil, err @@ -63,7 +58,7 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum return nil, nil, err } - ssize, err := pt.SectorSize() + ssize, err := spt.SectorSize() if err != nil { return nil, nil, err } @@ -102,7 +97,7 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum CommR: cids.Sealed, CommD: cids.Unsealed, SectorID: sid.Number, - ProofType: pt, + ProofType: spt, }) } diff --git a/cmd/lotus-shed/proofs.go b/cmd/lotus-shed/proofs.go index 3a4615658..f18dc93fb 100644 --- a/cmd/lotus-shed/proofs.go +++ b/cmd/lotus-shed/proofs.go @@ -88,7 +88,7 @@ var verifySealProofCmd = &cli.Command{ Number: snum, }, SealedCID: commr, - RegisteredProof: abi.RegisteredProof(cctx.Int64("proof-type")), + SealProof: abi.RegisteredSealProof(cctx.Int64("proof-type")), Proof: proof, DealIDs: nil, Randomness: abi.SealRandomness(ticket), diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 1bff7f456..5bb4a309b 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 1bff7f4563370ada590a605b5459b91e1662ebaa +Subproject commit 5bb4a309bce9d446ac618f34a8b9e2883af2002f diff --git a/genesis/types.go b/genesis/types.go index 4b95d3311..41d8a413c 100644 --- a/genesis/types.go +++ b/genesis/types.go @@ -22,7 +22,7 @@ type PreSeal struct { CommD cid.Cid SectorID abi.SectorNumber Deal market.DealProposal - ProofType abi.RegisteredProof + ProofType abi.RegisteredSealProof } type Miner struct { diff --git a/go.mod b/go.mod index 5cfe55a8d..df0c03c9f 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/drand/drand v0.9.2-0.20200529123141-6b138aefaef2 github.com/drand/kyber v1.0.2 github.com/fatih/color v1.8.0 - github.com/filecoin-project/chain-validation v0.0.6-0.20200605221044-7f78284bbc94 + github.com/filecoin-project/chain-validation v0.0.6-0.20200615142407-39c0fe23a3d5 github.com/filecoin-project/filecoin-ffi v0.26.1-0.20200508175440-05b30afeb00d github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2 @@ -23,16 +23,16 @@ require ( github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.3.0 github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 - github.com/filecoin-project/go-fil-markets v0.2.7 + github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615162707-5dd6b2eb253b github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/sector-storage v0.0.0-20200615123301-7d09fa88b4b0 + github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6 github.com/filecoin-project/specs-actors v0.6.0 github.com/filecoin-project/specs-storage v0.1.0 - github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa + github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect @@ -127,3 +127,5 @@ require ( replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v1.18.0 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi + +replace github.com/filecoin-project/go-fil-markets => /home/magik6k/gohack/github.com/filecoin-project/go-fil-markets diff --git a/go.sum b/go.sum index 5366cf6b6..c4a07cf66 100644 --- a/go.sum +++ b/go.sum @@ -184,6 +184,8 @@ github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGj github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/filecoin-project/chain-validation v0.0.6-0.20200605221044-7f78284bbc94 h1:svEal/usZ/REEjEz6ije3/ALn8duM2CckR0LjT1btx0= github.com/filecoin-project/chain-validation v0.0.6-0.20200605221044-7f78284bbc94/go.mod h1:aOmmdhO0xIRtWCwx3MyIv3tPCCorM1Bq7pNOJGMLndA= +github.com/filecoin-project/chain-validation v0.0.6-0.20200615142407-39c0fe23a3d5 h1:8Ytl1uk8/Sff7Z2vfb7MOEM3ZRlQKJPqn19FzDcr+bc= +github.com/filecoin-project/chain-validation v0.0.6-0.20200615142407-39c0fe23a3d5/go.mod h1:HEJn6kOXMNhCNBYNTO/lrEI7wSgqCOR6hN5ecfYUnC8= github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef h1:Wi5E+P1QfHP8IF27eUiTx5vYfqQZwfPxzq3oFEq8w8U= @@ -225,6 +227,10 @@ github.com/filecoin-project/sector-storage v0.0.0-20200609231555-252c2b0c969d h1 github.com/filecoin-project/sector-storage v0.0.0-20200609231555-252c2b0c969d/go.mod h1:SfaVNw/A6LwqqEt6wEMVBN5Grn3XC50j+yjmegHIe7Y= github.com/filecoin-project/sector-storage v0.0.0-20200615123301-7d09fa88b4b0 h1:DpdlpS3G2XN+F68rxOTAXsGL3cV7UkFr0TxdmKSiaJg= github.com/filecoin-project/sector-storage v0.0.0-20200615123301-7d09fa88b4b0/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= +github.com/filecoin-project/sector-storage v0.0.0-20200615142857-3d1225bf3449 h1:aFXA4iI8WumN1pgEv5gtAyZs3mEX7AdV2xqHf76Dn6o= +github.com/filecoin-project/sector-storage v0.0.0-20200615142857-3d1225bf3449/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= +github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6 h1:NIcubpeasVs++K5EFelMXeURRb8sWCuXQNOSWnvTc14= +github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= @@ -241,6 +247,10 @@ github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sg github.com/filecoin-project/specs-storage v0.1.0/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa h1:kTCIBKMhhhVcjCh1ws72vpdDr4cdZjCyIUDFijuWuvA= github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa/go.mod h1:S0u14Wr55mpe22lElCSKbXrhtWg/jquVTTMhefQ8f4Q= +github.com/filecoin-project/storage-fsm v0.0.0-20200615142724-95dbdc803c53 h1:mvvBaFPyYugJfmHgFHg0BgE0plwV2IhfgL6tLu02umM= +github.com/filecoin-project/storage-fsm v0.0.0-20200615142724-95dbdc803c53/go.mod h1:ZBID+J03DzANc0cv+sv03ROqfYYZbl62d3u8JftBH6k= +github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743 h1:a8f1p6UdeD+ZINBKJN4FhEos8uaKeASOAabq5RCpQdg= +github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743/go.mod h1:q1YCutTSMq/yGYvDPHReT37bPfDLHltnwJutzR9kOY0= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= diff --git a/lotuspond/spawn.go b/lotuspond/spawn.go index 32fc69b05..8100d784a 100644 --- a/lotuspond/spawn.go +++ b/lotuspond/spawn.go @@ -25,8 +25,8 @@ import ( ) func init() { - miner.SupportedProofTypes = map[abi.RegisteredProof]struct{}{ - abi.RegisteredProof_StackedDRG2KiBSeal: {}, + miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } } @@ -49,7 +49,7 @@ func (api *api) Spawn() (nodeInfo, error) { } sbroot := filepath.Join(dir, "preseal") - genm, ki, err := seed.PreSeal(genMiner, abi.RegisteredProof_StackedDRG2KiBSeal, 0, 2, sbroot, []byte("8"), nil) + genm, ki, err := seed.PreSeal(genMiner, abi.RegisteredSealProof_StackedDrg2KiBV1, 0, 2, sbroot, []byte("8"), nil) if err != nil { return nodeInfo{}, xerrors.Errorf("preseal failed: %w", err) } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 1d68a1858..e39527201 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -750,7 +750,7 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr initialPledge := big.Zero() { - ssize, err := precommit.Info.RegisteredProof.SectorSize() + ssize, err := precommit.Info.SealProof.SectorSize() if err != nil { return types.EmptyInt, err } diff --git a/node/node_test.go b/node/node_test.go index 8b17fac2e..a3a37bdd3 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -54,8 +54,8 @@ func init() { _ = logging.SetLogLevel("*", "INFO") power.ConsensusMinerMinPower = big.NewInt(2048) - saminer.SupportedProofTypes = map[abi.RegisteredProof]struct{}{ - abi.RegisteredProof_StackedDRG2KiBSeal: {}, + saminer.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } verifreg.MinVerifiedDealSize = big.NewInt(256) } @@ -189,7 +189,7 @@ func builder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestN if err != nil { t.Fatal(err) } - genm, k, err := seed.PreSeal(maddr, abi.RegisteredProof_StackedDRG2KiBPoSt, 0, test.GenesisPreseals, tdir, []byte("make genesis mem random"), nil) + genm, k, err := seed.PreSeal(maddr, abi.RegisteredSealProof_StackedDrg2KiBV1, 0, test.GenesisPreseals, tdir, []byte("make genesis mem random"), nil) if err != nil { t.Fatal(err) } diff --git a/storage/adapter_storage_miner.go b/storage/adapter_storage_miner.go index 7821b009c..5fc811997 100644 --- a/storage/adapter_storage_miner.go +++ b/storage/adapter_storage_miner.go @@ -98,7 +98,7 @@ func (s SealingAPIAdapter) StateWaitMsg(ctx context.Context, mcid cid.Cid) (seal }, nil } -func (s SealingAPIAdapter) StateComputeDataCommitment(ctx context.Context, maddr address.Address, sectorType abi.RegisteredProof, deals []abi.DealID, tok sealing.TipSetToken) (cid.Cid, error) { +func (s SealingAPIAdapter) StateComputeDataCommitment(ctx context.Context, maddr address.Address, sectorType abi.RegisteredSealProof, deals []abi.DealID, tok sealing.TipSetToken) (cid.Cid, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return cid.Undef, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) diff --git a/storage/miner.go b/storage/miner.go index 171c166aa..514af527a 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -135,7 +135,7 @@ type StorageWpp struct { prover storage.Prover verifier ffiwrapper.Verifier miner abi.ActorID - winnRpt abi.RegisteredProof + winnRpt abi.RegisteredPoStProof } func NewWinningPoStProver(api api.FullNode, prover storage.Prover, verifier ffiwrapper.Verifier, miner dtypes.MinerID) (*StorageWpp, error) { diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 8cd9fdc5f..01d840848 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -464,9 +464,9 @@ func (s *WindowPoStScheduler) sortedSectorInfo(ctx context.Context, deadlineSect sbsi := make([]abi.SectorInfo, len(sset)) for k, sector := range sset { sbsi[k] = abi.SectorInfo{ - SectorNumber: sector.ID, - SealedCID: sector.Info.Info.SealedCID, - RegisteredProof: sector.Info.Info.RegisteredProof, + SectorNumber: sector.ID, + SealedCID: sector.Info.Info.SealedCID, + SealProof: sector.Info.Info.SealProof, } } diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index 8b405747e..077209a4e 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -24,7 +24,7 @@ type WindowPoStScheduler struct { api storageMinerApi prover storage.Prover faultTracker sectorstorage.FaultTracker - proofType abi.RegisteredProof + proofType abi.RegisteredPoStProof partitionSectors uint64 actor address.Address From c832c0bf3fdfe56b87932fc3ddcc9988c4112765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 15 Jun 2020 18:31:18 +0200 Subject: [PATCH 126/160] mod tidy --- go.mod | 2 -- go.sum | 26 ++------------------------ 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index df0c03c9f..3b35c53d3 100644 --- a/go.mod +++ b/go.mod @@ -127,5 +127,3 @@ require ( replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v1.18.0 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi - -replace github.com/filecoin-project/go-fil-markets => /home/magik6k/gohack/github.com/filecoin-project/go-fil-markets diff --git a/go.sum b/go.sum index c4a07cf66..4ae1fb3af 100644 --- a/go.sum +++ b/go.sum @@ -182,8 +182,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= -github.com/filecoin-project/chain-validation v0.0.6-0.20200605221044-7f78284bbc94 h1:svEal/usZ/REEjEz6ije3/ALn8duM2CckR0LjT1btx0= -github.com/filecoin-project/chain-validation v0.0.6-0.20200605221044-7f78284bbc94/go.mod h1:aOmmdhO0xIRtWCwx3MyIv3tPCCorM1Bq7pNOJGMLndA= github.com/filecoin-project/chain-validation v0.0.6-0.20200615142407-39c0fe23a3d5 h1:8Ytl1uk8/Sff7Z2vfb7MOEM3ZRlQKJPqn19FzDcr+bc= github.com/filecoin-project/chain-validation v0.0.6-0.20200615142407-39c0fe23a3d5/go.mod h1:HEJn6kOXMNhCNBYNTO/lrEI7wSgqCOR6hN5ecfYUnC8= github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= @@ -193,7 +191,6 @@ github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef/go.m github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg= github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2 h1:jamfsxfK0Q9yCMHt8MPWx7Aa/O9k2Lve8eSc6FILYGQ= github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg= -github.com/filecoin-project/go-bitfield v0.0.0-20200309034705-8c7ac40bd550/go.mod h1:iodsLxOFZnqKtjj2zkgqzoGNrv6vUqj69AT/J8DKXEw= github.com/filecoin-project/go-bitfield v0.0.0-20200416002808-b3ee67ec9060/go.mod h1:iodsLxOFZnqKtjj2zkgqzoGNrv6vUqj69AT/J8DKXEw= github.com/filecoin-project/go-bitfield v0.0.1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e h1:gkG/7G+iKy4He+IiQNeQn+nndFznb/vCoOR8iRQsm60= @@ -206,8 +203,8 @@ github.com/filecoin-project/go-data-transfer v0.3.0 h1:BwBrrXu9Unh9JjjX4GAc5FfzU github.com/filecoin-project/go-data-transfer v0.3.0/go.mod h1:cONglGP4s/d+IUQw5mWZrQK+FQATQxr3AXzi4dRh0l4= github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1:yvQJCW9mmi9zy+51xA01Ea2X7/dL7r8eKDPuGUjRmbo= github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA= -github.com/filecoin-project/go-fil-markets v0.2.7 h1:bgdK/e+xW15aVZLtdFLzAHdrx1hqtGF9veg2lstLK6o= -github.com/filecoin-project/go-fil-markets v0.2.7/go.mod h1:LI3VFHse33aU0djAmFQ8+Hg39i0J8ibAoppGu6TbgkA= +github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615162707-5dd6b2eb253b h1:xhaFY+rdZMDErIkUHvkJWdN6H3h/dn/5U3GOX4HP/Cs= +github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615162707-5dd6b2eb253b/go.mod h1:nCjNrwgRsjX1TSiBUVgewa5PbP19x5Mq07uzlJD9YUw= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM= github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs= @@ -222,33 +219,14 @@ github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIi github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= -github.com/filecoin-project/sector-storage v0.0.0-20200508203401-a74812ba12f3/go.mod h1:B+xzopr/oWZJz2hBL5Ekb7Obcum5ntmfbaAUlaaho28= -github.com/filecoin-project/sector-storage v0.0.0-20200609231555-252c2b0c969d h1:q7KC8/yIirwVX7JCLgjEmo5LX77V8tYV0EzVCnCwrE8= -github.com/filecoin-project/sector-storage v0.0.0-20200609231555-252c2b0c969d/go.mod h1:SfaVNw/A6LwqqEt6wEMVBN5Grn3XC50j+yjmegHIe7Y= -github.com/filecoin-project/sector-storage v0.0.0-20200615123301-7d09fa88b4b0 h1:DpdlpS3G2XN+F68rxOTAXsGL3cV7UkFr0TxdmKSiaJg= -github.com/filecoin-project/sector-storage v0.0.0-20200615123301-7d09fa88b4b0/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= -github.com/filecoin-project/sector-storage v0.0.0-20200615142857-3d1225bf3449 h1:aFXA4iI8WumN1pgEv5gtAyZs3mEX7AdV2xqHf76Dn6o= -github.com/filecoin-project/sector-storage v0.0.0-20200615142857-3d1225bf3449/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6 h1:NIcubpeasVs++K5EFelMXeURRb8sWCuXQNOSWnvTc14= github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= -github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= -github.com/filecoin-project/specs-actors v0.5.4-0.20200521014528-0df536f7e461/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= -github.com/filecoin-project/specs-actors v0.5.5 h1:bDsowem6dLRc9B7g3sgFYvHgfWTH4D9q5ehWLGLdKCw= -github.com/filecoin-project/specs-actors v0.5.5/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= -github.com/filecoin-project/specs-actors v0.5.6 h1:WlhtoXwFoKlP1b06NI4NJaxC4m9EXNV+qFVl43/xRN8= -github.com/filecoin-project/specs-actors v0.5.6/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-actors v0.6.0 h1:IepUsmDGY60QliENVTkBTAkwqGWw9kNbbHOcU/9oiC0= github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= -github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 h1:T3f/zkuvgtgqcXrb0NO3BicuveGOxxUAMPa/Yif2kuE= -github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102/go.mod h1:xJ1/xl9+8zZeSSSFmDC3Wr6uusCTxyYPI0VeNVSFmPE= github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sgOVdJbvFjOnD5w94= github.com/filecoin-project/specs-storage v0.1.0/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= -github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa h1:kTCIBKMhhhVcjCh1ws72vpdDr4cdZjCyIUDFijuWuvA= -github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa/go.mod h1:S0u14Wr55mpe22lElCSKbXrhtWg/jquVTTMhefQ8f4Q= -github.com/filecoin-project/storage-fsm v0.0.0-20200615142724-95dbdc803c53 h1:mvvBaFPyYugJfmHgFHg0BgE0plwV2IhfgL6tLu02umM= -github.com/filecoin-project/storage-fsm v0.0.0-20200615142724-95dbdc803c53/go.mod h1:ZBID+J03DzANc0cv+sv03ROqfYYZbl62d3u8JftBH6k= github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743 h1:a8f1p6UdeD+ZINBKJN4FhEos8uaKeASOAabq5RCpQdg= github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743/go.mod h1:q1YCutTSMq/yGYvDPHReT37bPfDLHltnwJutzR9kOY0= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= From 58a79773ee8467fe3ac323543146554e5c69682a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 15 Jun 2020 18:34:36 +0200 Subject: [PATCH 127/160] fix debug build --- build/params_2k.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/params_2k.go b/build/params_2k.go index 56e1ca2f8..046753678 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -12,8 +12,8 @@ import ( func init() { power.ConsensusMinerMinPower = big.NewInt(2048) - miner.SupportedProofTypes = map[abi.RegisteredProof]struct{}{ - abi.RegisteredProof_StackedDRG2KiBSeal: {}, + miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } verifreg.MinVerifiedDealSize = big.NewInt(256) From 8039701f9ef8bbab42ceb9f4894c666cd320f123 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 15 Jun 2020 19:50:43 +0200 Subject: [PATCH 128/160] Use extra field in gastrace Signed-off-by: Jakub Sztandera --- chain/types/execresult.go | 2 +- chain/vm/gas.go | 10 +++++++++- chain/vm/gas_v0.go | 6 +++--- chain/vm/runtime.go | 14 +++++++++----- cli/state.go | 2 +- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/chain/types/execresult.go b/chain/types/execresult.go index b0e1f760d..443147f9e 100644 --- a/chain/types/execresult.go +++ b/chain/types/execresult.go @@ -30,7 +30,7 @@ type GasTrace struct { VirtualStorageGas int64 TimeTaken time.Duration - Extra interface{} + Extra interface{} `json:",omitempty"` Callers []uintptr `json:"-"` } diff --git a/chain/vm/gas.go b/chain/vm/gas.go index dc9966e57..2540afbf0 100644 --- a/chain/vm/gas.go +++ b/chain/vm/gas.go @@ -18,7 +18,9 @@ const ( ) type GasCharge struct { - Name string + Name string + Extra interface{} + ComputeGas int64 StorageGas int64 @@ -36,6 +38,12 @@ func (g GasCharge) WithVirtual(compute, storage int64) GasCharge { return out } +func (g GasCharge) WithExtra(extra interface{}) GasCharge { + out := g + out.Extra = extra + return out +} + func newGasCharge(name string, computeGas int64, storageGas int64) GasCharge { return GasCharge{ Name: name, diff --git a/chain/vm/gas_v0.go b/chain/vm/gas_v0.go index c631beaee..e2a908a23 100644 --- a/chain/vm/gas_v0.go +++ b/chain/vm/gas_v0.go @@ -108,12 +108,12 @@ func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.M // OnIpldGet returns the gas used for storing an object func (pl *pricelistV0) OnIpldGet(dataSize int) GasCharge { - return newGasCharge(fmt.Sprintf("OnIpldGet:%db", dataSize), pl.ipldGetBase+int64(dataSize)*pl.ipldGetPerByte, 0) + return newGasCharge("OnIpldGet", pl.ipldGetBase+int64(dataSize)*pl.ipldGetPerByte, 0).WithExtra(dataSize) } // OnIpldPut returns the gas used for storing an object func (pl *pricelistV0) OnIpldPut(dataSize int) GasCharge { - return newGasCharge(fmt.Sprintf("OnIpldPut:%db", dataSize), pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte) + return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte).WithExtra(dataSize) } // OnCreateActor returns the gas used for creating an actor @@ -133,7 +133,7 @@ func (pl *pricelistV0) OnVerifySignature(sigType crypto.SigType, planTextSize in return GasCharge{}, fmt.Errorf("cost function for signature type %d not supported", sigType) } sigName, _ := sigType.Name() - return newGasCharge("OnVerifySignature/"+sigName, costFn(int64(planTextSize)), 0), nil + return newGasCharge("OnVerifySignature", costFn(int64(planTextSize)), 0).WithExtra(sigName), nil } // OnHashing diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 9b83593ab..48563df64 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -524,14 +524,18 @@ func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError } gasTrace := types.GasTrace{ - Name: gas.Name, - TotalGas: toUse, - ComputeGas: gas.ComputeGas, - StorageGas: gas.StorageGas, + Name: gas.Name, + Extra: gas.Extra, + + TotalGas: toUse, + ComputeGas: gas.ComputeGas, + StorageGas: gas.StorageGas, + TotalVirtualGas: gas.VirtualCompute*GasComputeMulti + gas.VirtualStorage*GasStorageMulti, VirtualComputeGas: gas.VirtualCompute, VirtualStorageGas: gas.VirtualStorage, - Callers: callers[:cout], + + Callers: callers[:cout], } rt.executionTrace.GasCharges = append(rt.executionTrace.GasCharges, &gasTrace) rt.lastGasChargeTime = now diff --git a/cli/state.go b/cli/state.go index 753b0b017..1aad85299 100644 --- a/cli/state.go +++ b/cli/state.go @@ -1072,7 +1072,7 @@ var compStateMsg = ` {{- end}} {{range .GasCharges}} - {{.Name}} + {{.Name}}{{if .Extra}}:{{.Extra}}{{end}} {{template "gasC" .}} {{.TimeTaken}} From daca8651f41efacacd804b77db6c8b7959751f0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 15 Jun 2020 19:53:21 +0200 Subject: [PATCH 129/160] Fix bench --- cmd/lotus-bench/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 57f5241df..a10c3c00e 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -284,12 +284,12 @@ var sealBenchCmd = &cli.Command{ if !c.Bool("skip-commit2") { log.Info("generating winning post candidates") - wpt, err := spt.RegisteredWindowPoStProof() + wipt, err := spt.RegisteredWinningPoStProof() if err != nil { return err } - fcandidates, err := ffiwrapper.ProofVerifier.GenerateWinningPoStSectorChallenge(context.TODO(), wpt, mid, challenge[:], uint64(len(sealedSectors))) + fcandidates, err := ffiwrapper.ProofVerifier.GenerateWinningPoStSectorChallenge(context.TODO(), wipt, mid, challenge[:], uint64(len(sealedSectors))) if err != nil { return err } From f8669d4e9547d0f322cc735baf22529e58360e5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 15 Jun 2020 21:21:40 +0200 Subject: [PATCH 130/160] Update chain-val and sector-storage --- go.mod | 4 ++-- go.sum | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 3b35c53d3..89e2e2842 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/drand/drand v0.9.2-0.20200529123141-6b138aefaef2 github.com/drand/kyber v1.0.2 github.com/fatih/color v1.8.0 - github.com/filecoin-project/chain-validation v0.0.6-0.20200615142407-39c0fe23a3d5 + github.com/filecoin-project/chain-validation v0.0.6-0.20200615191232-6be1a8c6ed09 github.com/filecoin-project/filecoin-ffi v0.26.1-0.20200508175440-05b30afeb00d github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2 @@ -29,7 +29,7 @@ require ( github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6 + github.com/filecoin-project/sector-storage v0.0.0-20200615192001-42c9e08595b7 github.com/filecoin-project/specs-actors v0.6.0 github.com/filecoin-project/specs-storage v0.1.0 github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743 diff --git a/go.sum b/go.sum index 4ae1fb3af..83af4904c 100644 --- a/go.sum +++ b/go.sum @@ -182,8 +182,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= -github.com/filecoin-project/chain-validation v0.0.6-0.20200615142407-39c0fe23a3d5 h1:8Ytl1uk8/Sff7Z2vfb7MOEM3ZRlQKJPqn19FzDcr+bc= -github.com/filecoin-project/chain-validation v0.0.6-0.20200615142407-39c0fe23a3d5/go.mod h1:HEJn6kOXMNhCNBYNTO/lrEI7wSgqCOR6hN5ecfYUnC8= +github.com/filecoin-project/chain-validation v0.0.6-0.20200615191232-6be1a8c6ed09 h1:GuiNSEZ9nc05LUpKhABw/SO6t9wqCfsJX1D0ByWQjkc= +github.com/filecoin-project/chain-validation v0.0.6-0.20200615191232-6be1a8c6ed09/go.mod h1:HEJn6kOXMNhCNBYNTO/lrEI7wSgqCOR6hN5ecfYUnC8= github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef h1:Wi5E+P1QfHP8IF27eUiTx5vYfqQZwfPxzq3oFEq8w8U= @@ -219,8 +219,9 @@ github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIi github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= -github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6 h1:NIcubpeasVs++K5EFelMXeURRb8sWCuXQNOSWnvTc14= github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= +github.com/filecoin-project/sector-storage v0.0.0-20200615192001-42c9e08595b7 h1:cjsOpQKvZosPx9/qqq2bucHVdRyXzvBR1f37atiR3/0= +github.com/filecoin-project/sector-storage v0.0.0-20200615192001-42c9e08595b7/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= github.com/filecoin-project/specs-actors v0.6.0 h1:IepUsmDGY60QliENVTkBTAkwqGWw9kNbbHOcU/9oiC0= From 01c4726fd5fca11aed50f25a8d6287e72487885d Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 15 Jun 2020 13:02:57 -0700 Subject: [PATCH 131/160] add some smartness to bench analyze gas output --- cmd/lotus-bench/import.go | 66 +++++++++++++++++++++++++++++++++ cmd/lotus-storage-miner/init.go | 2 +- 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 11d58d9b8..8d62ac0d8 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "math" "os" "runtime/pprof" "sort" @@ -162,6 +163,43 @@ type Invocation struct { Invoc *api.InvocResult } +const GasPerNs = 10 + +func countGasCosts(et *types.ExecutionTrace) (int64, int64) { + var cgas, vgas int64 + + for _, gc := range et.GasCharges { + cgas += gc.ComputeGas + vgas += gc.VirtualComputeGas + } + + for _, sub := range et.Subcalls { + c, v := countGasCosts(&sub) + cgas += c + vgas += v + } + + return cgas, vgas +} + +func compStats(vals []float64) (float64, float64) { + var sum float64 + + for _, v := range vals { + sum += v + } + + av := sum / float64(len(vals)) + + var varsum float64 + for _, v := range vals { + delta := av - v + varsum += delta * delta + } + + return av, math.Sqrt(varsum / float64(len(vals))) +} + var importAnalyzeCmd = &cli.Command{ Name: "analyze", Action: func(cctx *cli.Context) error { @@ -180,6 +218,8 @@ var importAnalyzeCmd = &cli.Command{ return err } + chargeDeltas := make(map[string][]float64) + var invocs []Invocation var totalTime time.Duration for i, r := range results { @@ -191,9 +231,35 @@ var importAnalyzeCmd = &cli.Command{ TipSet: r.TipSet, Invoc: inv, }) + + cgas, vgas := countGasCosts(&inv.ExecutionTrace) + fmt.Printf("Invocation: %d %s: %s %d -> %0.2f\n", inv.Msg.Method, inv.Msg.To, inv.Duration, cgas+vgas, float64(GasPerNs*inv.Duration.Nanoseconds())/float64(cgas+vgas)) + + for _, gc := range inv.ExecutionTrace.GasCharges { + + compGas := gc.ComputeGas + gc.VirtualComputeGas + ratio := float64(compGas) / float64(gc.TimeTaken.Nanoseconds()) + + chargeDeltas[gc.Name] = append(chargeDeltas[gc.Name], 1/(ratio/GasPerNs)) + //fmt.Printf("%s: %d, %s: %0.2f\n", gc.Name, compGas, gc.TimeTaken, 1/(ratio/GasPerNs)) + } } } + var keys []string + for k := range chargeDeltas { + keys = append(keys, k) + } + + fmt.Println("Gas Price Deltas") + sort.Strings(keys) + for _, k := range keys { + vals := chargeDeltas[k] + av, stdev := compStats(vals) + + fmt.Printf("%s: incr by %f (%f)\n", k, av, stdev) + } + sort.Slice(invocs, func(i, j int) bool { return invocs[i].Invoc.Duration > invocs[j].Invoc.Duration }) diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 29e82cc2f..14972c69a 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -460,7 +460,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, } if cerr != nil { - return xerrors.Errorf("failed to configure storage miner: %w", err) + return xerrors.Errorf("failed to configure storage miner: %w", cerr) } } From 9262661135706070325d8b9bfea9c5e72841c719 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Mon, 15 Jun 2020 15:43:47 -0700 Subject: [PATCH 132/160] chore(deps): update to fil-markets 0.3.0 Updates to latest fil markets with resumability and other fixes --- go.mod | 2 +- go.sum | 6 ++++-- node/builder.go | 2 -- node/impl/storminer.go | 2 +- node/modules/client.go | 3 +-- node/modules/services.go | 18 ------------------ node/modules/storageminer.go | 4 ++-- 7 files changed, 9 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index 89e2e2842..ca503ce49 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.3.0 github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 - github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615162707-5dd6b2eb253b + github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615223906-0fb36d5347bd github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca diff --git a/go.sum b/go.sum index 83af4904c..c6c69454b 100644 --- a/go.sum +++ b/go.sum @@ -203,8 +203,8 @@ github.com/filecoin-project/go-data-transfer v0.3.0 h1:BwBrrXu9Unh9JjjX4GAc5FfzU github.com/filecoin-project/go-data-transfer v0.3.0/go.mod h1:cONglGP4s/d+IUQw5mWZrQK+FQATQxr3AXzi4dRh0l4= github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1:yvQJCW9mmi9zy+51xA01Ea2X7/dL7r8eKDPuGUjRmbo= github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA= -github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615162707-5dd6b2eb253b h1:xhaFY+rdZMDErIkUHvkJWdN6H3h/dn/5U3GOX4HP/Cs= -github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615162707-5dd6b2eb253b/go.mod h1:nCjNrwgRsjX1TSiBUVgewa5PbP19x5Mq07uzlJD9YUw= +github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615223906-0fb36d5347bd h1:yQqt+9CDKdZhg1VZEgV8O5tA+D1f0mairODT7dcwHtU= +github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615223906-0fb36d5347bd/go.mod h1:UXsXi43AyUQ5ieb4yIaLgk4PVt7TAbl1UCccuNw+7ds= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM= github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs= @@ -215,6 +215,8 @@ github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca h github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 h1:k9qVR9ItcziSB2rxtlkN/MDWNlbsI6yzec+zjUatLW0= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= +github.com/filecoin-project/go-statemachine v0.0.0-20200612181802-4eb3d0c68eba h1:GEWb/6KQyNZt4jm8fgVcIFPH0ElAGXfHM59ZSiqPTvY= +github.com/filecoin-project/go-statemachine v0.0.0-20200612181802-4eb3d0c68eba/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= diff --git a/node/builder.go b/node/builder.go index 25b027145..536d81901 100644 --- a/node/builder.go +++ b/node/builder.go @@ -106,7 +106,6 @@ const ( HandleIncomingBlocksKey HandleIncomingMessagesKey - RunDealClientKey RegisterClientValidatorKey // storage miner @@ -266,7 +265,6 @@ func Online() Option { Override(new(storagemarket.StorageClient), modules.StorageClient), Override(new(storagemarket.StorageClientNode), storageadapter.NewClientNodeAdapter), Override(RegisterClientValidatorKey, modules.RegisterClientValidator), - Override(RunDealClientKey, modules.RunDealClient), Override(new(beacon.RandomBeacon), modules.RandomBeacon), Override(new(*paychmgr.Store), paychmgr.NewStore), diff --git a/node/impl/storminer.go b/node/impl/storminer.go index de80eb4cd..a01cad9c2 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -202,7 +202,7 @@ func (sm *StorageMinerAPI) MarketListIncompleteDeals(ctx context.Context) ([]sto } func (sm *StorageMinerAPI) MarketSetPrice(ctx context.Context, p types.BigInt) error { - return sm.StorageProvider.AddAsk(abi.TokenAmount(p), 60*60*24*100) // lasts for 100 days? + return sm.StorageProvider.SetAsk(abi.TokenAmount(p), 60*60*24*100) // lasts for 100 days? } func (sm *StorageMinerAPI) DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) { diff --git a/node/modules/client.go b/node/modules/client.go index 12ec209b5..b4618015a 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -117,8 +117,7 @@ func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, r } lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { - c.Run(ctx) - return nil + return c.Start(ctx) }, OnStop: func(context.Context) error { c.Stop() diff --git a/node/modules/services.go b/node/modules/services.go index f4a007c2b..27244ad3a 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -1,8 +1,6 @@ package modules import ( - "context" - "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" eventbus "github.com/libp2p/go-eventbus" @@ -15,7 +13,6 @@ import ( "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" - "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/beacon" @@ -100,21 +97,6 @@ func HandleIncomingMessages(mctx helpers.MetricsCtx, lc fx.Lifecycle, ps *pubsub go sub.HandleIncomingMessages(ctx, mpool, msgsub) } -func RunDealClient(mctx helpers.MetricsCtx, lc fx.Lifecycle, c storagemarket.StorageClient) { - ctx := helpers.LifecycleCtx(mctx, lc) - - lc.Append(fx.Hook{ - OnStart: func(context.Context) error { - c.Run(ctx) - return nil - }, - OnStop: func(context.Context) error { - c.Stop() - return nil - }, - }) -} - func NewLocalDiscovery(ds dtypes.MetadataDS) *discovery.Local { return discovery.NewLocal(namespace.Wrap(ds, datastore.NewKey("/deals/local"))) } diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 440aa8593..fce9f9e1f 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -299,8 +299,8 @@ func NewStorageAsk(ctx helpers.MetricsCtx, fapi lapi.FullNode, ds dtypes.Metadat return nil, err } // Hacky way to set max piece size to the sector size - a := storedAsk.GetAsk(address.Address(minerAddress)).Ask - err = storedAsk.AddAsk(a.Price, a.Expiry-a.Timestamp, storagemarket.MaxPieceSize(abi.PaddedPieceSize(mi.SectorSize))) + a := storedAsk.GetAsk().Ask + err = storedAsk.SetAsk(a.Price, a.Expiry-a.Timestamp, storagemarket.MaxPieceSize(abi.PaddedPieceSize(mi.SectorSize))) if err != nil { return storedAsk, err } From a4226478198041b969b2f273bfa2d85266dfef3d Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Mon, 15 Jun 2020 15:53:39 -0700 Subject: [PATCH 133/160] fix(lint): fix lint error fix lint error and update to tagged package --- go.mod | 2 +- go.sum | 4 ++-- node/impl/storminer.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index ca503ce49..8457847a6 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.3.0 github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 - github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615223906-0fb36d5347bd + github.com/filecoin-project/go-fil-markets v0.3.0 github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca diff --git a/go.sum b/go.sum index c6c69454b..47a405efa 100644 --- a/go.sum +++ b/go.sum @@ -203,8 +203,8 @@ github.com/filecoin-project/go-data-transfer v0.3.0 h1:BwBrrXu9Unh9JjjX4GAc5FfzU github.com/filecoin-project/go-data-transfer v0.3.0/go.mod h1:cONglGP4s/d+IUQw5mWZrQK+FQATQxr3AXzi4dRh0l4= github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1:yvQJCW9mmi9zy+51xA01Ea2X7/dL7r8eKDPuGUjRmbo= github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA= -github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615223906-0fb36d5347bd h1:yQqt+9CDKdZhg1VZEgV8O5tA+D1f0mairODT7dcwHtU= -github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615223906-0fb36d5347bd/go.mod h1:UXsXi43AyUQ5ieb4yIaLgk4PVt7TAbl1UCccuNw+7ds= +github.com/filecoin-project/go-fil-markets v0.3.0 h1:7iCGiuTSia4f4DmOn3s96NWUwMNSOI0ZHel/XgeApAQ= +github.com/filecoin-project/go-fil-markets v0.3.0/go.mod h1:UXsXi43AyUQ5ieb4yIaLgk4PVt7TAbl1UCccuNw+7ds= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM= github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs= diff --git a/node/impl/storminer.go b/node/impl/storminer.go index a01cad9c2..0cc13177e 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -202,7 +202,7 @@ func (sm *StorageMinerAPI) MarketListIncompleteDeals(ctx context.Context) ([]sto } func (sm *StorageMinerAPI) MarketSetPrice(ctx context.Context, p types.BigInt) error { - return sm.StorageProvider.SetAsk(abi.TokenAmount(p), 60*60*24*100) // lasts for 100 days? + return sm.StorageProvider.SetAsk(p, 60*60*24*100) // lasts for 100 days? } func (sm *StorageMinerAPI) DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) { From f8c4b647828139199717ec86834c9f5603e64266 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 15 Jun 2020 16:05:29 -0700 Subject: [PATCH 134/160] improve chain import analyze output, add some rough virtual gas charges --- chain/vm/gas.go | 11 ++++++++++- chain/vm/gas_v0.go | 10 +++++----- chain/vm/syscalls.go | 4 +++- cmd/lotus-bench/import.go | 30 +++++++++++++++++++++++------- 4 files changed, 41 insertions(+), 14 deletions(-) diff --git a/chain/vm/gas.go b/chain/vm/gas.go index f1dcc933d..e6bde25bf 100644 --- a/chain/vm/gas.go +++ b/chain/vm/gas.go @@ -186,6 +186,15 @@ func (ps pricedSyscalls) VerifyConsensusFault(h1 []byte, h2 []byte, extra []byte } func (ps pricedSyscalls) BatchVerifySeals(inp map[address.Address][]abi.SealVerifyInfo) (map[address.Address][]bool, error) { - ps.chargeGas(newGasCharge("BatchVerifySeals", 0, 0)) // TODO: this is only called by the cron actor. Should we even charge gas? + var gasChargeSum GasCharge + gasChargeSum.Name = "BatchVerifySeals" + ps.chargeGas(gasChargeSum) // TODO: this is only called by the cron actor. Should we even charge gas? + + for _, svis := range inp { + for _, svi := range svis { + ch := ps.pl.OnVerifySeal(svi) + ps.chargeGas(newGasCharge("BatchVerifySingle", 0, 0).WithVirtual(ch.VirtualCompute+ch.ComputeGas, 0)) + } + } return ps.under.BatchVerifySeals(inp) } diff --git a/chain/vm/gas_v0.go b/chain/vm/gas_v0.go index 6994fd544..072c1f140 100644 --- a/chain/vm/gas_v0.go +++ b/chain/vm/gas_v0.go @@ -103,17 +103,17 @@ func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.M if methodNum != builtin.MethodSend { ret += pl.sendInvokeMethod } - return newGasCharge("OnMethodInvocation", ret, 0) + return newGasCharge("OnMethodInvocation", ret, 0).WithVirtual(ret*15000, 0) } // OnIpldGet returns the gas used for storing an object func (pl *pricelistV0) OnIpldGet(dataSize int) GasCharge { - return newGasCharge("OnIpldGet", pl.ipldGetBase+int64(dataSize)*pl.ipldGetPerByte, 0).WithExtra(dataSize) + return newGasCharge("OnIpldGet", pl.ipldGetBase+int64(dataSize)*pl.ipldGetPerByte, 0).WithExtra(dataSize).WithVirtual(pl.ipldGetBase*13750+(pl.ipldGetPerByte*100), 0) } // OnIpldPut returns the gas used for storing an object func (pl *pricelistV0) OnIpldPut(dataSize int) GasCharge { - return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte).WithExtra(dataSize) + return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte).WithExtra(dataSize).WithVirtual(pl.ipldPutBase*8700+(pl.ipldPutPerByte*100), 0) } // OnCreateActor returns the gas used for creating an actor @@ -144,13 +144,13 @@ func (pl *pricelistV0) OnHashing(dataSize int) GasCharge { // OnComputeUnsealedSectorCid func (pl *pricelistV0) OnComputeUnsealedSectorCid(proofType abi.RegisteredSealProof, pieces []abi.PieceInfo) GasCharge { // TODO: this needs more cost tunning, check with @lotus - return newGasCharge("OnComputeUnsealedSectorCid", pl.computeUnsealedSectorCidBase, 0) + return newGasCharge("OnComputeUnsealedSectorCid", pl.computeUnsealedSectorCidBase, 0).WithVirtual(pl.computeUnsealedSectorCidBase*24500, 0) } // OnVerifySeal func (pl *pricelistV0) OnVerifySeal(info abi.SealVerifyInfo) GasCharge { // TODO: this needs more cost tunning, check with @lotus - return newGasCharge("OnVerifySeal", pl.verifySealBase, 0) + return newGasCharge("OnVerifySeal", pl.verifySealBase, 0).WithVirtual(pl.verifySealBase*177500, 0) } // OnVerifyPost diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index a5910121d..a6a589761 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -241,10 +241,12 @@ func (ss *syscallShim) VerifySignature(sig crypto.Signature, addr address.Addres return sigs.Verify(&sig, kaddr, input) } +var BatchSealVerifyParallelism = goruntime.NumCPU() + func (ss *syscallShim) BatchVerifySeals(inp map[address.Address][]abi.SealVerifyInfo) (map[address.Address][]bool, error) { out := make(map[address.Address][]bool) - sema := make(chan struct{}, goruntime.NumCPU()) + sema := make(chan struct{}, BatchSealVerifyParallelism) var wg sync.WaitGroup for addr, seals := range inp { diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 8d62ac0d8..f9c20ac9a 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "math" "os" + "runtime" "runtime/pprof" "sort" "time" @@ -45,8 +46,14 @@ var importBenchCmd = &cli.Command{ Name: "height", Usage: "halt validation after given height", }, + &cli.IntFlag{ + Name: "batch-seal-verify-threads", + Usage: "set the parallelism factor for batch seal verification", + Value: runtime.NumCPU(), + }, }, Action: func(cctx *cli.Context) error { + vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads") if !cctx.Args().Present() { fmt.Println("must pass car file of chain to benchmark importing") return nil @@ -200,6 +207,21 @@ func compStats(vals []float64) (float64, float64) { return av, math.Sqrt(varsum / float64(len(vals))) } +func tallyGasCharges(charges map[string][]float64, et *types.ExecutionTrace) { + for _, gc := range et.GasCharges { + + compGas := gc.ComputeGas + gc.VirtualComputeGas + ratio := float64(compGas) / float64(gc.TimeTaken.Nanoseconds()) + + charges[gc.Name] = append(charges[gc.Name], 1/(ratio/GasPerNs)) + //fmt.Printf("%s: %d, %s: %0.2f\n", gc.Name, compGas, gc.TimeTaken, 1/(ratio/GasPerNs)) + for _, sub := range et.Subcalls { + tallyGasCharges(charges, &sub) + } + } + +} + var importAnalyzeCmd = &cli.Command{ Name: "analyze", Action: func(cctx *cli.Context) error { @@ -235,14 +257,8 @@ var importAnalyzeCmd = &cli.Command{ cgas, vgas := countGasCosts(&inv.ExecutionTrace) fmt.Printf("Invocation: %d %s: %s %d -> %0.2f\n", inv.Msg.Method, inv.Msg.To, inv.Duration, cgas+vgas, float64(GasPerNs*inv.Duration.Nanoseconds())/float64(cgas+vgas)) - for _, gc := range inv.ExecutionTrace.GasCharges { + tallyGasCharges(chargeDeltas, &inv.ExecutionTrace) - compGas := gc.ComputeGas + gc.VirtualComputeGas - ratio := float64(compGas) / float64(gc.TimeTaken.Nanoseconds()) - - chargeDeltas[gc.Name] = append(chargeDeltas[gc.Name], 1/(ratio/GasPerNs)) - //fmt.Printf("%s: %d, %s: %0.2f\n", gc.Name, compGas, gc.TimeTaken, 1/(ratio/GasPerNs)) - } } } From f31fcdae08b1ed26b74d8c52de35ee896933c3d6 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Mon, 15 Jun 2020 20:34:48 +0000 Subject: [PATCH 135/160] stats: increase miner power lookup perf --- tools/stats/metrics.go | 84 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 77 insertions(+), 7 deletions(-) diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index eb653e1d7..53fa6ed63 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "encoding/json" "fmt" @@ -13,9 +14,14 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/specs-actors/actors/builtin/power" + "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" + cbg "github.com/whyrusleeping/cbor-gen" + _ "github.com/influxdata/influxdb1-client" models "github.com/influxdata/influxdb1-client/models" client "github.com/influxdata/influxdb1-client/v2" @@ -135,6 +141,36 @@ func RecordTipsetPoints(ctx context.Context, api api.FullNode, pl *PointList, ti return nil } +type apiIpldStore struct { + ctx context.Context + api api.FullNode +} + +func (ht *apiIpldStore) Context() context.Context { + return ht.ctx +} + +func (ht *apiIpldStore) Get(ctx context.Context, c cid.Cid, out interface{}) error { + raw, err := ht.api.ChainReadObj(ctx, c) + if err != nil { + return err + } + + cu, ok := out.(cbg.CBORUnmarshaler) + if ok { + if err := cu.UnmarshalCBOR(bytes.NewReader(raw)); err != nil { + return err + } + return nil + } + + return fmt.Errorf("Object does not implement CBORUnmarshaler") +} + +func (ht *apiIpldStore) Put(ctx context.Context, v interface{}) (cid.Cid, error) { + return cid.Undef, fmt.Errorf("Put is not implemented on apiIpldStore") +} + func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointList, tipset *types.TipSet) error { pc, err := api.StatePledgeCollateral(ctx, tipset.Key()) if err != nil { @@ -158,24 +194,58 @@ func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointLis p = NewPoint("network.balance", netBalFilFloat) pl.AddPoint(p) - power, err := api.StateMinerPower(ctx, address.Address{}, tipset.Key()) + totalPower, err := api.StateMinerPower(ctx, address.Address{}, tipset.Key()) if err != nil { return err } - p = NewPoint("chain.power", power.TotalPower.QualityAdjPower.Int64()) + p = NewPoint("chain.power", totalPower.TotalPower.QualityAdjPower.Int64()) pl.AddPoint(p) - miners, err := api.StateListMiners(ctx, tipset.Key()) - for _, miner := range miners { - power, err := api.StateMinerPower(ctx, miner, tipset.Key()) + powerActor, err := api.StateGetActor(ctx, builtin.StoragePowerActorAddr, tipset.Key()) + if err != nil { + return err + } + + powerRaw, err := api.ChainReadObj(ctx, powerActor.Head) + if err != nil { + return err + } + + var powerActorState power.State + + if err := powerActorState.UnmarshalCBOR(bytes.NewReader(powerRaw)); err != nil { + return fmt.Errorf("failed to unmarshal power actor state: %w", err) + } + + s := &apiIpldStore{ctx, api} + mp, err := adt.AsMap(s, powerActorState.Claims) + if err != nil { + return err + } + + err = mp.ForEach(nil, func(key string) error { + addr, err := address.NewFromBytes([]byte(key)) if err != nil { return err } - p = NewPoint("chain.miner_power", power.MinerPower.QualityAdjPower.Int64()) - p.AddTag("miner", miner.String()) + var claim power.Claim + keyerAddr := adt.AddrKey(addr) + mp.Get(keyerAddr, &claim) + + if claim.QualityAdjPower.Int64() == 0 { + return nil + } + + p = NewPoint("chain.miner_power", claim.QualityAdjPower.Int64()) + p.AddTag("miner", addr.String()) pl.AddPoint(p) + + return nil + }) + if err != nil { + return err } return nil From 18dd1c277be738316616671efd3e0ff8a5d33dd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 16 Jun 2020 12:28:28 +0200 Subject: [PATCH 136/160] Update ffi --- extern/filecoin-ffi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 5bb4a309b..ca281af0b 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 5bb4a309bce9d446ac618f34a8b9e2883af2002f +Subproject commit ca281af0b6c00314382a75ae869e5cb22c83655b From bac6a47c6753382b5f746c9984d8e5a8c733d2a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 16 Jun 2020 12:30:54 +0200 Subject: [PATCH 137/160] specs-actors v0.6.1 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 8457847a6..14c3a39f3 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/sector-storage v0.0.0-20200615192001-42c9e08595b7 - github.com/filecoin-project/specs-actors v0.6.0 + github.com/filecoin-project/specs-actors v0.6.1 github.com/filecoin-project/specs-storage v0.1.0 github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index 47a405efa..6c45b1daa 100644 --- a/go.sum +++ b/go.sum @@ -228,6 +228,8 @@ github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.m github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= github.com/filecoin-project/specs-actors v0.6.0 h1:IepUsmDGY60QliENVTkBTAkwqGWw9kNbbHOcU/9oiC0= github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= +github.com/filecoin-project/specs-actors v0.6.1 h1:rhHlEzqcuuQU6oKc4csuq+/kQBDZ4EXtSomoN2XApCA= +github.com/filecoin-project/specs-actors v0.6.1/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sgOVdJbvFjOnD5w94= github.com/filecoin-project/specs-storage v0.1.0/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743 h1:a8f1p6UdeD+ZINBKJN4FhEos8uaKeASOAabq5RCpQdg= From fd6c8769dd266baf24f256577fbad016b769b805 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 10 Jun 2020 20:17:03 +0200 Subject: [PATCH 138/160] Update to new drand WIP Signed-off-by: Jakub Sztandera --- chain/beacon/drand/drand.go | 26 ++-- chain/beacon/drand/drand_test.go | 4 +- go.mod | 8 +- go.sum | 203 +++++++++++++++++++++++++++++-- 4 files changed, 212 insertions(+), 29 deletions(-) diff --git a/chain/beacon/drand/drand.go b/chain/beacon/drand/drand.go index 8742350ba..21bd2501a 100644 --- a/chain/beacon/drand/drand.go +++ b/chain/beacon/drand/drand.go @@ -8,8 +8,9 @@ import ( dchain "github.com/drand/drand/chain" dclient "github.com/drand/drand/client" - gclient "github.com/drand/drand/cmd/relay-gossip/client" + hclient "github.com/drand/drand/client/http" dlog "github.com/drand/drand/log" + gclient "github.com/drand/drand/lp2p/client" "github.com/drand/kyber" kzap "github.com/go-kit/kit/log/zap" "go.uber.org/zap/zapcore" @@ -79,11 +80,22 @@ func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub) (*DrandBeacon dlogger := dlog.NewKitLoggerFrom(kzap.NewZapSugarLogger( log.SugaredLogger.Desugar(), zapcore.InfoLevel)) + + var clients []dclient.Client + for _, url := range drandServers { + hc, err := hclient.NewWithInfo(url, drandChain, nil) + if err != nil { + return nil, xerrors.Errorf("could not create http drand client: %w", err) + } + clients = append(clients, hc) + + } + opts := []dclient.Option{ dclient.WithChainInfo(drandChain), - dclient.WithHTTPEndpoints(drandServers), dclient.WithCacheSize(1024), dclient.WithLogger(dlogger), + dclient.WithAutoWatch(), } if ps != nil { @@ -92,19 +104,11 @@ func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub) (*DrandBeacon log.Info("drand beacon without pubsub") } - client, err := dclient.New(opts...) + client, err := dclient.Wrap(clients, opts...) if err != nil { return nil, xerrors.Errorf("creating drand client") } - go func() { - // Explicitly Watch until that is fixed in drand - ch := client.Watch(context.Background()) - for range ch { - } - log.Error("dranch Watch bork") - }() - db := &DrandBeacon{ client: client, localCache: make(map[uint64]types.BeaconEntry), diff --git a/chain/beacon/drand/drand_test.go b/chain/beacon/drand/drand_test.go index 85b80cbe6..f35f3e4cc 100644 --- a/chain/beacon/drand/drand_test.go +++ b/chain/beacon/drand/drand_test.go @@ -5,12 +5,12 @@ import ( "testing" dchain "github.com/drand/drand/chain" - dclient "github.com/drand/drand/client" + hclient "github.com/drand/drand/client/http" "github.com/stretchr/testify/assert" ) func TestPrintGroupInfo(t *testing.T) { - c, err := dclient.NewHTTPClient(drandServers[0], nil, nil) + c, err := hclient.New(drandServers[0], nil, nil) assert.NoError(t, err) cg := c.(interface { FetchChainInfo(groupHash []byte) (*dchain.Info, error) diff --git a/go.mod b/go.mod index 89e2e2842..55d33c390 100644 --- a/go.mod +++ b/go.mod @@ -11,8 +11,8 @@ require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/coreos/go-systemd/v22 v22.0.0 github.com/docker/go-units v0.4.0 - github.com/drand/drand v0.9.2-0.20200529123141-6b138aefaef2 - github.com/drand/kyber v1.0.2 + github.com/drand/drand v0.9.2-0.20200616080806-a94e9c1636a4 + github.com/drand/kyber v1.1.0 github.com/fatih/color v1.8.0 github.com/filecoin-project/chain-validation v0.0.6-0.20200615191232-6be1a8c6ed09 github.com/filecoin-project/filecoin-ffi v0.26.1-0.20200508175440-05b30afeb00d @@ -36,7 +36,6 @@ require ( github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect - github.com/golang/protobuf v1.4.2 // indirect github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.4 github.com/gorilla/websocket v1.4.2 @@ -118,9 +117,10 @@ require ( go.uber.org/multierr v1.5.0 go.uber.org/zap v1.15.0 go4.org v0.0.0-20190313082347-94abd6928b1d // indirect - golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 + golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 + google.golang.org/api v0.25.0 // indirect launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect ) diff --git a/go.sum b/go.sum index 83af4904c..1e7e750aa 100644 --- a/go.sum +++ b/go.sum @@ -2,11 +2,32 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= contrib.go.opencensus.io/exporter/jaeger v0.1.0 h1:WNc9HbA38xEQmsI40Tjd/MNU/g8byN2Of7lwIjv0Jdc= contrib.go.opencensus.io/exporter/jaeger v0.1.0/go.mod h1:VYianECmuFPwU37O699Vc1GOcy+y8kOsfaxHRImmjbA= contrib.go.opencensus.io/exporter/prometheus v0.1.0 h1:SByaIoWwNgMdPSgl5sMqM2KDE5H/ukPWBRo314xiDvg= contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= @@ -16,6 +37,7 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkBy github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg= @@ -49,7 +71,7 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= -github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0 h1:5hryIiq9gtn+MiLVn0wP37kb/uTeRZgN08WoCsAhIhI= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -96,6 +118,9 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -119,6 +144,8 @@ github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= @@ -154,11 +181,15 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/drand/bls12-381 v0.3.2 h1:RImU8Wckmx8XQx1tp1q04OV73J9Tj6mmpQLYDP7V1XE= github.com/drand/bls12-381 v0.3.2/go.mod h1:dtcLgPtYT38L3NO6mPDYH0nbpc5tjPassDqiniuAt4Y= -github.com/drand/drand v0.9.2-0.20200529123141-6b138aefaef2 h1:IUxiqYU8BM8CyjHqxOTUbjrq7OvSDm8n1v9igaxtapM= -github.com/drand/drand v0.9.2-0.20200529123141-6b138aefaef2/go.mod h1:UIhZq1vHM1BQbpLa9K2HkxpCtbdsGkCc9MnCdKJxYjk= +github.com/drand/drand v0.9.2-0.20200616080806-a94e9c1636a4 h1:wEpu4hGFF0m0uDq/gxT9Ca/HWek0tvsMqsyPpLBWJ/E= +github.com/drand/drand v0.9.2-0.20200616080806-a94e9c1636a4/go.mod h1:Bu8QYdU0YdB2ZQZezHxabmOIciddiwLRnyV4nuZ2HQE= github.com/drand/kyber v1.0.1-0.20200110225416-8de27ed8c0e2/go.mod h1:UpXoA0Upd1N9l4TvRPHr1qAUBBERj6JQ/mnKI3BPEmw= github.com/drand/kyber v1.0.2 h1:dHjtWJZJdn3zBBZ9pqLsLfcR9ScvDvSqzS1sWA8seao= github.com/drand/kyber v1.0.2/go.mod h1:x6KOpK7avKj0GJ4emhXFP5n7M7W7ChAPmnQh/OL6vRw= +github.com/drand/kyber v1.1.0 h1:uBfD8gwpVufr+7Dvbxi4jGQ+qoMCO5tRfhYPyn+Tpqk= +github.com/drand/kyber v1.1.0/go.mod h1:x6KOpK7avKj0GJ4emhXFP5n7M7W7ChAPmnQh/OL6vRw= +github.com/drand/kyber-bls12381 v0.1.0 h1:/P4C65VnyEwxzR5ZYYVMNzY1If+aYBrdUU5ukwh7LQw= +github.com/drand/kyber-bls12381 v0.1.0/go.mod h1:N1emiHpm+jj7kMlxEbu3MUyOiooTgNySln564cgD9mk= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -243,6 +274,9 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= @@ -260,8 +294,11 @@ github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968 h1:s+PDl6lozQ+dEUtUtQn github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.1.0 h1:kFkMAZBNAn4j7K0GiZr8cRYzejq68VbheufiV3YuyFI= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/googleapis v1.4.0 h1:zgVt4UpGxcqVOw97aRGxT4svlcmdK35fynLNctY32zI= +github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -270,15 +307,23 @@ github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/status v1.0.3 h1:WkVBY59mw7qUNTr/bLwO7J2vesJ0rQ2C3tMXrTd3w5M= github.com/gogo/status v1.0.3/go.mod h1:SavQ51ycCLnc7dGyJxp8YAmudx8xqiVrRf+6IXRsugc= +github.com/gogo/status v1.1.0 h1:+eIkrewn5q6b30y+g/BJINVVdi2xH7je5MPJ3ZPK3JA= +github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -291,6 +336,7 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -310,12 +356,18 @@ github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbB github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f h1:KMlcu9X58lhTA/KrfX8Bi1LQSO4pzoVjTiL3h4Jk+Zk= @@ -336,12 +388,15 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.0 h1:0IKlLyQ3Hs9nDaiK5cSHAGmcQEIC8l2Ts1u6x5Dfrqg= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.0/go.mod h1:mJzapYve32yjrKlk9GbyCZHuPgZsrbyIbyKhSzOpg6s= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.14.3 h1:OCJlWkOUoTnl0neNGlf4fUm3TmbEtguw7vR+nGtnDjY= -github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= +github.com/grpc-ecosystem/grpc-gateway v1.14.6 h1:8ERzHx8aj1Sc47mu9n/AksaKCSWrMchFtkdrS4BIj5o= +github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4GkDEdKCRJduHpTxT3/jcw= +github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= github.com/gxed/go-shellwords v1.0.3/go.mod h1:N7paucT91ByIjmVJHhvoarjoQnmsi3Jd3vH7VqgtMxQ= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= @@ -384,6 +439,7 @@ github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOo github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d h1:/WZQPMZNsjZ7IlCpsLGdQBINg5bxKQ1K1sh6awxLtkA= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= @@ -523,7 +579,6 @@ github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBW github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= github.com/ipfs/go-log/v2 v2.0.8/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= -github.com/ipfs/go-log/v2 v2.1.0/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/ipfs/go-log/v2 v2.1.2-0.20200609205458-f8d20c392cb7 h1:LtL/rvdfbKSthZGmAAD9o4KKg6HA6Qn8gXCCdgnj7lw= github.com/ipfs/go-log/v2 v2.1.2-0.20200609205458-f8d20c392cb7/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/ipfs/go-merkledag v0.0.3/go.mod h1:Oc5kIXLHokkE1hWGMBHw+oxehkAaTOqtEb7Zbh6BhLA= @@ -599,6 +654,7 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -608,12 +664,16 @@ github.com/kabukky/httpscerts v0.0.0-20150320125433-617593d7dcb3/go.mod h1:BYpt4 github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kilic/bls12-381 v0.0.0-20200607163746-32e1441c8a9f h1:qET3Wx0v8tMtoTOQnsJXVvqvCopSf48qobR6tcJuDHo= +github.com/kilic/bls12-381 v0.0.0-20200607163746-32e1441c8a9f/go.mod h1:XXfR6YFCRSrkEXbNlIyDsgXVNJWVUV30m/ebkVy9n6s= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d h1:68u9r4wEvL3gYg2jvAOgROwZ3H+Y3hIDk4tbbmIjcYQ= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= @@ -777,8 +837,8 @@ github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYc github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= -github.com/libp2p/go-libp2p-pubsub v0.3.0/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato= github.com/libp2p/go-libp2p-pubsub v0.3.1/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato= +github.com/libp2p/go-libp2p-pubsub v0.3.2-0.20200527132641-c0712c6e92cf/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato= github.com/libp2p/go-libp2p-pubsub v0.3.2 h1:k3cJm5JW5mjaWZkobS50sJLJWaB2mBi0HW4eRlE8mSo= github.com/libp2p/go-libp2p-pubsub v0.3.2/go.mod h1:Uss7/Cfz872KggNb+doCVPHeCDmXB7z500m/R8DaAUk= github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU= @@ -1060,9 +1120,13 @@ github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoT github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02 h1:0R5mDLI66Qw13qN80TRz85zthQ2nf2+uDyiV23w6c3Q= github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02/go.mod h1:JNdpVEzCpXBgIiv4ds+TzhN1hrtxq6ClLrTlT9OQRSc= +github.com/opentracing-contrib/go-grpc v0.0.0-20191001143057-db30781987df h1:vdYtBU6zvL7v+Tr+0xFM/qhahw/EvY8DMMunZHKH6eE= +github.com/opentracing-contrib/go-grpc v0.0.0-20191001143057-db30781987df/go.mod h1:DYR5Eij8rJl8h7gblRrOZ8g0kW1umSpKqYIBTgeDtLo= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9 h1:QsgXACQhd9QJhEmRumbsMQQvBtmdS0mafoVEBplWXEg= github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w= +github.com/opentracing-contrib/go-stdlib v1.0.0 h1:TBS7YuVotp8myLon4Pv7BtCBzOTo1DeZCld0Z63mW2w= +github.com/opentracing-contrib/go-stdlib v1.0.0/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= @@ -1100,8 +1164,8 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= -github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A= +github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -1117,6 +1181,8 @@ github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/node_exporter v1.0.0-rc.0.0.20200428091818-01054558c289/go.mod h1:FGbBv5OPKjch+jNUJmEQpMZytIdyW0NdBtWFcfSKusc= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1128,6 +1194,8 @@ github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.1.0 h1:jhMy6QXfi3y2HEzFoyuCj40z4OZIIHHPtFyCMftmvKA= +github.com/prometheus/procfs v0.1.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -1175,6 +1243,8 @@ github.com/siebenmann/go-kstat v0.0.0-20160321171754-d34789b79745/go.mod h1:G81a github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= @@ -1306,6 +1376,7 @@ go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= @@ -1354,6 +1425,7 @@ golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -1368,19 +1440,38 @@ golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200427165652-729f1e841bcc/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM= +golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1402,6 +1493,8 @@ golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1410,17 +1503,27 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200519113804-d87ec0cfa476 h1:E7ct1C6/33eOdrGZKMoyntcEvs2dwZnDe30crG5vpYU= golang.org/x/net v0.0.0-20200519113804-d87ec0cfa476/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1429,6 +1532,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1453,10 +1558,13 @@ golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190524122548-abf6ff778158/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190524152521-dbbf3f1254d4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1466,25 +1574,34 @@ golang.org/x/sys v0.0.0-20190902133755-9109b7679e13/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191025090151-53bf42e6b339/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200427175716-29b57079015a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1492,6 +1609,7 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1503,26 +1621,44 @@ golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191030062658-86caa796c7ab/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200108195415-316d2f248479/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200216192241-b320d3a0f5a2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200318150045-ba25ddc85566 h1:OXjomkWHhzUx4+HldlJ2TsMxJdWgEo5CTtspD1wdhdk= golang.org/x/tools v0.0.0-20200318150045-ba25ddc85566/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4 h1:kDtqNkeBrZb8B+atrj50B5XLHpzXXqcCdZPP/ApQ5NY= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -1533,21 +1669,54 @@ google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx1 google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.3.2 h1:iTp+3yyl/KOtxa/d1/JUE0GGSoR6FuW5udver22iwpw= google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.25.0 h1:LodzhlzZEUfhXzNUMIfVlf9Gr6Ua5MMtoFWh7+f47qA= +google.golang.org/api v0.25.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c h1:hrpEMCZ2O7DR5gC1n2AJGVhrwiEjOi35+jxtIuZpTMo= -google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200608115520-7c474a2e3482 h1:i+Aiej6cta/Frzp13/swvwz5O00kYcSe0A/C5Wd7zX8= +google.golang.org/genproto v0.0.0-20200608115520-7c474a2e3482/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1555,21 +1724,29 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1603,6 +1780,7 @@ grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJd honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= @@ -1611,6 +1789,7 @@ howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqp howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= From 9690bc9c638909b7136887c6f6f5db8bec76a8a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 17 Jun 2020 00:35:51 +0200 Subject: [PATCH 139/160] seed: Cleanup unsealed sectors --- cmd/lotus-seed/seed/seed.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index ffb651c00..be366d4db 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -92,6 +92,10 @@ func PreSeal(maddr address.Address, spt abi.RegisteredSealProof, offset abi.Sect return nil, nil, xerrors.Errorf("trim cache: %w", err) } + if err := cleanupUnsealed(sbfs, sid); err != nil { + return nil, nil, xerrors.Errorf("remove unsealed file: %w", err) + } + log.Warn("PreCommitOutput: ", sid, cids.Sealed, cids.Unsealed) sealedSectors = append(sealedSectors, &genesis.PreSeal{ CommR: cids.Sealed, @@ -161,6 +165,16 @@ func PreSeal(maddr address.Address, spt abi.RegisteredSealProof, offset abi.Sect return miner, &minerAddr.KeyInfo, nil } +func cleanupUnsealed(sbfs *basicfs.Provider, sid abi.SectorID) error { + paths, done, err := sbfs.AcquireSector(context.TODO(), sid, stores.FTUnsealed, stores.FTNone, stores.PathSealing) + if err != nil { + return err + } + defer done() + + return os.Remove(paths.Unsealed) +} + func WriteGenesisMiner(maddr address.Address, sbroot string, gm *genesis.Miner, key *types.KeyInfo) error { output := map[string]genesis.Miner{ maddr.String(): *gm, From b845013836376c2f466fb047a2bca560490dd634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 16 Jun 2020 13:59:15 +0200 Subject: [PATCH 140/160] Set build params for testnet --- build/params_testnet.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build/params_testnet.go b/build/params_testnet.go index c222862eb..202dfb231 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -12,9 +12,8 @@ import ( ) func init() { - power.ConsensusMinerMinPower = big.NewInt(1024 << 20) + power.ConsensusMinerMinPower = big.NewInt(1024 << 30) miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ - abi.RegisteredSealProof_StackedDrg512MiBV1: {}, abi.RegisteredSealProof_StackedDrg32GiBV1: {}, abi.RegisteredSealProof_StackedDrg64GiBV1: {}, } From 6253c39100f4fcc16099e6f4fbcd620aad11c13c Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 16 Jun 2020 15:38:48 -0700 Subject: [PATCH 141/160] replace "set-price" with "set-ask" Fixes #2027 --- api/api_storage.go | 2 +- api/apistruct/struct.go | 12 +++--- cmd/lotus-storage-miner/main.go | 1 - cmd/lotus-storage-miner/market.go | 71 +++++++++++++++++++++++++------ node/impl/storminer.go | 9 +++- 5 files changed, 72 insertions(+), 23 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index 04ff8311c..e07505e3f 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -50,7 +50,7 @@ type StorageMiner interface { MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error) MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error) - MarketSetPrice(context.Context, types.BigInt) error + MarketSetAsk(ctx context.Context, price types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 02d9e5155..55c87dbd4 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -192,10 +192,10 @@ type StorageMinerStruct struct { MiningBase func(context.Context) (*types.TipSet, error) `perm:"read"` - MarketImportDealData func(context.Context, cid.Cid, string) error `perm:"write"` - MarketListDeals func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` - MarketListIncompleteDeals func(ctx context.Context) ([]storagemarket.MinerDeal, error) `perm:"read"` - MarketSetPrice func(context.Context, types.BigInt) error `perm:"admin"` + MarketImportDealData func(context.Context, cid.Cid, string) error `perm:"write"` + MarketListDeals func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` + MarketListIncompleteDeals func(ctx context.Context) ([]storagemarket.MinerDeal, error) `perm:"read"` + MarketSetAsk func(ctx context.Context, price types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error `perm:"admin"` PledgeSector func(context.Context) error `perm:"write"` @@ -841,8 +841,8 @@ func (c *StorageMinerStruct) MarketListIncompleteDeals(ctx context.Context) ([]s return c.Internal.MarketListIncompleteDeals(ctx) } -func (c *StorageMinerStruct) MarketSetPrice(ctx context.Context, p types.BigInt) error { - return c.Internal.MarketSetPrice(ctx, p) +func (c *StorageMinerStruct) MarketSetAsk(ctx context.Context, price types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error { + return c.Internal.MarketSetAsk(ctx, price, duration, minPieceSize, maxPieceSize) } func (c *StorageMinerStruct) DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error { diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index ba7722e4e..dc6de7029 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -30,7 +30,6 @@ func main() { stopCmd, sectorsCmd, storageCmd, - setPriceCmd, workersCmd, provingCmd, } diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index c2f2555fa..74a42d85d 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -4,10 +4,13 @@ import ( "encoding/json" "fmt" - "github.com/filecoin-project/lotus/chain/types" - lcli "github.com/filecoin-project/lotus/cli" + "github.com/docker/go-units" + "github.com/filecoin-project/specs-actors/actors/abi" "github.com/ipfs/go-cid" "github.com/urfave/cli/v2" + + "github.com/filecoin-project/lotus/chain/types" + lcli "github.com/filecoin-project/lotus/cli" ) var enableCmd = &cli.Command{ @@ -40,29 +43,70 @@ var disableCmd = &cli.Command{ }, } -var setPriceCmd = &cli.Command{ - Name: "set-price", - Usage: "Set price that miner will accept storage deals at (FIL / GiB / Epoch)", - Flags: []cli.Flag{}, +var setAskCmd = &cli.Command{ + Name: "set-ask", + Usage: "Configure the miner's ask", + Flags: []cli.Flag{ + &cli.Uint64Flag{ + Name: "price", + Usage: "Set the price of the ask (specified as FIL / GiB / Epoch) to `PRICE`", + Required: true, + }, + &cli.Uint64Flag{ + Name: "duration", + Usage: "Set the duration (specified as epochs) of the ask to `DURATION`", + DefaultText: "8640000", + Value: 8640000, + }, + &cli.StringFlag{ + Name: "min-piece-size", + Usage: "Set minimum piece size (without bit-padding, in bytes) in ask to `SIZE`", + DefaultText: "254B", + Value: "254B", + }, + &cli.StringFlag{ + Name: "max-piece-size", + Usage: "Set maximum piece size (without bit-padding, in bytes) in ask to `SIZE`", + DefaultText: "miner sector size, without bit-padding", + }, + }, Action: func(cctx *cli.Context) error { + ctx := lcli.DaemonContext(cctx) + api, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { return err } defer closer() - ctx := lcli.DaemonContext(cctx) + pri := types.NewInt(cctx.Uint64("price")) + dur := abi.ChainEpoch(cctx.Uint64("duration")) - if !cctx.Args().Present() { - return fmt.Errorf("must specify price to set") - } - - fp, err := types.ParseFIL(cctx.Args().First()) + min, err := units.RAMInBytes(cctx.String("min-piece-size")) if err != nil { return err } - return api.MarketSetPrice(ctx, types.BigInt(fp)) + max, err := units.RAMInBytes(cctx.String("max-piece-size")) + if err != nil { + return err + } + + if max == 0 { + maddr, err := api.ActorAddress(ctx) + if err != nil { + return err + } + + ssize, err := api.ActorSectorSize(ctx, maddr) + if err != nil { + return err + } + + max = int64(abi.PaddedPieceSize(ssize).Unpadded()) + } + + return api.MarketSetAsk(ctx, pri, dur, abi.UnpaddedPieceSize(min).Padded(), abi.UnpaddedPieceSize(max).Padded()) }, } @@ -74,6 +118,7 @@ var dealsCmd = &cli.Command{ dealsListCmd, enableCmd, disableCmd, + setAskCmd, }, } diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 0cc13177e..7ece10ac0 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -201,8 +201,13 @@ func (sm *StorageMinerAPI) MarketListIncompleteDeals(ctx context.Context) ([]sto return sm.StorageProvider.ListLocalDeals() } -func (sm *StorageMinerAPI) MarketSetPrice(ctx context.Context, p types.BigInt) error { - return sm.StorageProvider.SetAsk(p, 60*60*24*100) // lasts for 100 days? +func (sm *StorageMinerAPI) MarketSetAsk(ctx context.Context, price types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error { + options := []storagemarket.StorageAskOption{ + storagemarket.MinPieceSize(minPieceSize), + storagemarket.MaxPieceSize(maxPieceSize), + } + + return sm.StorageProvider.SetAsk(price, duration, options...) } func (sm *StorageMinerAPI) DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) { From 5eceed81e192a9aca66dbe06e6ae64d28e0c264e Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 16 Jun 2020 17:18:54 -0700 Subject: [PATCH 142/160] add "get-ask" command --- api/api_storage.go | 1 + api/apistruct/struct.go | 5 ++++ cmd/lotus-storage-miner/market.go | 46 +++++++++++++++++++++++++++++-- node/impl/storminer.go | 4 +++ 4 files changed, 53 insertions(+), 3 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index e07505e3f..90de01fb9 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -51,6 +51,7 @@ type StorageMiner interface { MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error) MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error) MarketSetAsk(ctx context.Context, price types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error + MarketGetAsk(ctx context.Context) (*storagemarket.SignedStorageAsk, error) DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 55c87dbd4..a1fe69f06 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -196,6 +196,7 @@ type StorageMinerStruct struct { MarketListDeals func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` MarketListIncompleteDeals func(ctx context.Context) ([]storagemarket.MinerDeal, error) `perm:"read"` MarketSetAsk func(ctx context.Context, price types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error `perm:"admin"` + MarketGetAsk func(ctx context.Context) (*storagemarket.SignedStorageAsk, error) `perm:"read"` PledgeSector func(context.Context) error `perm:"write"` @@ -845,6 +846,10 @@ func (c *StorageMinerStruct) MarketSetAsk(ctx context.Context, price types.BigIn return c.Internal.MarketSetAsk(ctx, price, duration, minPieceSize, maxPieceSize) } +func (c *StorageMinerStruct) MarketGetAsk(ctx context.Context) (*storagemarket.SignedStorageAsk, error) { + return c.Internal.MarketGetAsk(ctx) +} + func (c *StorageMinerStruct) DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error { return c.Internal.DealsImportData(ctx, dealPropCid, file) } diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 74a42d85d..b2b7155c3 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -3,12 +3,16 @@ package main import ( "encoding/json" "fmt" + "os" + "text/tabwriter" "github.com/docker/go-units" - "github.com/filecoin-project/specs-actors/actors/abi" "github.com/ipfs/go-cid" "github.com/urfave/cli/v2" + "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" ) @@ -55,8 +59,8 @@ var setAskCmd = &cli.Command{ &cli.Uint64Flag{ Name: "duration", Usage: "Set the duration (specified as epochs) of the ask to `DURATION`", - DefaultText: "8640000", - Value: 8640000, + DefaultText: "100000", + Value: 100000, }, &cli.StringFlag{ Name: "min-piece-size", @@ -110,6 +114,41 @@ var setAskCmd = &cli.Command{ }, } +var getAskCmd = &cli.Command{ + Name: "get-ask", + Usage: "Print the miner's ask", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + ctx := lcli.DaemonContext(cctx) + + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + sask, err := api.MarketGetAsk(ctx) + if err != nil { + return err + } + + var ask *storagemarket.StorageAsk + if sask != nil && sask.Ask != nil { + ask = sask.Ask + } + + w := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0) + fmt.Fprintf(w, "Price per GiB / Epoch\tMin. Piece Size (bytes, unpadded)\tMax. Piece Size (bytes, unpadded)\tExpiry\tSeq. No.\n") + if ask == nil { + fmt.Fprintf(w, "\n") + } else { + fmt.Fprintf(w, "%s\t%d\t%d\t%d\t%d\n", ask.Price, ask.MinPieceSize.Unpadded(), ask.MaxPieceSize.Unpadded(), ask.Expiry, ask.SeqNo) + } + + return w.Flush() + }, +} + var dealsCmd = &cli.Command{ Name: "deals", Usage: "interact with your deals", @@ -119,6 +158,7 @@ var dealsCmd = &cli.Command{ enableCmd, disableCmd, setAskCmd, + getAskCmd, }, } diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 7ece10ac0..ed94e173d 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -210,6 +210,10 @@ func (sm *StorageMinerAPI) MarketSetAsk(ctx context.Context, price types.BigInt, return sm.StorageProvider.SetAsk(price, duration, options...) } +func (sm *StorageMinerAPI) MarketGetAsk(ctx context.Context) (*storagemarket.SignedStorageAsk, error) { + return sm.StorageProvider.GetAsk(), nil +} + func (sm *StorageMinerAPI) DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) { return sm.StorageProvider.ListDeals(ctx) } From 4ba1846c11d031716ec0d336d8593d20736559c4 Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 16 Jun 2020 17:32:45 -0700 Subject: [PATCH 143/160] format the piece sizes --- cmd/lotus-storage-miner/market.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index b2b7155c3..60c53bff8 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -138,11 +138,11 @@ var getAskCmd = &cli.Command{ } w := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0) - fmt.Fprintf(w, "Price per GiB / Epoch\tMin. Piece Size (bytes, unpadded)\tMax. Piece Size (bytes, unpadded)\tExpiry\tSeq. No.\n") + fmt.Fprintf(w, "Price per GiB / Epoch\tMin. Piece Size (unpadded)\tMax. Piece Size (unpadded)\tExpiry\tSeq. No.\n") if ask == nil { fmt.Fprintf(w, "\n") } else { - fmt.Fprintf(w, "%s\t%d\t%d\t%d\t%d\n", ask.Price, ask.MinPieceSize.Unpadded(), ask.MaxPieceSize.Unpadded(), ask.Expiry, ask.SeqNo) + fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%d\n", ask.Price, types.SizeStr(types.NewInt(uint64(ask.MinPieceSize.Unpadded()))), types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize.Unpadded()))), ask.Expiry, ask.SeqNo) } return w.Flush() From 5acf5bf102d9f11eb63f49eaecaeb8604d202380 Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 16 Jun 2020 17:45:11 -0700 Subject: [PATCH 144/160] upper and lower bounds checking --- cmd/lotus-storage-miner/market.go | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 60c53bff8..84465f60d 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -8,6 +8,7 @@ import ( "github.com/docker/go-units" "github.com/ipfs/go-cid" + "github.com/pkg/errors" "github.com/urfave/cli/v2" "github.com/filecoin-project/go-fil-markets/storagemarket" @@ -91,23 +92,33 @@ var setAskCmd = &cli.Command{ return err } + if min < 254 { + return errors.New("minimum piece size (unpadded) is 254 B") + } + max, err := units.RAMInBytes(cctx.String("max-piece-size")) if err != nil { return err } + maddr, err := api.ActorAddress(ctx) + if err != nil { + return err + } + + ssize, err := api.ActorSectorSize(ctx, maddr) + if err != nil { + return err + } + + smax := int64(abi.PaddedPieceSize(ssize).Unpadded()) + if max == 0 { - maddr, err := api.ActorAddress(ctx) - if err != nil { - return err - } + max = smax + } - ssize, err := api.ActorSectorSize(ctx, maddr) - if err != nil { - return err - } - - max = int64(abi.PaddedPieceSize(ssize).Unpadded()) + if max > smax { + return errors.Errorf("max piece size (unpadded) %s cannot exceed miner sector size (unpadded) %s", types.SizeStr(types.NewInt(uint64(max))), types.SizeStr(types.NewInt(uint64(smax)))) } return api.MarketSetAsk(ctx, pri, dur, abi.UnpaddedPieceSize(min).Padded(), abi.UnpaddedPieceSize(max).Padded()) From 2676892886cf674528528b46432fa80a76baaefa Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 16 Jun 2020 17:47:04 -0700 Subject: [PATCH 145/160] go mod tidy --- go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/go.mod b/go.mod index d5e684e46..cb94ad055 100644 --- a/go.mod +++ b/go.mod @@ -103,6 +103,7 @@ require ( github.com/multiformats/go-multibase v0.0.2 github.com/multiformats/go-multihash v0.0.13 github.com/opentracing/opentracing-go v1.1.0 + github.com/pkg/errors v0.9.1 github.com/stretchr/objx v0.2.0 // indirect github.com/stretchr/testify v1.5.1 github.com/syndtr/goleveldb v1.0.0 From 139c3297ab59939285842baea007276019add121 Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 17 Jun 2020 08:34:50 -0700 Subject: [PATCH 146/160] change "duration" help text --- cmd/lotus-storage-miner/market.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 84465f60d..1163e09e8 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -59,7 +59,7 @@ var setAskCmd = &cli.Command{ }, &cli.Uint64Flag{ Name: "duration", - Usage: "Set the duration (specified as epochs) of the ask to `DURATION`", + Usage: "Set the number of epochs (from now) that the ask will expire `DURATION`", DefaultText: "100000", Value: 100000, }, From 5d4f1bb3f1b8004ce75cff2e3154d501014ea0a3 Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 17 Jun 2020 08:42:30 -0700 Subject: [PATCH 147/160] work with bit-padded byte quantities, as per PR feedback --- cmd/lotus-storage-miner/market.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 1163e09e8..8d677a55c 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -10,6 +10,7 @@ import ( "github.com/ipfs/go-cid" "github.com/pkg/errors" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/specs-actors/actors/abi" @@ -59,20 +60,20 @@ var setAskCmd = &cli.Command{ }, &cli.Uint64Flag{ Name: "duration", - Usage: "Set the number of epochs (from now) that the ask will expire `DURATION`", + Usage: "Set the number of epochs from the current chain head that the ask will expire `DURATION`", DefaultText: "100000", Value: 100000, }, &cli.StringFlag{ Name: "min-piece-size", - Usage: "Set minimum piece size (without bit-padding, in bytes) in ask to `SIZE`", - DefaultText: "254B", - Value: "254B", + Usage: "Set minimum piece size (w/bit-padding, in bytes) in ask to `SIZE`", + DefaultText: "256B", + Value: "256B", }, &cli.StringFlag{ Name: "max-piece-size", - Usage: "Set maximum piece size (without bit-padding, in bytes) in ask to `SIZE`", - DefaultText: "miner sector size, without bit-padding", + Usage: "Set maximum piece size (w/bit-padding, in bytes) in ask to `SIZE`", + DefaultText: "miner sector size", }, }, Action: func(cctx *cli.Context) error { @@ -92,8 +93,8 @@ var setAskCmd = &cli.Command{ return err } - if min < 254 { - return errors.New("minimum piece size (unpadded) is 254 B") + if min < 256 { + return xerrors.New("minimum piece size (w/bit-padding) is 256B") } max, err := units.RAMInBytes(cctx.String("max-piece-size")) @@ -111,17 +112,17 @@ var setAskCmd = &cli.Command{ return err } - smax := int64(abi.PaddedPieceSize(ssize).Unpadded()) + smax := int64(ssize) if max == 0 { max = smax } if max > smax { - return errors.Errorf("max piece size (unpadded) %s cannot exceed miner sector size (unpadded) %s", types.SizeStr(types.NewInt(uint64(max))), types.SizeStr(types.NewInt(uint64(smax)))) + return errors.Errorf("max piece size (w/bit-padding) %s cannot exceed miner sector size %s", types.SizeStr(types.NewInt(uint64(max))), types.SizeStr(types.NewInt(uint64(smax)))) } - return api.MarketSetAsk(ctx, pri, dur, abi.UnpaddedPieceSize(min).Padded(), abi.UnpaddedPieceSize(max).Padded()) + return api.MarketSetAsk(ctx, pri, dur, abi.PaddedPieceSize(min), abi.PaddedPieceSize(max)) }, } @@ -149,11 +150,11 @@ var getAskCmd = &cli.Command{ } w := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0) - fmt.Fprintf(w, "Price per GiB / Epoch\tMin. Piece Size (unpadded)\tMax. Piece Size (unpadded)\tExpiry\tSeq. No.\n") + fmt.Fprintf(w, "Price per GiB / Epoch\tMin. Piece Size (w/bit-padding)\tMax. Piece Size (w/bit-padding)\tExpiry\tSeq. No.\n") if ask == nil { fmt.Fprintf(w, "\n") } else { - fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%d\n", ask.Price, types.SizeStr(types.NewInt(uint64(ask.MinPieceSize.Unpadded()))), types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize.Unpadded()))), ask.Expiry, ask.SeqNo) + fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%d\n", ask.Price, types.SizeStr(types.NewInt(uint64(ask.MinPieceSize))), types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize))), ask.Expiry, ask.SeqNo) } return w.Flush() From 8b8bb5e833b82402fb7e764701227f1d56e18cb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 17 Jun 2020 16:44:59 +0200 Subject: [PATCH 148/160] rpc: add Closing method --- api/api_common.go | 2 ++ api/apistruct/struct.go | 7 ++++++- node/impl/common/common.go | 4 ++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/api/api_common.go b/api/api_common.go index e5556d16f..6d47e35f7 100644 --- a/api/api_common.go +++ b/api/api_common.go @@ -38,6 +38,8 @@ type Common interface { // trigger graceful shutdown Shutdown(context.Context) error + + Closing(context.Context) (<-chan struct{}, error) } // Version provides various build-time information diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 02d9e5155..dfa7a44ee 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -50,7 +50,8 @@ type CommonStruct struct { LogList func(context.Context) ([]string, error) `perm:"write"` LogSetLevel func(context.Context, string, string) error `perm:"write"` - Shutdown func(context.Context) error `perm:"admin"` + Shutdown func(context.Context) error `perm:"admin"` + Closing func(context.Context) (<-chan struct{}, error) `perm:"read"` } } @@ -313,6 +314,10 @@ func (c *CommonStruct) Shutdown(ctx context.Context) error { return c.Internal.Shutdown(ctx) } +func (c *CommonStruct) Closing(ctx context.Context) (<-chan struct{}, error) { + return c.Internal.Closing(ctx) +} + // FullNodeStruct func (c *FullNodeStruct) ClientListImports(ctx context.Context) ([]api.Import, error) { diff --git a/node/impl/common/common.go b/node/impl/common/common.go index aa85d9182..3a42872d9 100644 --- a/node/impl/common/common.go +++ b/node/impl/common/common.go @@ -139,4 +139,8 @@ func (a *CommonAPI) Shutdown(ctx context.Context) error { return nil } +func (a *CommonAPI) Closing(ctx context.Context) (<-chan struct{}, error) { + return make(chan struct{}), nil // relies on jsonrpc closing +} + var _ api.Common = &CommonAPI{} From fc7195f19ab9ccd2ce0e41cdca819843ee8466d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 17 Jun 2020 17:23:07 +0200 Subject: [PATCH 149/160] seal-worker: Wait for miner API on start --- cmd/lotus-seal-worker/main.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index 65418ead8..bb83ebf8c 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -3,11 +3,13 @@ package main import ( "context" "encoding/json" + "fmt" "io/ioutil" "net" "net/http" "os" "path/filepath" + "time" "github.com/google/uuid" "github.com/gorilla/mux" @@ -117,10 +119,18 @@ var runCmd = &cli.Command{ } // Connect to storage-miner + var nodeApi api.StorageMiner + var closer func() + var err error + for { + nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx) + if err == nil { + break + } + fmt.Printf("\r\x1b[0KConnecting to miner API... (%s)", err) + time.Sleep(time.Second) + continue - nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) - if err != nil { - return xerrors.Errorf("getting miner api: %w", err) } defer closer() ctx := lcli.ReqContext(cctx) From 2761872ea7e77bd709254b1637de541d3b171b8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 17 Jun 2020 17:43:16 +0200 Subject: [PATCH 150/160] seal-worker: Auto-restart when API connection is lost --- cmd/lotus-seal-worker/main.go | 44 ++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index bb83ebf8c..ff45687f8 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -9,6 +9,7 @@ import ( "net/http" "os" "path/filepath" + "syscall" "time" "github.com/google/uuid" @@ -130,8 +131,8 @@ var runCmd = &cli.Command{ fmt.Printf("\r\x1b[0KConnecting to miner API... (%s)", err) time.Sleep(time.Second) continue - } + defer closer() ctx := lcli.ReqContext(cctx) ctx, cancel := context.WithCancel(ctx) @@ -146,6 +147,8 @@ var runCmd = &cli.Command{ } log.Infof("Remote version %s", v) + watchMinerConn(ctx, cctx, nodeApi) + // Check params act, err := nodeApi.ActorAddress(ctx) @@ -327,3 +330,42 @@ var runCmd = &cli.Command{ return srv.Serve(nl) }, } + +func watchMinerConn(ctx context.Context, cctx *cli.Context, nodeApi api.StorageMiner) { + go func() { + closing, err := nodeApi.Closing(ctx) + if err != nil { + log.Errorf("failed to get remote closing channel: %+v", err) + } + + select { + case <-closing: + case <-ctx.Done(): + } + + if ctx.Err() != nil { + return // graceful shutdown + } + + log.Warnf("Connection with miner node lost, restarting") + + exe, err := os.Executable() + if err != nil { + log.Errorf("getting executable for auto-restart: %+v", err) + } + + log.Sync() + + // TODO: there are probably cleaner/more graceful ways to restart, + // but this is good enough for now (FSM can recover from the mess this creates) + if err := syscall.Exec(exe, []string{exe, "run", + fmt.Sprintf("--address=%s", cctx.String("address")), + fmt.Sprintf("--no-local-storage=%t", cctx.Bool("no-local-storage")), + fmt.Sprintf("--precommit1=%t", cctx.Bool("precommit1")), + fmt.Sprintf("--precommit2=%t", cctx.Bool("precommit2")), + fmt.Sprintf("--commit=%t", cctx.Bool("commit")), + }, os.Environ()); err != nil { + fmt.Println(err) + } + }() +} From 673a64318426dda40ca3244f82c7e9af662c8ceb Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 17 Jun 2020 08:57:18 -0700 Subject: [PATCH 151/160] use xerrors, as per feedback in PR --- cmd/lotus-storage-miner/market.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 8d677a55c..71be988e9 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -8,7 +8,6 @@ import ( "github.com/docker/go-units" "github.com/ipfs/go-cid" - "github.com/pkg/errors" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -119,7 +118,7 @@ var setAskCmd = &cli.Command{ } if max > smax { - return errors.Errorf("max piece size (w/bit-padding) %s cannot exceed miner sector size %s", types.SizeStr(types.NewInt(uint64(max))), types.SizeStr(types.NewInt(uint64(smax)))) + return xerrors.Errorf("max piece size (w/bit-padding) %s cannot exceed miner sector size %s", types.SizeStr(types.NewInt(uint64(max))), types.SizeStr(types.NewInt(uint64(smax)))) } return api.MarketSetAsk(ctx, pri, dur, abi.PaddedPieceSize(min), abi.PaddedPieceSize(max)) From 9d7be5dcbfac3e6cdb90e9ac5a33effa8b6c879c Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 17 Jun 2020 09:20:43 -0700 Subject: [PATCH 152/160] modify set-ask to work with human-readable clock time/duration --- cmd/lotus-storage-miner/market.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 71be988e9..a189706f3 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "text/tabwriter" + "time" "github.com/docker/go-units" "github.com/ipfs/go-cid" @@ -14,6 +15,7 @@ import ( "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" ) @@ -57,11 +59,11 @@ var setAskCmd = &cli.Command{ Usage: "Set the price of the ask (specified as FIL / GiB / Epoch) to `PRICE`", Required: true, }, - &cli.Uint64Flag{ + &cli.StringFlag{ Name: "duration", - Usage: "Set the number of epochs from the current chain head that the ask will expire `DURATION`", - DefaultText: "100000", - Value: 100000, + Usage: "Set duration of ask (a quantity of time after which the ask expires) `DURATION`", + DefaultText: "30d0h0m0s", + Value: "30d0h0m0s", }, &cli.StringFlag{ Name: "min-piece-size", @@ -85,11 +87,17 @@ var setAskCmd = &cli.Command{ defer closer() pri := types.NewInt(cctx.Uint64("price")) - dur := abi.ChainEpoch(cctx.Uint64("duration")) + + dur, err := time.ParseDuration(cctx.String("duration")) + if err != nil { + return xerrors.Errorf("cannot parse duration: %w", err) + } + + qty := dur.Seconds() / build.BlockDelay min, err := units.RAMInBytes(cctx.String("min-piece-size")) if err != nil { - return err + return xerrors.Errorf("cannot parse min-piece-size to quantity of bytes: %w", err) } if min < 256 { @@ -98,7 +106,7 @@ var setAskCmd = &cli.Command{ max, err := units.RAMInBytes(cctx.String("max-piece-size")) if err != nil { - return err + return xerrors.Errorf("cannot parse max-piece-size to quantity of bytes: %w", err) } maddr, err := api.ActorAddress(ctx) @@ -121,7 +129,7 @@ var setAskCmd = &cli.Command{ return xerrors.Errorf("max piece size (w/bit-padding) %s cannot exceed miner sector size %s", types.SizeStr(types.NewInt(uint64(max))), types.SizeStr(types.NewInt(uint64(smax)))) } - return api.MarketSetAsk(ctx, pri, dur, abi.PaddedPieceSize(min), abi.PaddedPieceSize(max)) + return api.MarketSetAsk(ctx, pri, abi.ChainEpoch(qty), abi.PaddedPieceSize(min), abi.PaddedPieceSize(max)) }, } From e60dd219dc8a89126fbf0c8470187fa59724e464 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 17 Jun 2020 18:21:40 +0200 Subject: [PATCH 153/160] wdpost: Don't return nil from checkSectors --- storage/wdpost_run.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 902c9b741..5c1170551 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -99,10 +99,6 @@ func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check *abi.BitFi log.Warnw("Checked sectors", "checked", len(tocheck), "good", len(sectors)) - if len(sectors) == 0 { // nothing to recover - return nil, nil - } - sbf := bitfield.New() for s := range sectors { (&sbf).Set(uint64(s.Number)) From 402cd8be1903bb50eff997ad1272e5810178aeb3 Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 17 Jun 2020 10:20:42 -0700 Subject: [PATCH 154/160] get-ask output should use durations, too --- cmd/lotus-storage-miner/market.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index a189706f3..977ae22de 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -62,8 +62,8 @@ var setAskCmd = &cli.Command{ &cli.StringFlag{ Name: "duration", Usage: "Set duration of ask (a quantity of time after which the ask expires) `DURATION`", - DefaultText: "30d0h0m0s", - Value: "30d0h0m0s", + DefaultText: "720h0m0s", + Value: "720h0m0s", }, &cli.StringFlag{ Name: "min-piece-size", @@ -157,13 +157,22 @@ var getAskCmd = &cli.Command{ } w := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0) - fmt.Fprintf(w, "Price per GiB / Epoch\tMin. Piece Size (w/bit-padding)\tMax. Piece Size (w/bit-padding)\tExpiry\tSeq. No.\n") + fmt.Fprintf(w, "Price per GiB / Epoch\tMin. Piece Size (w/bit-padding)\tMax. Piece Size (w/bit-padding)\tExpiry (Epoch)\tExpiry (Appx. Rem. Time)\tSeq. No.\n") if ask == nil { fmt.Fprintf(w, "\n") - } else { - fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%d\n", ask.Price, types.SizeStr(types.NewInt(uint64(ask.MinPieceSize))), types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize))), ask.Expiry, ask.SeqNo) + + return w.Flush() } + miningBaseTs, err := api.MiningBase(ctx) + if err != nil { + return err + } + + rem := time.Second * time.Duration((ask.Expiry-miningBaseTs.Height())*build.BlockDelay) + + fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%s\t%d\n", ask.Price, types.SizeStr(types.NewInt(uint64(ask.MinPieceSize))), types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize))), ask.Expiry, rem, ask.SeqNo) + return w.Flush() }, } From 791dff5a8712c3291a67f1977a08d27bfbdb27ba Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 17 Jun 2020 10:30:16 -0700 Subject: [PATCH 155/160] don't go into negative remaining time --- cmd/lotus-storage-miner/market.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 977ae22de..dd99a375c 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -169,7 +169,11 @@ var getAskCmd = &cli.Command{ return err } - rem := time.Second * time.Duration((ask.Expiry-miningBaseTs.Height())*build.BlockDelay) + dlt := ask.Expiry - miningBaseTs.Height() + rem := "" + if dlt > 0 { + rem = (time.Second * time.Duration(dlt*build.BlockDelay)).String() + } fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%s\t%d\n", ask.Price, types.SizeStr(types.NewInt(uint64(ask.MinPieceSize))), types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize))), ask.Expiry, rem, ask.SeqNo) From ed5b74b1ae377910aaf4398b5af885412142f806 Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 17 Jun 2020 10:42:09 -0700 Subject: [PATCH 156/160] tidy up --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index cb94ad055..d5e684e46 100644 --- a/go.mod +++ b/go.mod @@ -103,7 +103,6 @@ require ( github.com/multiformats/go-multibase v0.0.2 github.com/multiformats/go-multihash v0.0.13 github.com/opentracing/opentracing-go v1.1.0 - github.com/pkg/errors v0.9.1 github.com/stretchr/objx v0.2.0 // indirect github.com/stretchr/testify v1.5.1 github.com/syndtr/goleveldb v1.0.0 From fcdfda8ba2eb2e994d26eabe8be759c947f60ad7 Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 17 Jun 2020 10:56:42 -0700 Subject: [PATCH 157/160] use chain head instead of mining base --- cmd/lotus-storage-miner/market.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index dd99a375c..4dac236e4 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -140,13 +140,19 @@ var getAskCmd = &cli.Command{ Action: func(cctx *cli.Context) error { ctx := lcli.DaemonContext(cctx) - api, closer, err := lcli.GetStorageMinerAPI(cctx) + fnapi, closer, err := lcli.GetFullNodeAPI(cctx) if err != nil { return err } defer closer() - sask, err := api.MarketGetAsk(ctx) + smapi, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + sask, err := smapi.MarketGetAsk(ctx) if err != nil { return err } @@ -164,12 +170,12 @@ var getAskCmd = &cli.Command{ return w.Flush() } - miningBaseTs, err := api.MiningBase(ctx) + head, err := fnapi.ChainHead(ctx) if err != nil { return err } - dlt := ask.Expiry - miningBaseTs.Height() + dlt := ask.Expiry - head.Height() rem := "" if dlt > 0 { rem = (time.Second * time.Duration(dlt*build.BlockDelay)).String() From b91e7a98609d382a01cd00b457df2ec454dbd299 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 17 Jun 2020 20:00:30 +0200 Subject: [PATCH 158/160] Update to specs actors with ChargeGas interface Based on `lotus-0.6.1-chargegas` in specs-actors. Signed-off-by: Jakub Sztandera --- chain/vm/runtime.go | 14 +++++++++++--- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 48563df64..d6d49c214 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -253,7 +253,7 @@ func (rt *Runtime) CreateActor(codeID cid.Cid, address address.Address) { rt.Abortf(exitcode.SysErrorIllegalArgument, "Actor address already exists") } - rt.ChargeGas(rt.Pricelist().OnCreateActor()) + rt.chargeGas(rt.Pricelist().OnCreateActor()) err = rt.state.SetActor(address, &types.Actor{ Code: codeID, @@ -267,7 +267,7 @@ func (rt *Runtime) CreateActor(codeID cid.Cid, address address.Address) { } func (rt *Runtime) DeleteActor(addr address.Address) { - rt.ChargeGas(rt.Pricelist().OnDeleteActor()) + rt.chargeGas(rt.Pricelist().OnDeleteActor()) act, err := rt.state.GetActor(rt.Message().Receiver()) if err != nil { if xerrors.Is(err, types.ErrActorNotFound) { @@ -496,7 +496,15 @@ func (rt *Runtime) finilizeGasTracing() { } } -func (rt *Runtime) ChargeGas(gas GasCharge) { +// ChargeGas is spec actors function +func (rt *Runtime) ChargeGas(name string, compute int64, virtual int64) { + err := rt.chargeGasInternal(newGasCharge(name, compute, 0).WithVirtual(virtual, 0), 1) + if err != nil { + panic(err) + } +} + +func (rt *Runtime) chargeGas(gas GasCharge) { err := rt.chargeGasInternal(gas, 1) if err != nil { panic(err) diff --git a/go.mod b/go.mod index d5e684e46..c2471b82a 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/sector-storage v0.0.0-20200615192001-42c9e08595b7 - github.com/filecoin-project/specs-actors v0.6.1 + github.com/filecoin-project/specs-actors v0.6.2-0.20200617175406-de392ca14121 github.com/filecoin-project/specs-storage v0.1.0 github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index 53c989e62..14308aedb 100644 --- a/go.sum +++ b/go.sum @@ -261,6 +261,8 @@ github.com/filecoin-project/specs-actors v0.6.0 h1:IepUsmDGY60QliENVTkBTAkwqGWw9 github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-actors v0.6.1 h1:rhHlEzqcuuQU6oKc4csuq+/kQBDZ4EXtSomoN2XApCA= github.com/filecoin-project/specs-actors v0.6.1/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= +github.com/filecoin-project/specs-actors v0.6.2-0.20200617175406-de392ca14121 h1:oRA+b4iN4H86xXDXbU3TOyvmBZp7//c5VqTc0oJ6nLg= +github.com/filecoin-project/specs-actors v0.6.2-0.20200617175406-de392ca14121/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sgOVdJbvFjOnD5w94= github.com/filecoin-project/specs-storage v0.1.0/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743 h1:a8f1p6UdeD+ZINBKJN4FhEos8uaKeASOAabq5RCpQdg= From 5c5a3f22648bd410e82fc2fac3e37914454f9f26 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 17 Jun 2020 20:04:28 +0200 Subject: [PATCH 159/160] go mod tidy Signed-off-by: Jakub Sztandera --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 14308aedb..c48bc2071 100644 --- a/go.sum +++ b/go.sum @@ -259,8 +259,6 @@ github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.m github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= github.com/filecoin-project/specs-actors v0.6.0 h1:IepUsmDGY60QliENVTkBTAkwqGWw9kNbbHOcU/9oiC0= github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= -github.com/filecoin-project/specs-actors v0.6.1 h1:rhHlEzqcuuQU6oKc4csuq+/kQBDZ4EXtSomoN2XApCA= -github.com/filecoin-project/specs-actors v0.6.1/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-actors v0.6.2-0.20200617175406-de392ca14121 h1:oRA+b4iN4H86xXDXbU3TOyvmBZp7//c5VqTc0oJ6nLg= github.com/filecoin-project/specs-actors v0.6.2-0.20200617175406-de392ca14121/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sgOVdJbvFjOnD5w94= From 290512ee68c5f57904cb93034add64a3d2718fc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 17 Jun 2020 20:42:08 +0200 Subject: [PATCH 160/160] Update storage FSM with a bunch of fixes --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c2471b82a..dea622198 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/filecoin-project/sector-storage v0.0.0-20200615192001-42c9e08595b7 github.com/filecoin-project/specs-actors v0.6.2-0.20200617175406-de392ca14121 github.com/filecoin-project/specs-storage v0.1.0 - github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743 + github.com/filecoin-project/storage-fsm v0.0.0-20200617183754-4380106d3e94 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index c48bc2071..11848416b 100644 --- a/go.sum +++ b/go.sum @@ -263,8 +263,8 @@ github.com/filecoin-project/specs-actors v0.6.2-0.20200617175406-de392ca14121 h1 github.com/filecoin-project/specs-actors v0.6.2-0.20200617175406-de392ca14121/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sgOVdJbvFjOnD5w94= github.com/filecoin-project/specs-storage v0.1.0/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= -github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743 h1:a8f1p6UdeD+ZINBKJN4FhEos8uaKeASOAabq5RCpQdg= -github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743/go.mod h1:q1YCutTSMq/yGYvDPHReT37bPfDLHltnwJutzR9kOY0= +github.com/filecoin-project/storage-fsm v0.0.0-20200617183754-4380106d3e94 h1:zPKiZPMgkFF0Lq13hsk8lcWlxeVAs6vvJaa3uHn9v70= +github.com/filecoin-project/storage-fsm v0.0.0-20200617183754-4380106d3e94/go.mod h1:q1YCutTSMq/yGYvDPHReT37bPfDLHltnwJutzR9kOY0= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=