From ae83ca6893def6c113619c4ac60928812d8fdf33 Mon Sep 17 00:00:00 2001 From: Mike Greenberg Date: Thu, 4 Jun 2020 18:18:14 -0400 Subject: [PATCH 01/81] Express block validation, cpu/mem usage via OpenCensus --- chain/sync.go | 9 +++++++++ cmd/lotus/daemon.go | 8 ++++++++ metrics/metrics.go | 26 ++++++++++++++++---------- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index 23b790a06..681410740 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -528,8 +528,17 @@ func blockSanityChecks(h *types.BlockHeader) error { // Should match up with 'Semantical Validation' in validation.md in the spec func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) error { + validationStart := time.Now() + defer func() { + dur := time.Since(validationStart) + durMilli := dur.Seconds() * float64(1000) + stats.Record(ctx, metrics.BlockValidationDurationMilliseconds.M(durMilli)) + log.Infow("block validation", "took", dur, "height", b.Header.Height) + }() + ctx, span := trace.StartSpan(ctx, "validateBlock") defer span.End() + if build.InsecurePoStValidation { log.Warn("insecure test validation is enabled, if you see this outside of a test, it is a severe bug!") } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 51a1fd37f..4fcb20d53 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -18,6 +18,7 @@ import ( blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" + "go.opencensus.io/plugin/runmetrics" "go.opencensus.io/stats" "go.opencensus.io/stats/view" "go.opencensus.io/tag" @@ -114,6 +115,13 @@ var DaemonCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { + err := runmetrics.Enable(runmetrics.RunMetricOptions{ + EnableCPU: true, + EnableMemory: true, + }) + if err != nil { + return xerrors.Errorf("enabling runtime metrics: %w", err) + } if prof := cctx.String("pprof"); prof != "" { profile, err := os.Create(prof) if err != nil { diff --git a/metrics/metrics.go b/metrics/metrics.go index 340d57536..0ef63de4f 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -22,16 +22,17 @@ var ( // Measures var ( - LotusInfo = stats.Int64("info", "Arbitrary counter to tag lotus info to", stats.UnitDimensionless) - ChainNodeHeight = stats.Int64("chain/node_height", "Current Height of the node", stats.UnitDimensionless) - ChainNodeWorkerHeight = stats.Int64("chain/node_worker_height", "Current Height of workers on the node", stats.UnitDimensionless) - MessageReceived = stats.Int64("message/received", "Counter for total received messages", stats.UnitDimensionless) - MessageValidationFailure = stats.Int64("message/failure", "Counter for message validation failures", stats.UnitDimensionless) - MessageValidationSuccess = stats.Int64("message/success", "Counter for message validation successes", stats.UnitDimensionless) - BlockReceived = stats.Int64("block/received", "Counter for total received blocks", stats.UnitDimensionless) - BlockValidationFailure = stats.Int64("block/failure", "Counter for block validation failures", stats.UnitDimensionless) - BlockValidationSuccess = stats.Int64("block/success", "Counter for block validation successes", stats.UnitDimensionless) - PeerCount = stats.Int64("peer/count", "Current number of FIL peers", stats.UnitDimensionless) + LotusInfo = stats.Int64("info", "Arbitrary counter to tag lotus info to", stats.UnitDimensionless) + ChainNodeHeight = stats.Int64("chain/node_height", "Current Height of the node", stats.UnitDimensionless) + ChainNodeWorkerHeight = stats.Int64("chain/node_worker_height", "Current Height of workers on the node", stats.UnitDimensionless) + MessageReceived = stats.Int64("message/received", "Counter for total received messages", stats.UnitDimensionless) + MessageValidationFailure = stats.Int64("message/failure", "Counter for message validation failures", stats.UnitDimensionless) + MessageValidationSuccess = stats.Int64("message/success", "Counter for message validation successes", stats.UnitDimensionless) + BlockReceived = stats.Int64("block/received", "Counter for total received blocks", stats.UnitDimensionless) + BlockValidationFailure = stats.Int64("block/failure", "Counter for block validation failures", stats.UnitDimensionless) + BlockValidationSuccess = stats.Int64("block/success", "Counter for block validation successes", stats.UnitDimensionless) + BlockValidationDurationMilliseconds = stats.Float64("block/validation_ms", "Duration for Block Validation in ms", stats.UnitMilliseconds) + PeerCount = stats.Int64("peer/count", "Current number of FIL peers", stats.UnitDimensionless) ) var ( @@ -63,6 +64,10 @@ var ( Measure: BlockValidationSuccess, Aggregation: view.Count(), } + BlockValidationDurationView = &view.View{ + Measure: BlockValidationDurationMilliseconds, + Aggregation: view.Sum(), + } MessageReceivedView = &view.View{ Measure: MessageReceived, Aggregation: view.Count(), @@ -90,6 +95,7 @@ var DefaultViews = append([]*view.View{ BlockReceivedView, BlockValidationFailureView, BlockValidationSuccessView, + BlockValidationDurationView, MessageReceivedView, MessageValidationFailureView, MessageValidationSuccessView, From 751bc45befc9a7f227bc70fd2dcb80fb509de938 Mon Sep 17 00:00:00 2001 From: laser Date: Mon, 8 Jun 2020 12:20:44 -0700 Subject: [PATCH 02/81] display a little inline warning if balance is zero --- cli/wallet.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cli/wallet.go b/cli/wallet.go index f9f749d72..ccca4d23d 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -113,7 +113,12 @@ var walletBalance = &cli.Command{ return err } - fmt.Printf("%s\n", types.FIL(balance)) + if balance.Equals(types.NewInt(0)) { + fmt.Printf("%s (warning: may display 0 if chain sync in progress)\n", types.FIL(balance)) + } else { + fmt.Printf("%s\n", types.FIL(balance)) + } + return nil }, } From 6e11ff37962d101f5462a4bc5862b27112bc7ac6 Mon Sep 17 00:00:00 2001 From: Jim Pick Date: Mon, 8 Jun 2020 17:03:11 -0700 Subject: [PATCH 03/81] Make libp2p swarm Announce / NoAnnounce addresses configurable In go-ipfs, it is possible to manually configure addresses to announce or not announce to the swarm. https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#addressesannounce This PR adds the same feature to Lotus. It can be configured from the [libp2p] section of either the .lotus or .lotusstorage config.toml file. Example: [Libp2p] ListenAddresses = ["/ip4/0.0.0.0/tcp/34515", "/ip6/::/tcp/34515"] AnnounceAddresses = ["/ip4/52.13.91.110/tcp/34515"] --- documentation/en/setting-a-static-port.md | 14 ++++++++++++++ node/builder.go | 3 +++ node/config/def.go | 10 +++++++--- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/documentation/en/setting-a-static-port.md b/documentation/en/setting-a-static-port.md index 43a38daa9..714f455b0 100644 --- a/documentation/en/setting-a-static-port.md +++ b/documentation/en/setting-a-static-port.md @@ -15,6 +15,20 @@ To change the port to `1347`: After changing the port value, restart your **daemon**. +## Announce Addresses + +If the **swarm port** is port-forwarded from another address, it is possible to control what addresses +are announced to the network. + +```sh +[Libp2p] + AnnounceAddresses = ["/ip4//tcp/1347"] +``` + +If non-empty, this array specifies the swarm addresses to announce to the network. If empty, the daemon will announce inferred swarm addresses. + +Similarly, it is possible to set `NoAnnounceAddresses` with an array of addresses to not announce to the network. + ## Ubuntu's Uncomplicated Firewall Open firewall manually: diff --git a/node/builder.go b/node/builder.go index f503bff5c..35ef69bfe 100644 --- a/node/builder.go +++ b/node/builder.go @@ -371,6 +371,9 @@ func ConfigCommon(cfg *config.Common) Option { Override(new(dtypes.BootstrapPeers), modules.ConfigBootstrap(cfg.Libp2p.BootstrapPeers)), ), ), + Override(AddrsFactoryKey, lp2p.AddrsFactory( + cfg.Libp2p.AnnounceAddresses, + cfg.Libp2p.NoAnnounceAddresses)), ) } diff --git a/node/config/def.go b/node/config/def.go index 9ca97bb47..575284b95 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -39,9 +39,11 @@ type API struct { // Libp2p contains configs for libp2p type Libp2p struct { - ListenAddresses []string - BootstrapPeers []string - ProtectedPeers []string + ListenAddresses []string + AnnounceAddresses []string + NoAnnounceAddresses []string + BootstrapPeers []string + ProtectedPeers []string ConnMgrLow uint ConnMgrHigh uint @@ -78,6 +80,8 @@ func defCommon() Common { "/ip4/0.0.0.0/tcp/0", "/ip6/::/tcp/0", }, + AnnounceAddresses: []string{}, + NoAnnounceAddresses: []string{}, ConnMgrLow: 150, ConnMgrHigh: 180, From 093dc907b34af1e57ab44f835a57a84d967277da Mon Sep 17 00:00:00 2001 From: Rob Quist Date: Tue, 9 Jun 2020 20:57:21 +0200 Subject: [PATCH 04/81] Update documentation about Lotus Seal Worker Running the initial command `lotus-seal-worker run` gave an error about `--address` being a required parameter. After setting that to `127.0.0.1:2345` the miner started to freak out and sectors broke as the mining process locked up. It seems like in the mean time the sealing work has already been moved into the `lotus-storage-miner` and thus doesn't need to run alongside the miner. Changed the docs to what I think is correct now - but please check, I could be horribly wrong :) --- documentation/en/mining-lotus-seal-worker.md | 38 ++++---------------- 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/documentation/en/mining-lotus-seal-worker.md b/documentation/en/mining-lotus-seal-worker.md index 3e0240430..da87293a5 100644 --- a/documentation/en/mining-lotus-seal-worker.md +++ b/documentation/en/mining-lotus-seal-worker.md @@ -1,6 +1,6 @@ # Lotus Seal Worker -The **Lotus Seal Worker** is an extra process that can offload heavy processing tasks from your **Lotus Storage Miner**. It can be run on the same machine as your `lotus-storage-miner`, or on another machine communicating over a fast network. +The **Lotus Seal Worker** is an extra process that can offload heavy processing tasks from your **Lotus Storage Miner**. The sealing process automatically runs in the **Lotus Storage Miner** process, but you can use the Seal Worker on another machine communicating over a fast network to free up resources on the machine running the mining process. ## Note: Using the Lotus Seal Worker from China @@ -12,39 +12,13 @@ IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/" ## Get Started -Make sure that the `lotus-seal-worker` is installed by running: +Make sure that the `lotus-seal-worker` is compiled and installed by running: ```sh make lotus-seal-worker ``` -## Running Alongside Storage Miner - -You may wish to run the **Lotus Seal Worker** on the same computer as the **Lotus Storage Miner**. This allows you to easily set the process priority of the sealing tasks to be lower than the priority of your more important storage miner process. - -To do this, simply run `lotus-seal-worker run`, and the seal worker will automatically pick up the correct authentication tokens from the `LOTUS_STORAGE_PATH` miner repository. - -To check that the **Lotus Seal Worker** is properly connected to your storage miner, run `lotus-storage-miner info` and check that the remote worker count has increased. - -```sh -why@computer ~/lotus> lotus-storage-miner workers list -Worker 0, host computer - CPU: [ ] 0 core(s) in use - RAM: [|||||||||||||||||| ] 28% 18.1 GiB/62.7 GiB - VMEM: [|||||||||||||||||| ] 28% 18.1 GiB/62.7 GiB - GPU: GeForce RTX 2080, not used -Worker 1, host computer - CPU: [ ] 0 core(s) in use - RAM: [|||||||||||||||||| ] 28% 18.1 GiB/62.7 GiB - VMEM: [|||||||||||||||||| ] 28% 18.1 GiB/62.7 GiB - GPU: GeForce RTX 2080, not used -``` - -## Running Over the Network - -Warning: This setup is a little more complex than running it locally. - -To use an entirely separate computer for sealing tasks, you will want to run the `lotus-seal-worker` on a separate machine, connected to your **Lotus Storage Miner** via the local area network. +## Setting up the Storage Miner First, you will need to ensure your `lotus-storage-miner`'s API is accessible over the network. @@ -62,7 +36,7 @@ To make your node accessible over the local area network, you will need to deter A more permissive and less secure option is to change it to `0.0.0.0`. This will allow anyone who can connect to your computer on that port to access the [API](https://docs.lotu.sh/en+api). They will still need an auth token. -`RemoteListenAddress` must be set to an address which other nodes on your network will be able to reach +`RemoteListenAddress` must be set to an address which other nodes on your network will be able to reach. Next, you will need to [create an authentication token](https://docs.lotu.sh/en+api-scripting-support#generate-a-jwt-46). All Lotus APIs require authentication tokens to ensure your processes are as secure against attackers attempting to make unauthenticated requests to them. @@ -73,9 +47,11 @@ On the machine that will run `lotus-seal-worker`, set the `STORAGE_API_INFO` env Once this is set, run: ```sh -lotus-seal-worker run +lotus-seal-worker run --address 192.168.2.10:2345 ``` +Replace `192.168.2.10:2345` with the proper IP and port. + To check that the **Lotus Seal Worker** is connected to your **Lotus Storage Miner**, run `lotus-storage-miner workers list` and check that the remote worker count has increased. ```sh From 686c543778bc27d5ac10825eabb9956339b74e3f Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 9 Jun 2020 12:49:31 -0700 Subject: [PATCH 05/81] clean up some of the more spammy logs --- chain/store/store.go | 5 ----- chain/sub/incoming.go | 5 +---- chain/sync_manager.go | 2 +- chain/vm/runtime.go | 4 ++-- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index 94d94da79..dba5995de 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -18,7 +18,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/metrics" @@ -953,10 +952,6 @@ func (cs *ChainStore) GetTipsetByHeight(ctx context.Context, h abi.ChainEpoch, t return ts, nil } - if ts.Height()-h > build.ForkLengthThreshold { - log.Warnf("expensive call to GetTipsetByHeight, seeking %d levels", ts.Height()-h) - } - lbts, err := cs.cindex.GetTipsetByHeight(ctx, ts, h) if err != nil { return nil, err diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 2ef9c2718..ca498cfc5 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -61,8 +61,6 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha src := peer.ID(msg.GetFrom()) go func() { - log.Infof("New block over pubsub: %s", blk.Cid()) - start := time.Now() log.Debug("about to fetch messages for block from pubsub") bmsgs, err := s.Bsync.FetchMessagesByCids(context.TODO(), blk.BlsMessages) @@ -146,8 +144,7 @@ func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub // track validation time begin := time.Now() defer func() { - end := time.Now() - log.Infof("block validation time: %s", end.Sub(begin)) + log.Debugf("block validation time: %s", time.Since(begin)) }() stats.Record(ctx, metrics.BlockReceived.M(1)) diff --git a/chain/sync_manager.go b/chain/sync_manager.go index e00063961..deb626fda 100644 --- a/chain/sync_manager.go +++ b/chain/sync_manager.go @@ -280,7 +280,7 @@ func (sm *SyncManager) syncScheduler() { } func (sm *SyncManager) scheduleIncoming(ts *types.TipSet) { - log.Info("scheduling incoming tipset sync: ", ts.Cids()) + log.Debug("scheduling incoming tipset sync: ", ts.Cids()) if sm.getBootstrapState() == BSStateSelected { sm.setBootstrapState(BSStateScheduled) sm.syncTargets <- ts diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 948b0a021..006b19f46 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -139,7 +139,7 @@ func (rs *Runtime) shimCall(f func() interface{}) (rval []byte, aerr aerrors.Act defer func() { if r := recover(); r != nil { if ar, ok := r.(aerrors.ActorError); ok { - log.Errorf("VM.Call failure: %+v", ar) + log.Warnf("VM.Call failure: %+v", ar) aerr = ar return } @@ -309,7 +309,7 @@ func (rt *Runtime) Context() context.Context { } func (rs *Runtime) Abortf(code exitcode.ExitCode, msg string, args ...interface{}) { - log.Error("Abortf: ", fmt.Sprintf(msg, args...)) + log.Warnf("Abortf: ", fmt.Sprintf(msg, args...)) panic(aerrors.NewfSkip(2, code, msg, args...)) } From 2889efb608ca0b526e79c311e217f674c934c708 Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 9 Jun 2020 11:04:17 -0700 Subject: [PATCH 06/81] display quantities of bytes in human-readable format - client commP - client local - client find - client query-ask - client list-deals - state sector-size --- cli/chain.go | 3 +-- cli/client.go | 10 +++++----- cli/state.go | 5 ++--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/cli/chain.go b/cli/chain.go index 033c7b234..9d6856dd2 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -12,7 +12,6 @@ import ( "strings" "time" - "github.com/docker/go-units" "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/specs-actors/actors/abi" @@ -230,7 +229,7 @@ var chainStatObjCmd = &cli.Command{ } fmt.Printf("Links: %d\n", stats.Links) - fmt.Printf("Size: %s (%d)\n", units.BytesSize(float64(stats.Size)), stats.Size) + fmt.Printf("Size: %s (%d)\n", types.SizeStr(types.NewInt(stats.Size)), stats.Size) return nil }, } diff --git a/cli/client.go b/cli/client.go index 9787df37e..9d4479b62 100644 --- a/cli/client.go +++ b/cli/client.go @@ -143,7 +143,7 @@ var clientCommPCmd = &cli.Command{ } fmt.Println("CID: ", encoder.Encode(ret.Root)) - fmt.Println("Piece size: ", ret.Size) + fmt.Println("Piece size: ", types.SizeStr(types.NewInt(uint64(ret.Size)))) return nil }, } @@ -203,7 +203,7 @@ var clientLocalCmd = &cli.Command{ } for _, v := range list { - fmt.Printf("%s %s %d %s\n", encoder.Encode(v.Key), v.FilePath, v.Size, v.Status) + fmt.Printf("%s %s %s %s\n", encoder.Encode(v.Key), v.FilePath, types.SizeStr(types.NewInt(v.Size)), v.Status) } return nil }, @@ -371,7 +371,7 @@ var clientFindCmd = &cli.Command{ fmt.Printf("ERR %s@%s: %s\n", offer.Miner, offer.MinerPeerID, offer.Err) continue } - fmt.Printf("RETRIEVAL %s@%s-%sfil-%db\n", offer.Miner, offer.MinerPeerID, types.FIL(offer.MinPrice), offer.Size) + fmt.Printf("RETRIEVAL %s@%s-%sfil-%s\n", offer.Miner, offer.MinerPeerID, types.FIL(offer.MinPrice), types.SizeStr(types.NewInt(offer.Size))) } return nil @@ -520,7 +520,7 @@ var clientQueryAskCmd = &cli.Command{ fmt.Printf("Ask: %s\n", maddr) fmt.Printf("Price per GiB: %s\n", types.FIL(ask.Ask.Price)) - fmt.Printf("Max Piece size: %d\n", ask.Ask.MaxPieceSize) + fmt.Printf("Max Piece size: %s\n", types.SizeStr(types.NewInt(uint64(ask.Ask.MaxPieceSize)))) size := cctx.Int64("size") if size == 0 { @@ -597,7 +597,7 @@ var clientListDeals = &cli.Command{ slashed = fmt.Sprintf("Y (epoch %d)", d.OnChainDealState.SlashEpoch) } - fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s\t%d\t%s\n", d.LocalDeal.ProposalCid, d.LocalDeal.DealID, d.LocalDeal.Provider, storagemarket.DealStates[d.LocalDeal.State], onChain, slashed, d.LocalDeal.PieceCID, d.LocalDeal.Size, d.LocalDeal.PricePerEpoch, d.LocalDeal.Duration, d.LocalDeal.Message) + fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n", d.LocalDeal.ProposalCid, d.LocalDeal.DealID, d.LocalDeal.Provider, storagemarket.DealStates[d.LocalDeal.State], onChain, slashed, d.LocalDeal.PieceCID, types.SizeStr(types.NewInt(d.LocalDeal.Size)), d.LocalDeal.PricePerEpoch, d.LocalDeal.Duration, d.LocalDeal.Message) } return w.Flush() }, diff --git a/cli/state.go b/cli/state.go index 102c8d0bb..6b5c9035b 100644 --- a/cli/state.go +++ b/cli/state.go @@ -11,7 +11,6 @@ import ( "strings" "time" - "github.com/docker/go-units" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multihash" @@ -164,7 +163,7 @@ var stateMinerInfo = &cli.Command{ fmt.Printf("Owner:\t%s\n", mi.Owner) fmt.Printf("Worker:\t%s\n", mi.Worker) fmt.Printf("PeerID:\t%s\n", mi.PeerId) - fmt.Printf("SectorSize:\t%s (%d)\n", units.BytesSize(float64(mi.SectorSize)), mi.SectorSize) + fmt.Printf("SectorSize:\t%s (%d)\n", types.SizeStr(types.NewInt(uint64(mi.SectorSize))), mi.SectorSize) return nil }, @@ -718,7 +717,7 @@ var stateSectorSizeCmd = &cli.Command{ return err } - fmt.Printf("%d\n", mi.SectorSize) + fmt.Printf("%s (%d)\n", types.SizeStr(types.NewInt(uint64(mi.SectorSize))), mi.SectorSize) return nil }, } From 5cf76ba8b3737ce8d6c6ab1c2dd63d75b57cb517 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 8 Jun 2020 21:03:44 -0400 Subject: [PATCH 07/81] Create an api.MinerInfo that has peerID as a Peer ID --- api/api_full.go | 35 ++++++++++++++++++++++++++++++++++- api/apistruct/struct.go | 4 ++-- cli/state.go | 14 +------------- node/impl/full/state.go | 11 ++++++++--- storage/miner.go | 2 +- 5 files changed, 46 insertions(+), 20 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index dca416b94..746e2b166 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -146,7 +146,7 @@ type FullNode interface { StateMinerProvingSet(context.Context, address.Address, types.TipSetKey) ([]*ChainSectorInfo, error) StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error) StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error) - StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) + StateMinerInfo(context.Context, address.Address, types.TipSetKey) (MinerInfo, error) StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) (*miner.Deadlines, error) StateMinerFaults(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error) // Returns all non-expired Faults that occur within lookback epochs of the given tipset @@ -299,6 +299,39 @@ type MinerPower struct { TotalPower power.Claim } +type MinerInfo struct { + Owner address.Address // Must be an ID-address. + Worker address.Address // Must be an ID-address. + NewWorker address.Address // Must be an ID-address. + WorkerChangeEpoch abi.ChainEpoch + PeerId peer.ID + Multiaddrs []abi.Multiaddrs + SealProofType abi.RegisteredProof + SectorSize abi.SectorSize + WindowPoStPartitionSectors uint64 +} + +func NewApiMinerInfo(info miner.MinerInfo) MinerInfo { + mi := MinerInfo{ + Owner: info.Owner, + Worker: info.Worker, + NewWorker: address.Undef, + WorkerChangeEpoch: -1, + PeerId: peer.ID(info.PeerId), + Multiaddrs: info.Multiaddrs, + SealProofType: info.SealProofType, + SectorSize: info.SectorSize, + WindowPoStPartitionSectors: info.WindowPoStPartitionSectors, + } + + if info.PendingWorkerKey != nil { + mi.NewWorker = info.PendingWorkerKey.NewWorker + mi.WorkerChangeEpoch = info.PendingWorkerKey.EffectiveAt + } + + return mi +} + type QueryOffer struct { Err string diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 88f23d724..31cd6badd 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -125,7 +125,7 @@ type FullNodeStruct struct { StateMinerProvingSet func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"` StateMinerProvingDeadline func(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error) `perm:"read"` StateMinerPower func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) `perm:"read"` - StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) `perm:"read"` + StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) `perm:"read"` StateMinerDeadlines func(context.Context, address.Address, types.TipSetKey) (*miner.Deadlines, error) `perm:"read"` StateMinerFaults func(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error) `perm:"read"` StateAllMinerFaults func(context.Context, abi.ChainEpoch, types.TipSetKey) ([]*api.Fault, error) `perm:"read"` @@ -552,7 +552,7 @@ func (c *FullNodeStruct) StateMinerPower(ctx context.Context, a address.Address, return c.Internal.StateMinerPower(ctx, a, tsk) } -func (c *FullNodeStruct) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) { +func (c *FullNodeStruct) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (api.MinerInfo, error) { return c.Internal.StateMinerInfo(ctx, actor, tsk) } diff --git a/cli/state.go b/cli/state.go index c0cfd8eb4..6720ceacd 100644 --- a/cli/state.go +++ b/cli/state.go @@ -145,23 +145,11 @@ var stateMinerInfo = &cli.Command{ return err } - act, err := api.StateGetActor(ctx, addr, ts.Key()) + mi, err := api.StateMinerInfo(ctx, addr, ts.Key()) if err != nil { return err } - aso, err := api.ChainReadObj(ctx, act.Head) - if err != nil { - return err - } - - var mst miner2.State - if err := mst.UnmarshalCBOR(bytes.NewReader(aso)); err != nil { - return err - } - - mi := mst.Info - fmt.Printf("Owner:\t%s\n", mi.Owner) fmt.Printf("Worker:\t%s\n", mi.Worker) fmt.Printf("PeerID:\t%s\n", mi.PeerId) diff --git a/node/impl/full/state.go b/node/impl/full/state.go index c4a9f67fd..fae2a7b67 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -77,12 +77,17 @@ func (a *StateAPI) StateMinerProvingSet(ctx context.Context, addr address.Addres return stmgr.GetProvingSetRaw(ctx, a.StateManager, mas) } -func (a *StateAPI) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) { +func (a *StateAPI) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (api.MinerInfo, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { - return miner.MinerInfo{}, xerrors.Errorf("loading tipset %s: %w", tsk, err) + return api.MinerInfo{}, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return stmgr.StateMinerInfo(ctx, a.StateManager, ts, actor) + + mi, err := stmgr.StateMinerInfo(ctx, a.StateManager, ts, actor) + if err != nil { + return api.MinerInfo{}, err + } + return api.NewApiMinerInfo(mi), nil } func (a *StateAPI) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) (*miner.Deadlines, error) { diff --git a/storage/miner.go b/storage/miner.go index 117c91a57..171c166aa 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -51,7 +51,7 @@ type storageMinerApi interface { StateMinerSectors(context.Context, address.Address, *abi.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error) StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) - StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) + StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error) StateMinerInitialPledgeCollateral(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (types.BigInt, error) StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) // TODO: removeme eventually From 1ccad2222752aa7cf4b7ee6fcf520a5da32d64db Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 9 Jun 2020 19:08:43 -0400 Subject: [PATCH 08/81] Move api.MinerInfo to api/types --- api/api_full.go | 33 --------------------------------- api/types.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 746e2b166..63d9a5e5d 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -299,39 +299,6 @@ type MinerPower struct { TotalPower power.Claim } -type MinerInfo struct { - Owner address.Address // Must be an ID-address. - Worker address.Address // Must be an ID-address. - NewWorker address.Address // Must be an ID-address. - WorkerChangeEpoch abi.ChainEpoch - PeerId peer.ID - Multiaddrs []abi.Multiaddrs - SealProofType abi.RegisteredProof - SectorSize abi.SectorSize - WindowPoStPartitionSectors uint64 -} - -func NewApiMinerInfo(info miner.MinerInfo) MinerInfo { - mi := MinerInfo{ - Owner: info.Owner, - Worker: info.Worker, - NewWorker: address.Undef, - WorkerChangeEpoch: -1, - PeerId: peer.ID(info.PeerId), - Multiaddrs: info.Multiaddrs, - SealProofType: info.SealProofType, - SectorSize: info.SectorSize, - WindowPoStPartitionSectors: info.WindowPoStPartitionSectors, - } - - if info.PendingWorkerKey != nil { - mi.NewWorker = info.PendingWorkerKey.NewWorker - mi.WorkerChangeEpoch = info.PendingWorkerKey.EffectiveAt - } - - return mi -} - type QueryOffer struct { Err string diff --git a/api/types.go b/api/types.go index fcc013441..f5a595353 100644 --- a/api/types.go +++ b/api/types.go @@ -2,6 +2,9 @@ package api import ( "encoding/json" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/libp2p/go-libp2p-core/peer" ma "github.com/multiformats/go-multiaddr" @@ -39,3 +42,36 @@ type PubsubScore struct { ID peer.ID Score float64 } + +type MinerInfo struct { + Owner address.Address // Must be an ID-address. + Worker address.Address // Must be an ID-address. + NewWorker address.Address // Must be an ID-address. + WorkerChangeEpoch abi.ChainEpoch + PeerId peer.ID + Multiaddrs []abi.Multiaddrs + SealProofType abi.RegisteredProof + SectorSize abi.SectorSize + WindowPoStPartitionSectors uint64 +} + +func NewApiMinerInfo(info miner.MinerInfo) MinerInfo { + mi := MinerInfo{ + Owner: info.Owner, + Worker: info.Worker, + NewWorker: address.Undef, + WorkerChangeEpoch: -1, + PeerId: peer.ID(info.PeerId), + Multiaddrs: info.Multiaddrs, + SealProofType: info.SealProofType, + SectorSize: info.SectorSize, + WindowPoStPartitionSectors: info.WindowPoStPartitionSectors, + } + + if info.PendingWorkerKey != nil { + mi.NewWorker = info.PendingWorkerKey.NewWorker + mi.WorkerChangeEpoch = info.PendingWorkerKey.EffectiveAt + } + + return mi +} From 469eef823146e3e30790eeae12f0f1da18e24769 Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Tue, 9 Jun 2020 20:17:28 -0300 Subject: [PATCH 09/81] Lotus architecture notes (#1768) Co-authored-by: Aayush Rajasekaran --- chain/sync.go | 2 + chain/sync_manager.go | 1 + documentation/en/.library.json | 6 + documentation/en/architecture.md | 394 ++++++++++++++++++ .../en/dev/WIP-arch-complementary-notes.md | 153 +++++++ 5 files changed, 556 insertions(+) create mode 100644 documentation/en/architecture.md create mode 100644 documentation/en/dev/WIP-arch-complementary-notes.md diff --git a/chain/sync.go b/chain/sync.go index 681410740..07d470d28 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -429,6 +429,8 @@ func (syncer *Syncer) Sync(ctx context.Context, maybeHead *types.TipSet) error { return xerrors.Errorf("collectChain failed: %w", err) } + // At this point we have accepted and synced to the new `maybeHead` + // (`StageSyncComplete`). if err := syncer.store.PutTipSet(ctx, maybeHead); err != nil { span.AddAttributes(trace.StringAttribute("put_error", err.Error())) span.SetStatus(trace.Status{ diff --git a/chain/sync_manager.go b/chain/sync_manager.go index deb626fda..e3fbdf4e1 100644 --- a/chain/sync_manager.go +++ b/chain/sync_manager.go @@ -35,6 +35,7 @@ type SyncManager struct { syncStates []*SyncerState + // Normally this handler is set to `(*Syncer).Sync()`. doSync func(context.Context, *types.TipSet) error stop chan struct{} diff --git a/documentation/en/.library.json b/documentation/en/.library.json index 018e6189b..f637c813f 100644 --- a/documentation/en/.library.json +++ b/documentation/en/.library.json @@ -71,6 +71,12 @@ ] }, { + "title": "Architecture", + "slug": "en+arch", + "github": "en/architecture.md", + "value": null, + "posts": [] + }, { "title": "Storage Mining", "slug": "en+mining", "github": "en/mining.md", diff --git a/documentation/en/architecture.md b/documentation/en/architecture.md new file mode 100644 index 000000000..970f2a13d --- /dev/null +++ b/documentation/en/architecture.md @@ -0,0 +1,394 @@ +# Lotus + +Lotus is an implementation of the [Filecoin Distributed Storage Network](https://filecoin.io/). +A Lotus node syncs blockchains that follow the +Filecoin protocol, validating the blocks and state transitions. +The specification for the Filecoin protocol can be found [here](https://filecoin-project.github.io/specs/). + +For information on how to setup and operate a Lotus node, +please follow the instructions [here](https://lotu.sh/en+getting-started). + +# Components + +At a high level, a Lotus node comprises the following components: + +FIXME: No mention of block production here, cross-reference with schomatis's miner doc +- The Syncer, which manages the process of syncing the blockchain +- The State Manager, which can compute the state at any given point in the chain +- The Virtual Machine (VM), which executes messages +- The Repository, where all data is stored +- P2P stuff (FIXME missing libp2p listed under other PL dependencies)? allows hello, blocksync, retrieval, storage +- API / CLI (FIXME missing, in scratchpad) +- Other Filecoin dependencies (specs actors, proofs, storage, etc., FIXME missing) +- Is the Builder worth its own component? +- Other PL dependencies (IPFS, libp2p, IPLD? FIXME, missing) +- External libraries used by Lotus and other deps (FIXME, missing) + +# Preliminaries + +We discuss some key Filecoin concepts here, aiming to explain them by contrasting them with analogous concepts +in other well-known blockchains like Ethereum. We only provide brief descriptions here; elaboration +can be found in the [spec](https://filecoin-project.github.io/specs/). + +### Tipsets + +Unlike in Ethereum, a block can have multiple parents in Filecoin. We thus refer to the parent set of a block, +instead of a single parent. +A [tipset](https://filecoin-project.github.io/specs/#systems__filecoin_blockchain__struct__tipset) +is any set of blocks that share the same parent set. + +There is no concept of "block difficulty" in Filecoin. Instead, +the weight of a tipset is simply the number of blocks in the chain that ends in that tipset. Note that a longer chain +can have less weight than a shorter chain with more blocks per tipset. + +We also allow for "null" tipsets, which include zero blocks. This allows miners to "skip" a round, and build on top +of an imaginary empty tipset if they want to. + +We call the heaviest tipset in a chain the "head" of the chain. + +### Actors and Messages + +An [Actor](https://filecoin-project.github.io/specs/#systems__filecoin_vm__actor) + is analogous to a smart contract in Ethereum. Filecoin does not allow users to define their own +actors, but comes with several [builtin actors](https://github.com/filecoin-project/specs-actors), +which can be thought of as pre-compiled contracts. + +A [Message](https://filecoin-project.github.io/specs/#systems__filecoin_vm__message) +is analogous to transactions in Ethereum. + +# Sync + +Sync refers to the process by which a Lotus node synchronizes to the heaviest chain being advertised by its peers. +At a high-level, Lotus syncs in a manner similar to most other blockchains; a Lotus node listens to the various +chains its peers claim to be at, picks the heaviest one, requests the blocks in the chosen chain, +and validates each block in that chain, running all state transitions along the way. + +The majority of the sync functionality happens in the [`Syncer`](https://github.com/filecoin-project/lotus/blob/master/chain/sync.go), +internally managed by a [`SyncManager`](https://github.com/filecoin-project/lotus/blob/master/chain/sync_manager.go). + +We now discuss the various stages of the sync process. + +## Sync setup + +When a Lotus node connects to a new peer, we exchange the head of our chain +with the new peer through [the `hello` protocol](https://github.com/filecoin-project/lotus/blob/master/node/hello/hello.go). +If the peer's head is heavier than ours, we try to sync to it. Note +that we do NOT update our chain head at this stage. + +## Fetching and Persisting Block Headers + +Note: The API refers to these stages as `StageHeaders` and `StagePersistHeaders`. + +We proceed in the sync process by requesting block headers from the peer, +moving back from their head, until we reach a tipset that we have in common +(such a common tipset must exist, thought it may simply be the genesis block). +The functionality can be found in `Syncer::collectHeaders()`. + +If the common tipset is our head, we treat the sync as a "fast-forward", else we must +drop part of our chain to connect to the peer's head (referred to as "forking"). + +FIXME: This next para might be best replaced with a link to the validation doc +Some of the possible causes of failure in this stage include: + +- The chain is linked to a block that we have previously marked as bad, +and stored in a [`BadBlockCache`](https://github.com/filecoin-project/lotus/blob/master/chain/badtscache.go). +- The beacon entries in a block are inconsistent (FIXME: more details about what is validated here wouldn't be bad). +- Switching to this new chain would involve a chain reorganization beyond the allowed threshold (SPECK-CHECK). + +## Fetching and Validating Blocks + +Note: The API refers to this stage as `StageMessages`. + +Having acquired the headers and found a common tipset, we then move forward, requesting the full blocks, including the messages. + +For each block, we first confirm the syntactic validity of the block (SPECK-CHECK), +which includes the syntactic validity of messages included +in the block. +We then apply the messages, running all the state transitions, and compare the state root we calculate with the provided state root. + + +FIXME: This next para might be best replaced with a link to the validation doc +Some of the possible causes of failure in this stage include: + +- a block is syntactically invalid (including potentially containing syntactically invalid messages) +- the computed state root after applying the block doesn't match the block's state root +- FIXME: Check what's covered by syntactic validity, and add anything important that isn't (like proof validity, future checks, etc.) + +The core functionality can be found in `Syncer::ValidateTipset()`, with `Syncer::checkBlockMessages()` performing +syntactic validation of messages. + +## Setting the head + +Note: The API refers to this stage as `StageSyncComplete`. + +If all validations pass we will now set that head as our heaviest tipset in +[`ChainStore`](https://github.com/filecoin-project/lotus/blob/master/chain/store/store.go). +We already have the full state, since we calculated +it during the sync process. + +FIXME (aayush) I don't fuilly understand the next 2 paragraphs, but it seems important. Confirm and polish. +Relevant issue in IPFS: https://github.com/ipfs/ipfs-docs/issues/264 + +It is important to note at this point that similar to the IPFS architecture of addressing by content and not by location/address (FIXME: check and link to IPFS docs) the "actual" chain stored in the node repo is *relative* to which CID we look for. We always have stored a series of Filecoin blocks pointing to other blocks, each a potential chain in itself by following its parent's reference, and its parent's parent, and so on up to the genesis block. (FIXME: We need a diagram here, one of the Filecoin blog entries might have something similar to what we are describing here.) It only depends on *where* (location) do we start to look for. The *only* address/location reference we hold of the chain, a relative reference, is the `heaviest` pointer. This is reflected by the fact that we don't store it in the `Blockstore` by a fixed, *absolute*, CID that reflects its contents, as this will change each time we sync to a new head (FIXME: link to the immutability IPFS doc that I need to write). + +FIXME: Create a further reading appendix, move this next para to it, along with other +extraneous content +This is one of the few items we store in `Datastore` by key, location, allowing its contents to change on every sync. This is reflected in the `(*ChainStore) writeHead()` function (called by `takeHeaviestTipSet()` above) where we reference the pointer by the explicit `chainHeadKey` address (the string `"head"`, not a hash embedded in a CID), and similarly in `(*ChainStore).Load()` when we start the node and create the `ChainStore`. Compare this to a Filecoin block or message which are immutable, stored in the `Blockstore` by CID, once created they never change. + +## Keeping up with the chain + +A Lotus node also listens for new blocks broadcast by its peers over the `gossipsub` channel (see FIXME for more). +If we have validated such a block's parent tipset, and adding it to our tipset at its height would lead to a heavier +head, then we validate and add this block. The validation described is identical to that invoked during the sync +process (indeed, it's the same codepath). + +# State + +In Filecoin, the chain state at any given point is a collection of data stored under a root CID +encapsulated in the [`StateTree`](https://github.com/filecoin-project/lotus/blob/master/chain/state/statetree.go), +and accessed through the +[`StateManager`](https://github.com/filecoin-project/lotus/blob/master/chain/stmgr/stmgr.go). +The state at the chain's head is thus easily tracked and updated in a state root CID. +(FIXME: Talk about CIDs somewhere, we might want to explain some of the modify/flush/update-root mechanism here.)) + +## Calculating a Tipset State + +Recall that a tipset is a set of blocks that have identical parents (that is, that are built on top of the same tipset). +The genesis tipset comprises the genesis block(s), and has some state corresponding to it. + +The methods `TipSetState()` and `computeTipSetState()` in +[`StateManager`](https://github.com/filecoin-project/lotus/blob/master/chain/stmgr/stmgr.go) + are responsible for computing +the state that results from applying a tipset. This involves applying all the messages included +in the tipset, and performing implicit operations like awarding block rewards. + +Any valid block built on top of a tipset `ts` should have its Parent State Root equal to the result of +calculating the tipset state of `ts`. Note that this means that all blocks in a tipset must have the same Parent +State Root (which is to be expected, since they have the same parent tipset) + +### Preparing to apply a tipset + +When `StateManager::computeTipsetState()` is called with a tipset, `ts`, +it retrieves the parent state root of the blocks in `ts`. It also creates a list of `BlockMessages`, which wraps the BLS +and SecP messages in a block along with the miner that produced the block. + +Control then flows to `StateManager::ApplyBlocks()`, which builds a VM to apply the messages given to it. The VM +is initialized with the parent state root of the blocks in `ts`. We apply the blocks in `ts` in order (see FIXME for +ordering of blocks in a tipset). + +### Applying a block + +For each block, we prepare to apply the ordered messages (first BLS, then SecP). Before applying a message, we check if +we have already applied a message with that CID within the scope of this method. If so, we simply skip that message; +this is how duplicate messages included in the same tipset are skipped (with only the miner of the "first" block to +include the message getting the reward). For the actual process of message application, see FIXME (need an +internal link here), for now we +simply assume that the outcome of the VM applying a message is either an error, or a +[`MessageReceipt`](https://github.com/filecoin-project/lotus/blob/master/chain/types/message_receipt.go) + and some +other information. + +We treat an error from the VM as a showstopper; there is no recovery, and no meaningful state can be computed for `ts`. +Given a successful receipt, we add the rewards and penalties to what the miner has earned so far. Once all the messages +included in a block have been applied (or skipped if they're a duplicate), we use an implicit message to call +the Reward Actor. This awards the miner their reward for having won a block, and also awards / penalizes them based +on the message rewards and penalties we tracked. + +We then proceed to apply the next block in `ts`, using the same VM. This means that the state changes that result +from applying a message are visible when applying all subsequent messages, even if they are included in a different block. + +### Finishing up + +Having applied all the blocks, we send one more implicit message, to the Cron Actor, which handles operations that +must be performed at the end of every epoch (see FIXME for more). The resulting state after calling the Cron Actor +is the computed state of the tipset. + +# Virtual Machine + +The Virtual Machine (VM) is responsible for executing messages. +The [Lotus Virtual Machine](https://github.com/filecoin-project/lotus/blob/master/chain/vm/vm.go) +invokes the appropriate methods in the builtin actors, and provides +a [`Runtime`](https://github.com/filecoin-project/specs-actors/blob/master/actors/runtime/runtime.go) +interface to the [builtin actors](https://github.com/filecoin-project/specs-actors) +that exposes their state, allows them to take certain actions, and meters +their gas usage. The VM also performs balance transfers, creates new account actors as needed, and tracks the gas reward, +penalty, return value, and exit code. + +## Applying a Message + +The primary entrypoint of the VM is the `ApplyMessage()` method. This method should not return an error +unless something goes unrecoverably wrong. + +The first thing this method does is assess if the message provided meets any of the penalty criteria. +If so, a penalty is issued, and the method returns. Next, the entire gas cost of the message is transferred to +a temporary gas holder account. It is from this gas holder that gas will be deducted; if it runs out of gas, the message +fails. Any unused gas in this holder will be refunded to the message's sender at the end of message execution. + +The VM then increments the sender's nonce, takes a snapshot of the state, and invokes `VM::send()`. + +The `send()` method creates a [`Runtime`](https://github.com/filecoin-project/lotus/blob/master/chain/vm/runtime.go) + for the subsequent message execution. +It then transfers the message's value to the recipient, creating a new account actor if needed. + +### Method Invocation + +We use reflection to translate a Filecoin message for the VM to an actual Go function, relying on the VM's +[`invoker`](https://github.com/filecoin-project/lotus/blob/master/chain/vm/invoker.go) structure. +Each actor has its own set of codes defined in `specs-actors/actors/builtin/methods.go`. +The `invoker` structure maps the builtin actors' CIDs + to a list of `invokeFunc` (one per exported method), which each take the `Runtime` (for state manipulation) + and the serialized input parameters. + +FIXME (aayush) Polish this next para. + +The basic layout (without reflection details) of `(*invoker).transform()` is as follows. From each actor registered in `NewInvoker()` we take its `Exports()` methods converting them to `invokeFunc`s. The actual method is wrapped in another function that takes care of decoding the serialized parameters and the runtime, this function is passed to `shimCall()` that will encapsulate the actors code being run inside a `defer` function to `recover()` from panics (we fail in the actors code with panics to unwrap the stack). The return values will then be (CBOR) marshaled and returned to the VM. + +### Returning from the VM + +Once method invocation is complete (including any subcalls), we return to `ApplyMessage()`, which receives +the serialized response and the [`ActorError`](https://github.com/filecoin-project/lotus/blob/master/chain/actors/aerrors/error.go). +The sender will be charged the appropriate amount of gas for the returned response, which gets put into the +[`MessageReceipt`](https://github.com/filecoin-project/lotus/blob/master/chain/types/message_receipt.go). + +The method then refunds any unused gas to the sender, sets up the gas reward for the miner, and +wraps all of this into an `ApplyRet`, which is returned. + +# Building a Lotus node + +When we launch a Lotus node with the command `./lotus daemon` +(see [here](https://github.com/filecoin-project/lotus/blob/master/cmd/lotus/daemon.go) for more), +the node is created through [dependency injection](https://godoc.org/go.uber.org/fx). +This relies on reflection, which makes some of the references hard to follow. +The node sets up all of the subsystems it needs to run, such as the repository, the network connections, thechain sync +service, etc. +This setup is orchestrated through calls to the `node.Override` function. +The structure of each call indicates the type of component it will set up +(many defined in [`node/modules/dtypes/`](https://github.com/filecoin-project/lotus/tree/master/node/modules/dtypes)), +and the function that will provide it. +The dependency is implicit in the argument of the provider function. + +As an example, consider the `modules.ChainStore()` function that provides the +[`ChainStore`](https://github.com/filecoin-project/lotus/blob/master/chain/store/store.go) structure. +It takes as one of its parameters the [`ChainBlockstore`](https://github.com/filecoin-project/lotus/blob/master/node/modules/dtypes/storage.go) +type, which becomes one of its dependencies. +For the node to be built successfully the `ChainBlockstore` will need to be provided before `ChainStore`, a requirement +that is made explicit in another `Override()` call that sets the provider of that type as the `ChainBlockstore()` function. + +## The Repository + +The repo is the directory where all of a node's information is stored. The node is entirely defined by its repo, which +makes it easy to port to another location. This one-to-one relationship means we can speak +of the node as the repo it is associated with, instead of the daemon process that runs from that repo. + +Only one daemon can run be running with an associated repo at a time. +A process signals that it is running a node associated with a particular repo, by creating and acquiring +a `repo.lock`. + +```sh +lsof ~/.lotus/repo.lock +# COMMAND PID +# lotus 52356 +``` +Trying to launch a second daemon hooked to the same repo leads to a `repo is already locked (lotus daemon already running)` +error. + +The `node.Repo()` function (`node/builder.go`) contains most of the dependencies (specified as `Override()` calls) +needed to properly set up the node's repo. We list the most salient ones here. + +### Datastore + +`Datastore` and `ChainBlockstore`: Data related to the node state is saved in the repo's `Datastore`, +an IPFS interface defined [here](github.com/ipfs/go-datastore/datastore.go). +Lotus creates this interface from a [Badger DB](https://github.com/dgraph-io/badger) in + [`FsRepo`](https://github.com/filecoin-project/lotus/blob/master/node/repo/fsrepo.go). +Every piece of data is fundamentally a key-value pair in the `datastore` directory of the repo. +There are several abstractions laid on top of it that appear through the code depending on *how* we access it, +but it is important to remember that we're always accessing it from the same place. + +FIXME: Maybe mention the `Batching` interface as the developer will stumble upon it before reaching the `Datastore` one. + +#### Blocks + +FIXME: IPFS blocks vs Filecoin blocks ideally happens before this / here + +The [`Blockstore` interface](`github.com/ipfs/go-ipfs-blockstore/blockstore.go`) structures the key-value pair +into the CID format for the key and the [`Block` interface](`github.com/ipfs/go-block-format/blocks.go`) for the value. +The `Block` value is just a raw string of bytes addressed by its hash, which is included in the CID key. + +`ChainBlockstore` creates a `Blockstore` in the repo under the `/blocks` namespace. +Every key stored there will have the `blocks` prefix so that it does not collide with other stores that use the same repo. + +FIXME: Link to IPFS documentation about DAG, CID, and related, especially we need a diagram that shows how do we wrap each datastore inside the next layer (datastore, batching, block store, gc, etc). + +#### Metadata + +`modules.Datastore()` creates a `dtypes.MetadataDS`, which is an alias for the basic `Datastore` interface. +Metadata is stored here under the `/metadata` prefix. +(FIXME: Explain *what* is metadata in contrast with the block store, namely we store the pointer to the heaviest chain, we might just link to that unwritten section here later.) + +FIXME: Explain the key store related calls (maybe remove, per Schomatis) + +### LockedRepo + +`LockedRepo()`: This method doesn't create or initialize any new structures, but rather registers an + `OnStop` [hook](https://godoc.org/go.uber.org/fx/internal/lifecycle#Hook) + that will close the locked repository associated with it on shutdown. + + +### Repo types / Node types + +FIXME: This section needs to be clarified / corrected...I don't fully understand the config differences (what do they have in common, if anything?) + +At the end of the `Repo()` function we see two mutually exclusive configuration calls based on the `RepoType` (`node/repo/fsrepo.go`). +```Go + ApplyIf(isType(repo.FullNode), ConfigFullNode(c)), + ApplyIf(isType(repo.StorageMiner), ConfigStorageMiner(c)), +``` +As we said, the repo fully identifies the node so a repo type is also a *node* type, in this case a full node or a storage miner. (FIXME: What is the difference between the two, does *full* imply miner?) In this case the `daemon` command will create a `FullNode`, this is specified in the command logic itself in `main.DaemonCmd()`, the `FsRepo` created (and passed to `node.Repo()`) will be initiated with that type (see `(*FsRepo).Init(t RepoType)`). + +## Online + +FIXME: Much of this might need to be subsumed into the p2p section + +The `node.Online()` configuration function (`node/builder.go`) initializes components that involve connecting to, +or interacting with, the Filecoin network. These connections are managed through the libp2p stack (FIXME link to this section when it exists). +We discuss some of the components found in the full node type (that is, included in the `ApplyIf(isType(repo.FullNode),` call). + +#### Chainstore + +`modules.ChainStore()` creates the [`store.ChainStore`](https://github.com/filecoin-project/lotus/blob/master/chain/store/store.go)) +that wraps the stores + previously instantiated in `Repo()`. It is the main point of entry for the node to all chain-related data + (FIXME: this is incorrect, we sometimes access its underlying block store directly, and probably shouldn't). + It also holds the crucial `heaviest` pointer, which indicates the current head of the chain. + + #### ChainExchange and ChainBlockservice +`ChainExchange()` and `ChainBlockservice()` establish a BitSwap connection (FIXME libp2p link) +to exchange chain information in the form of `blocks.Block`s stored in the repo. (See sync section for more details, the Filecoin blocks and messages are backed by these raw IPFS blocks that together form the different structures that define the state of the current/heaviest chain.) + +#### Incoming handlers +`HandleIncomingBlocks()` and `HandleIncomingMessages()` start the services in charge of processing new Filecoin blocks +and messages from the network (see `` for more information about the topics the node is subscribed to, FIXME: should that be part of the libp2p section or should we expand on gossipsub separately?). + +#### Hello +`RunHello()`: starts the services to both send (`(*Service).SayHello()`) and receive (`(*Service).HandleStream()`, `node/hello/hello.go`) +`hello` messages. When nodes establish a new connection with each other, they exchange these messages +to share chain-related information (namely their genesis block and their heaviest tipset). + +#### Syncer +`NewSyncer()` creates the `Syncer` structure and starts the services related to the chain sync process (FIXME link). + +### Ordering the dependencies + +We can establish the dependency relations by looking at the parameters that each function needs and by understanding +the architecture of the node and how the different components relate to each other (the chief purpose of this document). + +As an example, the sync mechanism depends on the node being able to exchange different IPFS blocks with the network, +so as to be able to request the "missing pieces" needed to construct the chain. This dependency is reflected by `NewSyncer()` +having a `blocksync.BlockSync` parameter, which in turn depends on `ChainBlockservice()` and `ChainExchange()`. +The chain exchange service further depends on the chain store to save and retrieve chain data, which is reflected +in `ChainExchange()` having `ChainGCBlockstore` as a parameter (which is just a wrapper around `ChainBlockstore` capable + of garbage collection). + +This block store is the same store underlying the chain store, which is an indirect dependency of `NewSyncer()` (through the `StateManager`). +(FIXME: This last line is flaky, we need to resolve the hierarchy better, we sometimes refer to the chain store and sometimes to its underlying block store. We need a diagram to visualize all the different components just mentioned otherwise it is too hard to follow. We probably even need to skip some of the connections mentioned.) \ No newline at end of file diff --git a/documentation/en/dev/WIP-arch-complementary-notes.md b/documentation/en/dev/WIP-arch-complementary-notes.md new file mode 100644 index 000000000..00bedb56a --- /dev/null +++ b/documentation/en/dev/WIP-arch-complementary-notes.md @@ -0,0 +1,153 @@ +# Genesis block + +Seems a good way to start exploring the VM state though the instantiation of its different actors like the storage power. + +Explain where do we load the genesis block, the CAR entries, and we set the root of the state. Follow the daemon command option, `chain.LoadGenesis()` saves all the blocks of the CAR file into the store provided by `ChainBlockstore` (this should already be explained in the previous section). The CAR root (MT root?) of those blocks is decoded into the `BlockHeader` that will be the Filecoin (genesis) block of the chain, but most of the information was stored in the raw data (non-Filecoin, what's the correct term?) blocks forwarded directly to the chain, the block header just has a pointer to it. + +`SetGenesis` block with name 0. `(ChainStore).SetGenesis()` stores it there. + +`MakeInitialStateTree` (`chain/gen/genesis/genesis.go`, used to construct the genesis block (`MakeGenesisBlock()`), constructs the state tree (`NewStateTree`) which is just a "pointer" (root node in the HAMT) to the different actors. It will be continuously used in `(*StateTree).SetActor()` an `types.Actor` structure under a certain `Address` (in the HAMT). (How does the `stateSnaps` work? It has no comments.) + +From this point we can follow different setup function like: + +* `SetupInitActor()`: see the `AddressMap`. + +* `SetupStoragePowerActor`: initial (zero) power state of the chain, most important attributes. + +* Account actors in the `template.Accounts`: `SetActor`. + +Which other actor type could be helpful at this point? + +# Basic concepts + +What should be clear at this point either from this document or the spec. + +## Addresses + +## Accounts + +# Sync Topics PubSub + +Gossip sub spec and some introduction. + +# Look at the constructor of a miner + +Follow the `lotus-storage-miner` command to see how a miner is created, from the command to the message to the storage power logic. + +# Directory structure so far, main structures seen, their relation + +List what are the main directories we should be looking at (e.g., `chain/`) and the most important structures (e.g., `StateTree`, `Runtime`, etc.) + +# Tests + +Run a few messages and observe state changes. What is the easiest test that also let's us "interact" with it (modify something and observe the difference). + +### Filecoin blocks vs IPFS blocks + +The term *block* has different meanings depending on the context, many times both meanings coexist at once in the code and it is important to distinguish them. (FIXME: link to IPFS blocks and related doc throughout this explanation). In terms of the lower IPFS layer, in charge of storing and retrieving data, both present at the repo or accessible through the network (e.g., through the BitSwap protocol discussed later), a block is a string of raw bytes identified by its hash, embedded and fully qualified in a CID identifier. IPFS blocks are the "building blocks" of almost any other piece of (chain) data described in the Filecoin protocol. + +In contrast, in the higher Filecoin (application) layer, a block is roughly (FIXME: link to spec definition, if we have any) a set of zero or more messages grouped together by a single miner which is itself grouped with other blocks (from other miners) in the same round to form a tipset. The Filecoin blockchain is a series of "chained" tipsets, each referencing its parent by its header's *CID*, that is, its header as seen as a single IPFS block, this is where both layers interact. + +Using now the full Go package qualifiers to avoid any ambiguity, the Filecoin block, `github.com/filecoin-project/lotus/chain/types.FullBlock`, is defined as, + +```Go +package types + +import "github.com/ipfs/go-cid" + +type FullBlock struct { + Header *BlockHeader + BlsMessages []*Message + SecpkMessages []*SignedMessage +} + +func (fb *FullBlock) Cid() cid.Cid { + return fb.Header.Cid() +} +``` + +It has, besides the Filecoin messages, a header with protocol related information (e.g., its `Height`) which is (like virtually any other piece of data in the Filecoin protocol) stored, retrieved and shared as an IPFS block with its corresponding CID, + +```Go +func (b *BlockHeader) Cid() cid.Cid { + sb, err := b.ToStorageBlock() + + return sb.Cid() +} + +func (b *BlockHeader) ToStorageBlock() (block.Block, error) { + data, err := b.Serialize() + + return github.com/ipfs/go-block-format.block.NewBlockWithCid(data) +} +``` + +These edited extracts from the `BlockHeader` show how it's treated as an IPFS block, `github.com/ipfs/go-block-format.block.BasicBlock`, to be both stored and referenced by its block storage CID. + +This duality permeates the code (and the Filecoin spec for that matter) but it is usually clear within the context to which block we are referring to. Normally the unqualified *block* is reserved for the Filecoin block and we won't usually refer to the IPFS one but only implicitly through the concept of its CID. With enough understanding of both stack's architecture the two definitions can coexist without much confusion as we will abstract away the IPFS layer and just use the CID as an identifier that we now its unique for two sequences of different *raw* byte strings. + +(FIXME: We use to do this presentation when talking about `gossipsub` topics and incoming blocks, and had to deal with, besides the block ambiguity, a similar confusion with the *message* term, used in libp2p to name anything that comes through the network, needing to present the extremely confusing hierarchy of a libp2p message containing a Filecoin block, identified by a IPFS block CID, containing Filecoin messages.) + +FIXME: Move the following tipset definition to sync or wherever is most needed, to avoid making this more confusing. + +Messages from the same round are collected into a block set (`chain/store/fts.go`): + +```Go +type FullTipSet struct { + Blocks []*types.FullBlock + tipset *types.TipSet + cids []cid.Cid +} +``` + +The "tipset" denomination might be a bit misleading as it doesn't refer *only* to the tip, the block set from the last round in the chain, but to *any* set of blocks, depending on the context the tipset is the actual tip or not. From its own perspective any block set is always the tip because it assumes nothing from following blocks. + +# CLI, API + +Explain how do we communicate with the node, both in terms of the CLI and the programmatic way (to create our own tools). + +## Client/server architecture + +In terms of the Filecoin network the node is a peer on a distributed hierarchy, but in terms of how we interact with the node we have client/server architecture. + +The node itself was initiated with the `daemon` command, it already started syncing to the chain by default. Along with that service it also started a [JSON-RPC](https://en.wikipedia.org/wiki/JSON-RPC) server to allow a client to interact with it. (FIXME: Check if this client is local or can be remote, link to external documentation of connection API.) + +We can connect to this server through the Lotus CLI. Virtually any other command other than `daemon` will run a client that will connect (by default) to the address specified in the `api` file in the repo associated with the node (by default in `~/.lotus`), e.g., + +```sh +cat ~/.lotus/api && echo +# /ip4/127.0.0.1/tcp/1234/http + +# With `lotus daemon` running in another terminal. +nc -v -z 127.0.0.1 1234 + +# Start daemon and turn off the logs to not clutter the command line. +bash -c "lotus daemon &" && + lotus wait-api && + lotus log set-level error # Or a env.var in the daemon command. + +nc -v -z 127.0.0.1 1234 +# Connection to 127.0.0.1 1234 port [tcp/*] succeeded! + +killall lotus +# FIXME: We need a lotus stop command: +# https://github.com/filecoin-project/lotus/issues/1827 +``` + +FIXME: Link to more in-depth documentation of the CLI architecture, maybe some IPFS documentation (since they share some common logic). + +## Node API + +The JSON-RPC server exposes the node API, the `FullNode` interface (defined in `api/api_full.go`). When we issue a command like `lotus sync status` to query the progress of the node sync we don't access the node's internals, those are decoupled in a separate daemon process, we call the `SyncState` function (of the `FullNode` API interface) through the RPC client started by our own command (see `NewFullNodeRPC` in `api/client/client.go` for more details). + +FIXME: Link to (and create) documentation about API fulfillment. + +Because we rely heavily on reflection for this part of the code the call chain is not easily visible by just following the references through the symbolic analysis of the IDE. If we start by the `lotus sync` command definition (in `cli/sync.go`), we eventually end up in the method interface `SyncState`, and when we look for its implementation we will find two functions: + +* `(*SyncAPI).SyncState()` (in `node/impl/full/sync.go`): this is the actual implementation of the API function that shows what the node (here acting as the RPC server) will execute when it receives the RPC request issued from the CLI acting as the client. + +* `(*FullNodeStruct).SyncState()`: this is an "empty placeholder" structure that will get later connected to the JSON-RPC client logic (see `NewMergeClient` in `lib/jsonrpc/client.go`, which is called by `NewFullNodeRPC`). (FIXME: check if this is accurate). The CLI (JSON-RPC client) will actually execute this function which will connect to the server and send the corresponding JSON request that will trigger the call of `(*SyncAPI).SyncState()` with the node implementation. + +This means that when we are tracking the logic of a CLI command we will eventually find this bifurcation and need to study the code of the server-side implementation in `node/impl/full` (mostly in the `common/` and `full/` directories). If we understand this architecture going directly to that part of the code abstracts away the JSON-RPC client/server logic and we can think that the CLI is actually running the node's logic. + +FIXME: Explain that "*the* node" is actually an API structure like `impl.FullNodeAPI` with the different API subcomponents like `full.SyncAPI`. We won't see a *single* node structure, each API (full node, minder, etc) will gather the necessary subcomponents it needs to service its calls. \ No newline at end of file From 6d90f12bb689e1c095452014c337ad50e5fd013e Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 9 Jun 2020 18:19:33 -0400 Subject: [PATCH 10/81] Doc fix: use correct file name --- documentation/en/.library.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/en/.library.json b/documentation/en/.library.json index f637c813f..545eb4a5b 100644 --- a/documentation/en/.library.json +++ b/documentation/en/.library.json @@ -58,8 +58,8 @@ }, { "title": "Use Lotus with systemd", - "slug": "en+install-system-services", - "github": "en/install-system-services.md", + "slug": "en+install-systemd-services", + "github": "en/install-systemd-services.md", "value": null }, { From 6d06bfb452a6be9393340fd6904a4f4057efc59c Mon Sep 17 00:00:00 2001 From: RobQuistNL Date: Wed, 10 Jun 2020 11:41:58 +0200 Subject: [PATCH 11/81] Update documentation --- documentation/en/hardware-mining.md | 2 ++ documentation/en/mining-lotus-seal-worker.md | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/documentation/en/hardware-mining.md b/documentation/en/hardware-mining.md index 16bffaf9c..5578f7d62 100644 --- a/documentation/en/hardware-mining.md +++ b/documentation/en/hardware-mining.md @@ -16,6 +16,8 @@ The setup below is a minimal example for sealing 32 GiB sectors on Lotus: Note that 1GB sectors don't require as high of specs, but are likely to be removed as we improve the performance of 32GB sector sealing. +AMD CPU's are **highly recommended**, because of the `Intel SHA Extensions` instruction set that is available there since the `Zen` microarchitecture. Hence, AMD CPU's seem to perform much better on the testnet than other CPU's. Contrary to what the name implies, this extended instruction set is not available on recent Intel desktop/server chips. + ## Testnet discoveries - If you only have 128GiB of ram, enabling 256GB of **NVMe** swap on an SSD will help you avoid out-of-memory issues while mining. diff --git a/documentation/en/mining-lotus-seal-worker.md b/documentation/en/mining-lotus-seal-worker.md index da87293a5..6cd7b2c7a 100644 --- a/documentation/en/mining-lotus-seal-worker.md +++ b/documentation/en/mining-lotus-seal-worker.md @@ -68,3 +68,14 @@ Worker 1, host othercomputer VMEM: [|||||||||||||| ] 23% 14 GiB/62.7 GiB GPU: GeForce RTX 2080, not used ``` + +### Running locally for manually managing process priority + +You can also run the **Lotus Seal Worker** on the same machine as your **Lotus Storage Miner**, so you can manually manage the process priority. +To do so you have to first **disable all seal task types** in the miner config. This is important to prevent conflicts between the two processes. + +You can then run the storage miner on your local-loopback interface; + +```sh +lotus-seal-worker run --address 127.0.0.1:2345 +``` \ No newline at end of file From af4c4d4f1f96a7c4dba01522a50222632856a476 Mon Sep 17 00:00:00 2001 From: RobQuistNL Date: Wed, 10 Jun 2020 11:44:35 +0200 Subject: [PATCH 12/81] Prevent words from being marked as glossary links --- documentation/en/hardware-mining.md | 2 +- documentation/en/mining-lotus-seal-worker.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/en/hardware-mining.md b/documentation/en/hardware-mining.md index 5578f7d62..a7c410184 100644 --- a/documentation/en/hardware-mining.md +++ b/documentation/en/hardware-mining.md @@ -16,7 +16,7 @@ The setup below is a minimal example for sealing 32 GiB sectors on Lotus: Note that 1GB sectors don't require as high of specs, but are likely to be removed as we improve the performance of 32GB sector sealing. -AMD CPU's are **highly recommended**, because of the `Intel SHA Extensions` instruction set that is available there since the `Zen` microarchitecture. Hence, AMD CPU's seem to perform much better on the testnet than other CPU's. Contrary to what the name implies, this extended instruction set is not available on recent Intel desktop/server chips. +AMD CPU's are __highly recommended__, because of the `Intel SHA Extensions` instruction set that is available there since the `Zen` microarchitecture. Hence, AMD CPU's seem to perform much better on the testnet than other CPU's. Contrary to what the name implies, this extended instruction set is not available on recent Intel desktop/server chips. ## Testnet discoveries diff --git a/documentation/en/mining-lotus-seal-worker.md b/documentation/en/mining-lotus-seal-worker.md index 6cd7b2c7a..aba115661 100644 --- a/documentation/en/mining-lotus-seal-worker.md +++ b/documentation/en/mining-lotus-seal-worker.md @@ -72,7 +72,7 @@ Worker 1, host othercomputer ### Running locally for manually managing process priority You can also run the **Lotus Seal Worker** on the same machine as your **Lotus Storage Miner**, so you can manually manage the process priority. -To do so you have to first **disable all seal task types** in the miner config. This is important to prevent conflicts between the two processes. +To do so you have to first __disable all seal task types__ in the miner config. This is important to prevent conflicts between the two processes. You can then run the storage miner on your local-loopback interface; From e2d2af3afadb16756347541b7389a229a5f960bc Mon Sep 17 00:00:00 2001 From: waynewyang Date: Wed, 10 Jun 2020 20:34:04 +0800 Subject: [PATCH 13/81] error check --- cli/chain.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cli/chain.go b/cli/chain.go index 9d6856dd2..0bf7741cb 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -883,6 +883,9 @@ var slashConsensusFault = &cli.Command{ BlockHeader1: bh1, BlockHeader2: bh2, }) + if err != nil { + return err + } if cctx.String("miner") == "" { return xerrors.Errorf("--miner flag is required") From 68959582b9b78cbd4973e3b5284b8e85c0e0eb1b Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 11 Jun 2020 00:09:48 +0200 Subject: [PATCH 14/81] docs: add a note about open files limit --- documentation/en/join-testnet.md | 2 ++ documentation/en/setup-troubleshooting.md | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/documentation/en/join-testnet.md b/documentation/en/join-testnet.md index bdd1b4a56..d01d7eafa 100644 --- a/documentation/en/join-testnet.md +++ b/documentation/en/join-testnet.md @@ -28,6 +28,8 @@ lotus net peers | wc -l In order to connect to the network, you need to be connected to at least 1 peer. If you’re seeing 0 peers, read our [troubleshooting notes](https://docs.lotu.sh/en+setup-troubleshooting). +Make sure that you have a reasonable "open files limit" set on your machine, such as 10000. If you're seeing a lower value, such as 256 (default on macOS), read our [troubleshooting notes](https://docs.lotu.sh/en+setup-troubleshooting) on how to update it prior to starting the Lotus daemon. + ## Chain sync While the daemon is running, the next requirement is to sync the chain. Run the command below to view the chain sync progress. To see current chain height, visit the [network stats page](https://stats.testnet.filecoin.io/). diff --git a/documentation/en/setup-troubleshooting.md b/documentation/en/setup-troubleshooting.md index 055a9c501..8a23544d9 100644 --- a/documentation/en/setup-troubleshooting.md +++ b/documentation/en/setup-troubleshooting.md @@ -29,4 +29,18 @@ ERROR hello hello/hello.go:81 other peer has different genesis! - repo is already locked ``` -- You already have another lotus daemon running. \ No newline at end of file +- You already have another lotus daemon running. + +## Config: Open files limit + +On most systems you can check the open files limit with: + +```sh +ulimit -n +``` + +You can also modify this number by using the `ulimit` command. It gives you the ability to control the resources available for the shell or process started by it. If the number is below 10000, you can change it with the following command prior to starting the Lotus daemon: + +```sh +ulimit -n 10000 +``` From 295c0861c408faef2832f754006fc922c9bc7fd4 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 10 Jun 2020 22:15:46 -0700 Subject: [PATCH 15/81] fix chain index seeking through long ranges of null rounds --- chain/store/index.go | 12 +++++-- chain/store/index_test.go | 76 +++++++++++++++++++++++++++++++++++++++ chain/types/mock/chain.go | 7 +++- 3 files changed, 91 insertions(+), 4 deletions(-) create mode 100644 chain/store/index_test.go diff --git a/chain/store/index.go b/chain/store/index.go index 15d5d7025..8747c79c6 100644 --- a/chain/store/index.go +++ b/chain/store/index.go @@ -97,9 +97,15 @@ func (ci *ChainIndex) fillCache(tsk types.TipSetKey) (*lbEntry, error) { rheight -= ci.skipLength - skipTarget, err := ci.walkBack(parent, rheight) - if err != nil { - return nil, err + var skipTarget *types.TipSet + if parent.Height() < rheight { + skipTarget = parent + + } else { + skipTarget, err = ci.walkBack(parent, rheight) + if err != nil { + return nil, xerrors.Errorf("fillCache walkback: %w", err) + } } lbe := &lbEntry{ diff --git a/chain/store/index_test.go b/chain/store/index_test.go new file mode 100644 index 000000000..6b97a8614 --- /dev/null +++ b/chain/store/index_test.go @@ -0,0 +1,76 @@ +package store_test + +import ( + "bytes" + "context" + "testing" + + "github.com/filecoin-project/lotus/chain/gen" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types/mock" + "github.com/filecoin-project/specs-actors/actors/abi" + datastore "github.com/ipfs/go-datastore" + syncds "github.com/ipfs/go-datastore/sync" + blockstore "github.com/ipfs/go-ipfs-blockstore" + "github.com/stretchr/testify/assert" +) + +func TestIndexSeeks(t *testing.T) { + cg, err := gen.NewGenerator() + if err != nil { + t.Fatal(err) + } + + gencar, err := cg.GenesisCar() + if err != nil { + t.Fatal(err) + } + + gen := cg.Genesis() + + ctx := context.TODO() + + nbs := blockstore.NewBlockstore(syncds.MutexWrap(datastore.NewMapDatastore())) + cs := store.NewChainStore(nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil) + + _, err = cs.Import(bytes.NewReader(gencar)) + if err != nil { + t.Fatal(err) + } + + cur := mock.TipSet(gen) + if err := cs.PutTipSet(ctx, mock.TipSet(gen)); err != nil { + t.Fatal(err) + } + cs.SetGenesis(gen) + + for i := 0; i < 100; i++ { + nextts := mock.TipSet(mock.MkBlock(cur, 1, 1)) + + if err := cs.PutTipSet(ctx, nextts); err != nil { + t.Fatal(err) + } + cur = nextts + } + + skip := mock.MkBlock(cur, 1, 1) + skip.Height += 50 + + skipts := mock.TipSet(skip) + + if err := cs.PutTipSet(ctx, skipts); err != nil { + t.Fatal(err) + } + + ts, err := cs.GetTipsetByHeight(ctx, skip.Height-10, skipts, false) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, abi.ChainEpoch(151), ts.Height()) + + ts2, err := cs.GetTipsetByHeight(ctx, 90, skipts, false) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, abi.ChainEpoch(90), ts2.Height()) +} diff --git a/chain/types/mock/chain.go b/chain/types/mock/chain.go index 5154ab115..00d0eecc9 100644 --- a/chain/types/mock/chain.go +++ b/chain/types/mock/chain.go @@ -49,6 +49,11 @@ func MkBlock(parents *types.TipSet, weightInc uint64, ticketNonce uint64) *types panic(err) } + pstateRoot := c + if parents != nil { + pstateRoot = parents.Blocks()[0].ParentStateRoot + } + var pcids []cid.Cid var height abi.ChainEpoch weight := types.NewInt(weightInc) @@ -72,7 +77,7 @@ func MkBlock(parents *types.TipSet, weightInc uint64, ticketNonce uint64) *types ParentWeight: weight, Messages: c, Height: height, - ParentStateRoot: c, + ParentStateRoot: pstateRoot, BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS, Data: []byte("boo! im a signature")}, } } From 1aff04faab5826d1256889335ba627085dbcf03e Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 10 Jun 2020 22:17:29 -0700 Subject: [PATCH 16/81] test a few more cases --- chain/store/index_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/chain/store/index_test.go b/chain/store/index_test.go index 6b97a8614..b1a85996f 100644 --- a/chain/store/index_test.go +++ b/chain/store/index_test.go @@ -68,9 +68,11 @@ func TestIndexSeeks(t *testing.T) { } assert.Equal(t, abi.ChainEpoch(151), ts.Height()) - ts2, err := cs.GetTipsetByHeight(ctx, 90, skipts, false) - if err != nil { - t.Fatal(err) + for i := 0; i <= 100; i++ { + ts3, err := cs.GetTipsetByHeight(ctx, abi.ChainEpoch(i), skipts, false) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, abi.ChainEpoch(i), ts3.Height()) } - assert.Equal(t, abi.ChainEpoch(90), ts2.Height()) } From e0a8e993bc9667187e2f5dd9b043aa2ea57fb4b2 Mon Sep 17 00:00:00 2001 From: yaohcn Date: Thu, 11 Jun 2020 16:35:46 +0800 Subject: [PATCH 17/81] bench prove nedd miner-addr flag --- cmd/lotus-bench/main.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 41af08352..7d7991636 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -598,6 +598,11 @@ var proveCmd = &cli.Command{ Name: "no-gpu", Usage: "disable gpu usage for the benchmark run", }, + &cli.StringFlag{ + Name: "miner-addr", + Usage: "pass miner address (only necessary if using existing sectorbuilder)", + Value: "t01000", + }, }, Action: func(c *cli.Context) error { if c.Bool("no-gpu") { @@ -659,7 +664,7 @@ var proveCmd = &cli.Command{ fmt.Printf("proof: %x\n", proof) - fmt.Printf("----\nresults (v23) (%d)\n", c2in.SectorSize) + fmt.Printf("----\nresults (v27) (%d)\n", c2in.SectorSize) dur := sealCommit2.Sub(start) fmt.Printf("seal: commit phase 2: %s (%s)\n", dur, bps(abi.SectorSize(c2in.SectorSize), dur)) From 2cd6347a134f0364de3a547360fb26e42ef3bbd7 Mon Sep 17 00:00:00 2001 From: Mike Greenberg Date: Thu, 4 Jun 2020 18:18:14 -0400 Subject: [PATCH 18/81] Express block validation, cpu/mem usage via OpenCensus --- chain/sync.go | 9 +++++++++ cmd/lotus/daemon.go | 8 ++++++++ metrics/metrics.go | 26 ++++++++++++++++---------- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index f8a73356f..960ee1ea1 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -528,8 +528,17 @@ func blockSanityChecks(h *types.BlockHeader) error { // ValidateBlock should match up with 'Semantical Validation' in validation.md in the spec func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) error { + validationStart := time.Now() + defer func() { + dur := time.Since(validationStart) + durMilli := dur.Seconds() * float64(1000) + stats.Record(ctx, metrics.BlockValidationDurationMilliseconds.M(durMilli)) + log.Infow("block validation", "took", dur, "height", b.Header.Height) + }() + ctx, span := trace.StartSpan(ctx, "validateBlock") defer span.End() + if build.InsecurePoStValidation { log.Warn("insecure test validation is enabled, if you see this outside of a test, it is a severe bug!") } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index fd9ceea85..912e65253 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -19,6 +19,7 @@ import ( "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" "github.com/urfave/cli/v2" + "go.opencensus.io/plugin/runmetrics" "go.opencensus.io/stats" "go.opencensus.io/stats/view" "go.opencensus.io/tag" @@ -114,6 +115,13 @@ var DaemonCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { + err := runmetrics.Enable(runmetrics.RunMetricOptions{ + EnableCPU: true, + EnableMemory: true, + }) + if err != nil { + return xerrors.Errorf("enabling runtime metrics: %w", err) + } if prof := cctx.String("pprof"); prof != "" { profile, err := os.Create(prof) if err != nil { diff --git a/metrics/metrics.go b/metrics/metrics.go index 340d57536..0ef63de4f 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -22,16 +22,17 @@ var ( // Measures var ( - LotusInfo = stats.Int64("info", "Arbitrary counter to tag lotus info to", stats.UnitDimensionless) - ChainNodeHeight = stats.Int64("chain/node_height", "Current Height of the node", stats.UnitDimensionless) - ChainNodeWorkerHeight = stats.Int64("chain/node_worker_height", "Current Height of workers on the node", stats.UnitDimensionless) - MessageReceived = stats.Int64("message/received", "Counter for total received messages", stats.UnitDimensionless) - MessageValidationFailure = stats.Int64("message/failure", "Counter for message validation failures", stats.UnitDimensionless) - MessageValidationSuccess = stats.Int64("message/success", "Counter for message validation successes", stats.UnitDimensionless) - BlockReceived = stats.Int64("block/received", "Counter for total received blocks", stats.UnitDimensionless) - BlockValidationFailure = stats.Int64("block/failure", "Counter for block validation failures", stats.UnitDimensionless) - BlockValidationSuccess = stats.Int64("block/success", "Counter for block validation successes", stats.UnitDimensionless) - PeerCount = stats.Int64("peer/count", "Current number of FIL peers", stats.UnitDimensionless) + LotusInfo = stats.Int64("info", "Arbitrary counter to tag lotus info to", stats.UnitDimensionless) + ChainNodeHeight = stats.Int64("chain/node_height", "Current Height of the node", stats.UnitDimensionless) + ChainNodeWorkerHeight = stats.Int64("chain/node_worker_height", "Current Height of workers on the node", stats.UnitDimensionless) + MessageReceived = stats.Int64("message/received", "Counter for total received messages", stats.UnitDimensionless) + MessageValidationFailure = stats.Int64("message/failure", "Counter for message validation failures", stats.UnitDimensionless) + MessageValidationSuccess = stats.Int64("message/success", "Counter for message validation successes", stats.UnitDimensionless) + BlockReceived = stats.Int64("block/received", "Counter for total received blocks", stats.UnitDimensionless) + BlockValidationFailure = stats.Int64("block/failure", "Counter for block validation failures", stats.UnitDimensionless) + BlockValidationSuccess = stats.Int64("block/success", "Counter for block validation successes", stats.UnitDimensionless) + BlockValidationDurationMilliseconds = stats.Float64("block/validation_ms", "Duration for Block Validation in ms", stats.UnitMilliseconds) + PeerCount = stats.Int64("peer/count", "Current number of FIL peers", stats.UnitDimensionless) ) var ( @@ -63,6 +64,10 @@ var ( Measure: BlockValidationSuccess, Aggregation: view.Count(), } + BlockValidationDurationView = &view.View{ + Measure: BlockValidationDurationMilliseconds, + Aggregation: view.Sum(), + } MessageReceivedView = &view.View{ Measure: MessageReceived, Aggregation: view.Count(), @@ -90,6 +95,7 @@ var DefaultViews = append([]*view.View{ BlockReceivedView, BlockValidationFailureView, BlockValidationSuccessView, + BlockValidationDurationView, MessageReceivedView, MessageValidationFailureView, MessageValidationSuccessView, From 512270593b77bd53224c67095762ffc8d565b85f Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 9 Jun 2020 12:49:31 -0700 Subject: [PATCH 19/81] clean up some of the more spammy logs --- chain/store/store.go | 5 ----- chain/sub/incoming.go | 5 +---- chain/sync_manager.go | 2 +- chain/vm/runtime.go | 4 ++-- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index 8f15b568f..7f552313f 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -18,7 +18,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/metrics" @@ -964,10 +963,6 @@ func (cs *ChainStore) GetTipsetByHeight(ctx context.Context, h abi.ChainEpoch, t return ts, nil } - if ts.Height()-h > build.ForkLengthThreshold { - log.Warnf("expensive call to GetTipsetByHeight, seeking %d levels", ts.Height()-h) - } - lbts, err := cs.cindex.GetTipsetByHeight(ctx, ts, h) if err != nil { return nil, err diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 8b5643de7..ba61dc5c5 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -60,8 +60,6 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha src := msg.GetFrom() go func() { - log.Infof("New block over pubsub: %s", blk.Cid()) - start := time.Now() log.Debug("about to fetch messages for block from pubsub") bmsgs, err := s.Bsync.FetchMessagesByCids(context.TODO(), blk.BlsMessages) @@ -145,8 +143,7 @@ func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub // track validation time begin := time.Now() defer func() { - end := time.Now() - log.Infof("block validation time: %s", end.Sub(begin)) + log.Debugf("block validation time: %s", time.Since(begin)) }() stats.Record(ctx, metrics.BlockReceived.M(1)) diff --git a/chain/sync_manager.go b/chain/sync_manager.go index e00063961..deb626fda 100644 --- a/chain/sync_manager.go +++ b/chain/sync_manager.go @@ -280,7 +280,7 @@ func (sm *SyncManager) syncScheduler() { } func (sm *SyncManager) scheduleIncoming(ts *types.TipSet) { - log.Info("scheduling incoming tipset sync: ", ts.Cids()) + log.Debug("scheduling incoming tipset sync: ", ts.Cids()) if sm.getBootstrapState() == BSStateSelected { sm.setBootstrapState(BSStateScheduled) sm.syncTargets <- ts diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index d9268afe4..6e425474c 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -139,7 +139,7 @@ func (rt *Runtime) shimCall(f func() interface{}) (rval []byte, aerr aerrors.Act defer func() { if r := recover(); r != nil { if ar, ok := r.(aerrors.ActorError); ok { - log.Errorf("VM.Call failure: %+v", ar) + log.Warnf("VM.Call failure: %+v", ar) aerr = ar return } @@ -309,7 +309,7 @@ func (rt *Runtime) Context() context.Context { } func (rt *Runtime) Abortf(code exitcode.ExitCode, msg string, args ...interface{}) { - log.Error("Abortf: ", fmt.Sprintf(msg, args...)) + log.Warnf("Abortf: ", fmt.Sprintf(msg, args...)) panic(aerrors.NewfSkip(2, code, msg, args...)) } From fe60ffb6ff016d7e26f55c76c3bf6c6835a37a16 Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 9 Jun 2020 11:04:17 -0700 Subject: [PATCH 20/81] display quantities of bytes in human-readable format - client commP - client local - client find - client query-ask - client list-deals - state sector-size --- cli/chain.go | 3 +-- cli/client.go | 10 +++++----- cli/state.go | 5 ++--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/cli/chain.go b/cli/chain.go index d76335263..01714d850 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -12,7 +12,6 @@ import ( "strings" "time" - "github.com/docker/go-units" "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/specs-actors/actors/abi" @@ -230,7 +229,7 @@ var chainStatObjCmd = &cli.Command{ } fmt.Printf("Links: %d\n", stats.Links) - fmt.Printf("Size: %s (%d)\n", units.BytesSize(float64(stats.Size)), stats.Size) + fmt.Printf("Size: %s (%d)\n", types.SizeStr(types.NewInt(stats.Size)), stats.Size) return nil }, } diff --git a/cli/client.go b/cli/client.go index d6b59db53..e512f9bd9 100644 --- a/cli/client.go +++ b/cli/client.go @@ -143,7 +143,7 @@ var clientCommPCmd = &cli.Command{ } fmt.Println("CID: ", encoder.Encode(ret.Root)) - fmt.Println("Piece size: ", ret.Size) + fmt.Println("Piece size: ", types.SizeStr(types.NewInt(uint64(ret.Size)))) return nil }, } @@ -203,7 +203,7 @@ var clientLocalCmd = &cli.Command{ } for _, v := range list { - fmt.Printf("%s %s %d %s\n", encoder.Encode(v.Key), v.FilePath, v.Size, v.Status) + fmt.Printf("%s %s %s %s\n", encoder.Encode(v.Key), v.FilePath, types.SizeStr(types.NewInt(v.Size)), v.Status) } return nil }, @@ -371,7 +371,7 @@ var clientFindCmd = &cli.Command{ fmt.Printf("ERR %s@%s: %s\n", offer.Miner, offer.MinerPeerID, offer.Err) continue } - fmt.Printf("RETRIEVAL %s@%s-%sfil-%db\n", offer.Miner, offer.MinerPeerID, types.FIL(offer.MinPrice), offer.Size) + fmt.Printf("RETRIEVAL %s@%s-%sfil-%s\n", offer.Miner, offer.MinerPeerID, types.FIL(offer.MinPrice), types.SizeStr(types.NewInt(offer.Size))) } return nil @@ -520,7 +520,7 @@ var clientQueryAskCmd = &cli.Command{ fmt.Printf("Ask: %s\n", maddr) fmt.Printf("Price per GiB: %s\n", types.FIL(ask.Ask.Price)) - fmt.Printf("Max Piece size: %d\n", ask.Ask.MaxPieceSize) + fmt.Printf("Max Piece size: %s\n", types.SizeStr(types.NewInt(uint64(ask.Ask.MaxPieceSize)))) size := cctx.Int64("size") if size == 0 { @@ -597,7 +597,7 @@ var clientListDeals = &cli.Command{ slashed = fmt.Sprintf("Y (epoch %d)", d.OnChainDealState.SlashEpoch) } - fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s\t%d\t%s\n", d.LocalDeal.ProposalCid, d.LocalDeal.DealID, d.LocalDeal.Provider, storagemarket.DealStates[d.LocalDeal.State], onChain, slashed, d.LocalDeal.PieceCID, d.LocalDeal.Size, d.LocalDeal.PricePerEpoch, d.LocalDeal.Duration, d.LocalDeal.Message) + fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n", d.LocalDeal.ProposalCid, d.LocalDeal.DealID, d.LocalDeal.Provider, storagemarket.DealStates[d.LocalDeal.State], onChain, slashed, d.LocalDeal.PieceCID, types.SizeStr(types.NewInt(d.LocalDeal.Size)), d.LocalDeal.PricePerEpoch, d.LocalDeal.Duration, d.LocalDeal.Message) } return w.Flush() }, diff --git a/cli/state.go b/cli/state.go index 6720ceacd..b741ddcd1 100644 --- a/cli/state.go +++ b/cli/state.go @@ -11,7 +11,6 @@ import ( "strings" "time" - "github.com/docker/go-units" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multihash" @@ -153,7 +152,7 @@ var stateMinerInfo = &cli.Command{ fmt.Printf("Owner:\t%s\n", mi.Owner) fmt.Printf("Worker:\t%s\n", mi.Worker) fmt.Printf("PeerID:\t%s\n", mi.PeerId) - fmt.Printf("SectorSize:\t%s (%d)\n", units.BytesSize(float64(mi.SectorSize)), mi.SectorSize) + fmt.Printf("SectorSize:\t%s (%d)\n", types.SizeStr(types.NewInt(uint64(mi.SectorSize))), mi.SectorSize) return nil }, @@ -707,7 +706,7 @@ var stateSectorSizeCmd = &cli.Command{ return err } - fmt.Printf("%d\n", mi.SectorSize) + fmt.Printf("%s (%d)\n", types.SizeStr(types.NewInt(uint64(mi.SectorSize))), mi.SectorSize) return nil }, } From dc112a5a00d3f34c2e671d543fad48d420e9e1e3 Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Tue, 9 Jun 2020 20:17:28 -0300 Subject: [PATCH 21/81] Lotus architecture notes (#1768) Co-authored-by: Aayush Rajasekaran --- chain/sync.go | 2 + chain/sync_manager.go | 1 + documentation/en/.library.json | 6 + documentation/en/architecture.md | 394 ++++++++++++++++++ .../en/dev/WIP-arch-complementary-notes.md | 153 +++++++ 5 files changed, 556 insertions(+) create mode 100644 documentation/en/architecture.md create mode 100644 documentation/en/dev/WIP-arch-complementary-notes.md diff --git a/chain/sync.go b/chain/sync.go index 960ee1ea1..03ccc3ab7 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -429,6 +429,8 @@ func (syncer *Syncer) Sync(ctx context.Context, maybeHead *types.TipSet) error { return xerrors.Errorf("collectChain failed: %w", err) } + // At this point we have accepted and synced to the new `maybeHead` + // (`StageSyncComplete`). if err := syncer.store.PutTipSet(ctx, maybeHead); err != nil { span.AddAttributes(trace.StringAttribute("put_error", err.Error())) span.SetStatus(trace.Status{ diff --git a/chain/sync_manager.go b/chain/sync_manager.go index deb626fda..e3fbdf4e1 100644 --- a/chain/sync_manager.go +++ b/chain/sync_manager.go @@ -35,6 +35,7 @@ type SyncManager struct { syncStates []*SyncerState + // Normally this handler is set to `(*Syncer).Sync()`. doSync func(context.Context, *types.TipSet) error stop chan struct{} diff --git a/documentation/en/.library.json b/documentation/en/.library.json index 018e6189b..f637c813f 100644 --- a/documentation/en/.library.json +++ b/documentation/en/.library.json @@ -71,6 +71,12 @@ ] }, { + "title": "Architecture", + "slug": "en+arch", + "github": "en/architecture.md", + "value": null, + "posts": [] + }, { "title": "Storage Mining", "slug": "en+mining", "github": "en/mining.md", diff --git a/documentation/en/architecture.md b/documentation/en/architecture.md new file mode 100644 index 000000000..970f2a13d --- /dev/null +++ b/documentation/en/architecture.md @@ -0,0 +1,394 @@ +# Lotus + +Lotus is an implementation of the [Filecoin Distributed Storage Network](https://filecoin.io/). +A Lotus node syncs blockchains that follow the +Filecoin protocol, validating the blocks and state transitions. +The specification for the Filecoin protocol can be found [here](https://filecoin-project.github.io/specs/). + +For information on how to setup and operate a Lotus node, +please follow the instructions [here](https://lotu.sh/en+getting-started). + +# Components + +At a high level, a Lotus node comprises the following components: + +FIXME: No mention of block production here, cross-reference with schomatis's miner doc +- The Syncer, which manages the process of syncing the blockchain +- The State Manager, which can compute the state at any given point in the chain +- The Virtual Machine (VM), which executes messages +- The Repository, where all data is stored +- P2P stuff (FIXME missing libp2p listed under other PL dependencies)? allows hello, blocksync, retrieval, storage +- API / CLI (FIXME missing, in scratchpad) +- Other Filecoin dependencies (specs actors, proofs, storage, etc., FIXME missing) +- Is the Builder worth its own component? +- Other PL dependencies (IPFS, libp2p, IPLD? FIXME, missing) +- External libraries used by Lotus and other deps (FIXME, missing) + +# Preliminaries + +We discuss some key Filecoin concepts here, aiming to explain them by contrasting them with analogous concepts +in other well-known blockchains like Ethereum. We only provide brief descriptions here; elaboration +can be found in the [spec](https://filecoin-project.github.io/specs/). + +### Tipsets + +Unlike in Ethereum, a block can have multiple parents in Filecoin. We thus refer to the parent set of a block, +instead of a single parent. +A [tipset](https://filecoin-project.github.io/specs/#systems__filecoin_blockchain__struct__tipset) +is any set of blocks that share the same parent set. + +There is no concept of "block difficulty" in Filecoin. Instead, +the weight of a tipset is simply the number of blocks in the chain that ends in that tipset. Note that a longer chain +can have less weight than a shorter chain with more blocks per tipset. + +We also allow for "null" tipsets, which include zero blocks. This allows miners to "skip" a round, and build on top +of an imaginary empty tipset if they want to. + +We call the heaviest tipset in a chain the "head" of the chain. + +### Actors and Messages + +An [Actor](https://filecoin-project.github.io/specs/#systems__filecoin_vm__actor) + is analogous to a smart contract in Ethereum. Filecoin does not allow users to define their own +actors, but comes with several [builtin actors](https://github.com/filecoin-project/specs-actors), +which can be thought of as pre-compiled contracts. + +A [Message](https://filecoin-project.github.io/specs/#systems__filecoin_vm__message) +is analogous to transactions in Ethereum. + +# Sync + +Sync refers to the process by which a Lotus node synchronizes to the heaviest chain being advertised by its peers. +At a high-level, Lotus syncs in a manner similar to most other blockchains; a Lotus node listens to the various +chains its peers claim to be at, picks the heaviest one, requests the blocks in the chosen chain, +and validates each block in that chain, running all state transitions along the way. + +The majority of the sync functionality happens in the [`Syncer`](https://github.com/filecoin-project/lotus/blob/master/chain/sync.go), +internally managed by a [`SyncManager`](https://github.com/filecoin-project/lotus/blob/master/chain/sync_manager.go). + +We now discuss the various stages of the sync process. + +## Sync setup + +When a Lotus node connects to a new peer, we exchange the head of our chain +with the new peer through [the `hello` protocol](https://github.com/filecoin-project/lotus/blob/master/node/hello/hello.go). +If the peer's head is heavier than ours, we try to sync to it. Note +that we do NOT update our chain head at this stage. + +## Fetching and Persisting Block Headers + +Note: The API refers to these stages as `StageHeaders` and `StagePersistHeaders`. + +We proceed in the sync process by requesting block headers from the peer, +moving back from their head, until we reach a tipset that we have in common +(such a common tipset must exist, thought it may simply be the genesis block). +The functionality can be found in `Syncer::collectHeaders()`. + +If the common tipset is our head, we treat the sync as a "fast-forward", else we must +drop part of our chain to connect to the peer's head (referred to as "forking"). + +FIXME: This next para might be best replaced with a link to the validation doc +Some of the possible causes of failure in this stage include: + +- The chain is linked to a block that we have previously marked as bad, +and stored in a [`BadBlockCache`](https://github.com/filecoin-project/lotus/blob/master/chain/badtscache.go). +- The beacon entries in a block are inconsistent (FIXME: more details about what is validated here wouldn't be bad). +- Switching to this new chain would involve a chain reorganization beyond the allowed threshold (SPECK-CHECK). + +## Fetching and Validating Blocks + +Note: The API refers to this stage as `StageMessages`. + +Having acquired the headers and found a common tipset, we then move forward, requesting the full blocks, including the messages. + +For each block, we first confirm the syntactic validity of the block (SPECK-CHECK), +which includes the syntactic validity of messages included +in the block. +We then apply the messages, running all the state transitions, and compare the state root we calculate with the provided state root. + + +FIXME: This next para might be best replaced with a link to the validation doc +Some of the possible causes of failure in this stage include: + +- a block is syntactically invalid (including potentially containing syntactically invalid messages) +- the computed state root after applying the block doesn't match the block's state root +- FIXME: Check what's covered by syntactic validity, and add anything important that isn't (like proof validity, future checks, etc.) + +The core functionality can be found in `Syncer::ValidateTipset()`, with `Syncer::checkBlockMessages()` performing +syntactic validation of messages. + +## Setting the head + +Note: The API refers to this stage as `StageSyncComplete`. + +If all validations pass we will now set that head as our heaviest tipset in +[`ChainStore`](https://github.com/filecoin-project/lotus/blob/master/chain/store/store.go). +We already have the full state, since we calculated +it during the sync process. + +FIXME (aayush) I don't fuilly understand the next 2 paragraphs, but it seems important. Confirm and polish. +Relevant issue in IPFS: https://github.com/ipfs/ipfs-docs/issues/264 + +It is important to note at this point that similar to the IPFS architecture of addressing by content and not by location/address (FIXME: check and link to IPFS docs) the "actual" chain stored in the node repo is *relative* to which CID we look for. We always have stored a series of Filecoin blocks pointing to other blocks, each a potential chain in itself by following its parent's reference, and its parent's parent, and so on up to the genesis block. (FIXME: We need a diagram here, one of the Filecoin blog entries might have something similar to what we are describing here.) It only depends on *where* (location) do we start to look for. The *only* address/location reference we hold of the chain, a relative reference, is the `heaviest` pointer. This is reflected by the fact that we don't store it in the `Blockstore` by a fixed, *absolute*, CID that reflects its contents, as this will change each time we sync to a new head (FIXME: link to the immutability IPFS doc that I need to write). + +FIXME: Create a further reading appendix, move this next para to it, along with other +extraneous content +This is one of the few items we store in `Datastore` by key, location, allowing its contents to change on every sync. This is reflected in the `(*ChainStore) writeHead()` function (called by `takeHeaviestTipSet()` above) where we reference the pointer by the explicit `chainHeadKey` address (the string `"head"`, not a hash embedded in a CID), and similarly in `(*ChainStore).Load()` when we start the node and create the `ChainStore`. Compare this to a Filecoin block or message which are immutable, stored in the `Blockstore` by CID, once created they never change. + +## Keeping up with the chain + +A Lotus node also listens for new blocks broadcast by its peers over the `gossipsub` channel (see FIXME for more). +If we have validated such a block's parent tipset, and adding it to our tipset at its height would lead to a heavier +head, then we validate and add this block. The validation described is identical to that invoked during the sync +process (indeed, it's the same codepath). + +# State + +In Filecoin, the chain state at any given point is a collection of data stored under a root CID +encapsulated in the [`StateTree`](https://github.com/filecoin-project/lotus/blob/master/chain/state/statetree.go), +and accessed through the +[`StateManager`](https://github.com/filecoin-project/lotus/blob/master/chain/stmgr/stmgr.go). +The state at the chain's head is thus easily tracked and updated in a state root CID. +(FIXME: Talk about CIDs somewhere, we might want to explain some of the modify/flush/update-root mechanism here.)) + +## Calculating a Tipset State + +Recall that a tipset is a set of blocks that have identical parents (that is, that are built on top of the same tipset). +The genesis tipset comprises the genesis block(s), and has some state corresponding to it. + +The methods `TipSetState()` and `computeTipSetState()` in +[`StateManager`](https://github.com/filecoin-project/lotus/blob/master/chain/stmgr/stmgr.go) + are responsible for computing +the state that results from applying a tipset. This involves applying all the messages included +in the tipset, and performing implicit operations like awarding block rewards. + +Any valid block built on top of a tipset `ts` should have its Parent State Root equal to the result of +calculating the tipset state of `ts`. Note that this means that all blocks in a tipset must have the same Parent +State Root (which is to be expected, since they have the same parent tipset) + +### Preparing to apply a tipset + +When `StateManager::computeTipsetState()` is called with a tipset, `ts`, +it retrieves the parent state root of the blocks in `ts`. It also creates a list of `BlockMessages`, which wraps the BLS +and SecP messages in a block along with the miner that produced the block. + +Control then flows to `StateManager::ApplyBlocks()`, which builds a VM to apply the messages given to it. The VM +is initialized with the parent state root of the blocks in `ts`. We apply the blocks in `ts` in order (see FIXME for +ordering of blocks in a tipset). + +### Applying a block + +For each block, we prepare to apply the ordered messages (first BLS, then SecP). Before applying a message, we check if +we have already applied a message with that CID within the scope of this method. If so, we simply skip that message; +this is how duplicate messages included in the same tipset are skipped (with only the miner of the "first" block to +include the message getting the reward). For the actual process of message application, see FIXME (need an +internal link here), for now we +simply assume that the outcome of the VM applying a message is either an error, or a +[`MessageReceipt`](https://github.com/filecoin-project/lotus/blob/master/chain/types/message_receipt.go) + and some +other information. + +We treat an error from the VM as a showstopper; there is no recovery, and no meaningful state can be computed for `ts`. +Given a successful receipt, we add the rewards and penalties to what the miner has earned so far. Once all the messages +included in a block have been applied (or skipped if they're a duplicate), we use an implicit message to call +the Reward Actor. This awards the miner their reward for having won a block, and also awards / penalizes them based +on the message rewards and penalties we tracked. + +We then proceed to apply the next block in `ts`, using the same VM. This means that the state changes that result +from applying a message are visible when applying all subsequent messages, even if they are included in a different block. + +### Finishing up + +Having applied all the blocks, we send one more implicit message, to the Cron Actor, which handles operations that +must be performed at the end of every epoch (see FIXME for more). The resulting state after calling the Cron Actor +is the computed state of the tipset. + +# Virtual Machine + +The Virtual Machine (VM) is responsible for executing messages. +The [Lotus Virtual Machine](https://github.com/filecoin-project/lotus/blob/master/chain/vm/vm.go) +invokes the appropriate methods in the builtin actors, and provides +a [`Runtime`](https://github.com/filecoin-project/specs-actors/blob/master/actors/runtime/runtime.go) +interface to the [builtin actors](https://github.com/filecoin-project/specs-actors) +that exposes their state, allows them to take certain actions, and meters +their gas usage. The VM also performs balance transfers, creates new account actors as needed, and tracks the gas reward, +penalty, return value, and exit code. + +## Applying a Message + +The primary entrypoint of the VM is the `ApplyMessage()` method. This method should not return an error +unless something goes unrecoverably wrong. + +The first thing this method does is assess if the message provided meets any of the penalty criteria. +If so, a penalty is issued, and the method returns. Next, the entire gas cost of the message is transferred to +a temporary gas holder account. It is from this gas holder that gas will be deducted; if it runs out of gas, the message +fails. Any unused gas in this holder will be refunded to the message's sender at the end of message execution. + +The VM then increments the sender's nonce, takes a snapshot of the state, and invokes `VM::send()`. + +The `send()` method creates a [`Runtime`](https://github.com/filecoin-project/lotus/blob/master/chain/vm/runtime.go) + for the subsequent message execution. +It then transfers the message's value to the recipient, creating a new account actor if needed. + +### Method Invocation + +We use reflection to translate a Filecoin message for the VM to an actual Go function, relying on the VM's +[`invoker`](https://github.com/filecoin-project/lotus/blob/master/chain/vm/invoker.go) structure. +Each actor has its own set of codes defined in `specs-actors/actors/builtin/methods.go`. +The `invoker` structure maps the builtin actors' CIDs + to a list of `invokeFunc` (one per exported method), which each take the `Runtime` (for state manipulation) + and the serialized input parameters. + +FIXME (aayush) Polish this next para. + +The basic layout (without reflection details) of `(*invoker).transform()` is as follows. From each actor registered in `NewInvoker()` we take its `Exports()` methods converting them to `invokeFunc`s. The actual method is wrapped in another function that takes care of decoding the serialized parameters and the runtime, this function is passed to `shimCall()` that will encapsulate the actors code being run inside a `defer` function to `recover()` from panics (we fail in the actors code with panics to unwrap the stack). The return values will then be (CBOR) marshaled and returned to the VM. + +### Returning from the VM + +Once method invocation is complete (including any subcalls), we return to `ApplyMessage()`, which receives +the serialized response and the [`ActorError`](https://github.com/filecoin-project/lotus/blob/master/chain/actors/aerrors/error.go). +The sender will be charged the appropriate amount of gas for the returned response, which gets put into the +[`MessageReceipt`](https://github.com/filecoin-project/lotus/blob/master/chain/types/message_receipt.go). + +The method then refunds any unused gas to the sender, sets up the gas reward for the miner, and +wraps all of this into an `ApplyRet`, which is returned. + +# Building a Lotus node + +When we launch a Lotus node with the command `./lotus daemon` +(see [here](https://github.com/filecoin-project/lotus/blob/master/cmd/lotus/daemon.go) for more), +the node is created through [dependency injection](https://godoc.org/go.uber.org/fx). +This relies on reflection, which makes some of the references hard to follow. +The node sets up all of the subsystems it needs to run, such as the repository, the network connections, thechain sync +service, etc. +This setup is orchestrated through calls to the `node.Override` function. +The structure of each call indicates the type of component it will set up +(many defined in [`node/modules/dtypes/`](https://github.com/filecoin-project/lotus/tree/master/node/modules/dtypes)), +and the function that will provide it. +The dependency is implicit in the argument of the provider function. + +As an example, consider the `modules.ChainStore()` function that provides the +[`ChainStore`](https://github.com/filecoin-project/lotus/blob/master/chain/store/store.go) structure. +It takes as one of its parameters the [`ChainBlockstore`](https://github.com/filecoin-project/lotus/blob/master/node/modules/dtypes/storage.go) +type, which becomes one of its dependencies. +For the node to be built successfully the `ChainBlockstore` will need to be provided before `ChainStore`, a requirement +that is made explicit in another `Override()` call that sets the provider of that type as the `ChainBlockstore()` function. + +## The Repository + +The repo is the directory where all of a node's information is stored. The node is entirely defined by its repo, which +makes it easy to port to another location. This one-to-one relationship means we can speak +of the node as the repo it is associated with, instead of the daemon process that runs from that repo. + +Only one daemon can run be running with an associated repo at a time. +A process signals that it is running a node associated with a particular repo, by creating and acquiring +a `repo.lock`. + +```sh +lsof ~/.lotus/repo.lock +# COMMAND PID +# lotus 52356 +``` +Trying to launch a second daemon hooked to the same repo leads to a `repo is already locked (lotus daemon already running)` +error. + +The `node.Repo()` function (`node/builder.go`) contains most of the dependencies (specified as `Override()` calls) +needed to properly set up the node's repo. We list the most salient ones here. + +### Datastore + +`Datastore` and `ChainBlockstore`: Data related to the node state is saved in the repo's `Datastore`, +an IPFS interface defined [here](github.com/ipfs/go-datastore/datastore.go). +Lotus creates this interface from a [Badger DB](https://github.com/dgraph-io/badger) in + [`FsRepo`](https://github.com/filecoin-project/lotus/blob/master/node/repo/fsrepo.go). +Every piece of data is fundamentally a key-value pair in the `datastore` directory of the repo. +There are several abstractions laid on top of it that appear through the code depending on *how* we access it, +but it is important to remember that we're always accessing it from the same place. + +FIXME: Maybe mention the `Batching` interface as the developer will stumble upon it before reaching the `Datastore` one. + +#### Blocks + +FIXME: IPFS blocks vs Filecoin blocks ideally happens before this / here + +The [`Blockstore` interface](`github.com/ipfs/go-ipfs-blockstore/blockstore.go`) structures the key-value pair +into the CID format for the key and the [`Block` interface](`github.com/ipfs/go-block-format/blocks.go`) for the value. +The `Block` value is just a raw string of bytes addressed by its hash, which is included in the CID key. + +`ChainBlockstore` creates a `Blockstore` in the repo under the `/blocks` namespace. +Every key stored there will have the `blocks` prefix so that it does not collide with other stores that use the same repo. + +FIXME: Link to IPFS documentation about DAG, CID, and related, especially we need a diagram that shows how do we wrap each datastore inside the next layer (datastore, batching, block store, gc, etc). + +#### Metadata + +`modules.Datastore()` creates a `dtypes.MetadataDS`, which is an alias for the basic `Datastore` interface. +Metadata is stored here under the `/metadata` prefix. +(FIXME: Explain *what* is metadata in contrast with the block store, namely we store the pointer to the heaviest chain, we might just link to that unwritten section here later.) + +FIXME: Explain the key store related calls (maybe remove, per Schomatis) + +### LockedRepo + +`LockedRepo()`: This method doesn't create or initialize any new structures, but rather registers an + `OnStop` [hook](https://godoc.org/go.uber.org/fx/internal/lifecycle#Hook) + that will close the locked repository associated with it on shutdown. + + +### Repo types / Node types + +FIXME: This section needs to be clarified / corrected...I don't fully understand the config differences (what do they have in common, if anything?) + +At the end of the `Repo()` function we see two mutually exclusive configuration calls based on the `RepoType` (`node/repo/fsrepo.go`). +```Go + ApplyIf(isType(repo.FullNode), ConfigFullNode(c)), + ApplyIf(isType(repo.StorageMiner), ConfigStorageMiner(c)), +``` +As we said, the repo fully identifies the node so a repo type is also a *node* type, in this case a full node or a storage miner. (FIXME: What is the difference between the two, does *full* imply miner?) In this case the `daemon` command will create a `FullNode`, this is specified in the command logic itself in `main.DaemonCmd()`, the `FsRepo` created (and passed to `node.Repo()`) will be initiated with that type (see `(*FsRepo).Init(t RepoType)`). + +## Online + +FIXME: Much of this might need to be subsumed into the p2p section + +The `node.Online()` configuration function (`node/builder.go`) initializes components that involve connecting to, +or interacting with, the Filecoin network. These connections are managed through the libp2p stack (FIXME link to this section when it exists). +We discuss some of the components found in the full node type (that is, included in the `ApplyIf(isType(repo.FullNode),` call). + +#### Chainstore + +`modules.ChainStore()` creates the [`store.ChainStore`](https://github.com/filecoin-project/lotus/blob/master/chain/store/store.go)) +that wraps the stores + previously instantiated in `Repo()`. It is the main point of entry for the node to all chain-related data + (FIXME: this is incorrect, we sometimes access its underlying block store directly, and probably shouldn't). + It also holds the crucial `heaviest` pointer, which indicates the current head of the chain. + + #### ChainExchange and ChainBlockservice +`ChainExchange()` and `ChainBlockservice()` establish a BitSwap connection (FIXME libp2p link) +to exchange chain information in the form of `blocks.Block`s stored in the repo. (See sync section for more details, the Filecoin blocks and messages are backed by these raw IPFS blocks that together form the different structures that define the state of the current/heaviest chain.) + +#### Incoming handlers +`HandleIncomingBlocks()` and `HandleIncomingMessages()` start the services in charge of processing new Filecoin blocks +and messages from the network (see `` for more information about the topics the node is subscribed to, FIXME: should that be part of the libp2p section or should we expand on gossipsub separately?). + +#### Hello +`RunHello()`: starts the services to both send (`(*Service).SayHello()`) and receive (`(*Service).HandleStream()`, `node/hello/hello.go`) +`hello` messages. When nodes establish a new connection with each other, they exchange these messages +to share chain-related information (namely their genesis block and their heaviest tipset). + +#### Syncer +`NewSyncer()` creates the `Syncer` structure and starts the services related to the chain sync process (FIXME link). + +### Ordering the dependencies + +We can establish the dependency relations by looking at the parameters that each function needs and by understanding +the architecture of the node and how the different components relate to each other (the chief purpose of this document). + +As an example, the sync mechanism depends on the node being able to exchange different IPFS blocks with the network, +so as to be able to request the "missing pieces" needed to construct the chain. This dependency is reflected by `NewSyncer()` +having a `blocksync.BlockSync` parameter, which in turn depends on `ChainBlockservice()` and `ChainExchange()`. +The chain exchange service further depends on the chain store to save and retrieve chain data, which is reflected +in `ChainExchange()` having `ChainGCBlockstore` as a parameter (which is just a wrapper around `ChainBlockstore` capable + of garbage collection). + +This block store is the same store underlying the chain store, which is an indirect dependency of `NewSyncer()` (through the `StateManager`). +(FIXME: This last line is flaky, we need to resolve the hierarchy better, we sometimes refer to the chain store and sometimes to its underlying block store. We need a diagram to visualize all the different components just mentioned otherwise it is too hard to follow. We probably even need to skip some of the connections mentioned.) \ No newline at end of file diff --git a/documentation/en/dev/WIP-arch-complementary-notes.md b/documentation/en/dev/WIP-arch-complementary-notes.md new file mode 100644 index 000000000..00bedb56a --- /dev/null +++ b/documentation/en/dev/WIP-arch-complementary-notes.md @@ -0,0 +1,153 @@ +# Genesis block + +Seems a good way to start exploring the VM state though the instantiation of its different actors like the storage power. + +Explain where do we load the genesis block, the CAR entries, and we set the root of the state. Follow the daemon command option, `chain.LoadGenesis()` saves all the blocks of the CAR file into the store provided by `ChainBlockstore` (this should already be explained in the previous section). The CAR root (MT root?) of those blocks is decoded into the `BlockHeader` that will be the Filecoin (genesis) block of the chain, but most of the information was stored in the raw data (non-Filecoin, what's the correct term?) blocks forwarded directly to the chain, the block header just has a pointer to it. + +`SetGenesis` block with name 0. `(ChainStore).SetGenesis()` stores it there. + +`MakeInitialStateTree` (`chain/gen/genesis/genesis.go`, used to construct the genesis block (`MakeGenesisBlock()`), constructs the state tree (`NewStateTree`) which is just a "pointer" (root node in the HAMT) to the different actors. It will be continuously used in `(*StateTree).SetActor()` an `types.Actor` structure under a certain `Address` (in the HAMT). (How does the `stateSnaps` work? It has no comments.) + +From this point we can follow different setup function like: + +* `SetupInitActor()`: see the `AddressMap`. + +* `SetupStoragePowerActor`: initial (zero) power state of the chain, most important attributes. + +* Account actors in the `template.Accounts`: `SetActor`. + +Which other actor type could be helpful at this point? + +# Basic concepts + +What should be clear at this point either from this document or the spec. + +## Addresses + +## Accounts + +# Sync Topics PubSub + +Gossip sub spec and some introduction. + +# Look at the constructor of a miner + +Follow the `lotus-storage-miner` command to see how a miner is created, from the command to the message to the storage power logic. + +# Directory structure so far, main structures seen, their relation + +List what are the main directories we should be looking at (e.g., `chain/`) and the most important structures (e.g., `StateTree`, `Runtime`, etc.) + +# Tests + +Run a few messages and observe state changes. What is the easiest test that also let's us "interact" with it (modify something and observe the difference). + +### Filecoin blocks vs IPFS blocks + +The term *block* has different meanings depending on the context, many times both meanings coexist at once in the code and it is important to distinguish them. (FIXME: link to IPFS blocks and related doc throughout this explanation). In terms of the lower IPFS layer, in charge of storing and retrieving data, both present at the repo or accessible through the network (e.g., through the BitSwap protocol discussed later), a block is a string of raw bytes identified by its hash, embedded and fully qualified in a CID identifier. IPFS blocks are the "building blocks" of almost any other piece of (chain) data described in the Filecoin protocol. + +In contrast, in the higher Filecoin (application) layer, a block is roughly (FIXME: link to spec definition, if we have any) a set of zero or more messages grouped together by a single miner which is itself grouped with other blocks (from other miners) in the same round to form a tipset. The Filecoin blockchain is a series of "chained" tipsets, each referencing its parent by its header's *CID*, that is, its header as seen as a single IPFS block, this is where both layers interact. + +Using now the full Go package qualifiers to avoid any ambiguity, the Filecoin block, `github.com/filecoin-project/lotus/chain/types.FullBlock`, is defined as, + +```Go +package types + +import "github.com/ipfs/go-cid" + +type FullBlock struct { + Header *BlockHeader + BlsMessages []*Message + SecpkMessages []*SignedMessage +} + +func (fb *FullBlock) Cid() cid.Cid { + return fb.Header.Cid() +} +``` + +It has, besides the Filecoin messages, a header with protocol related information (e.g., its `Height`) which is (like virtually any other piece of data in the Filecoin protocol) stored, retrieved and shared as an IPFS block with its corresponding CID, + +```Go +func (b *BlockHeader) Cid() cid.Cid { + sb, err := b.ToStorageBlock() + + return sb.Cid() +} + +func (b *BlockHeader) ToStorageBlock() (block.Block, error) { + data, err := b.Serialize() + + return github.com/ipfs/go-block-format.block.NewBlockWithCid(data) +} +``` + +These edited extracts from the `BlockHeader` show how it's treated as an IPFS block, `github.com/ipfs/go-block-format.block.BasicBlock`, to be both stored and referenced by its block storage CID. + +This duality permeates the code (and the Filecoin spec for that matter) but it is usually clear within the context to which block we are referring to. Normally the unqualified *block* is reserved for the Filecoin block and we won't usually refer to the IPFS one but only implicitly through the concept of its CID. With enough understanding of both stack's architecture the two definitions can coexist without much confusion as we will abstract away the IPFS layer and just use the CID as an identifier that we now its unique for two sequences of different *raw* byte strings. + +(FIXME: We use to do this presentation when talking about `gossipsub` topics and incoming blocks, and had to deal with, besides the block ambiguity, a similar confusion with the *message* term, used in libp2p to name anything that comes through the network, needing to present the extremely confusing hierarchy of a libp2p message containing a Filecoin block, identified by a IPFS block CID, containing Filecoin messages.) + +FIXME: Move the following tipset definition to sync or wherever is most needed, to avoid making this more confusing. + +Messages from the same round are collected into a block set (`chain/store/fts.go`): + +```Go +type FullTipSet struct { + Blocks []*types.FullBlock + tipset *types.TipSet + cids []cid.Cid +} +``` + +The "tipset" denomination might be a bit misleading as it doesn't refer *only* to the tip, the block set from the last round in the chain, but to *any* set of blocks, depending on the context the tipset is the actual tip or not. From its own perspective any block set is always the tip because it assumes nothing from following blocks. + +# CLI, API + +Explain how do we communicate with the node, both in terms of the CLI and the programmatic way (to create our own tools). + +## Client/server architecture + +In terms of the Filecoin network the node is a peer on a distributed hierarchy, but in terms of how we interact with the node we have client/server architecture. + +The node itself was initiated with the `daemon` command, it already started syncing to the chain by default. Along with that service it also started a [JSON-RPC](https://en.wikipedia.org/wiki/JSON-RPC) server to allow a client to interact with it. (FIXME: Check if this client is local or can be remote, link to external documentation of connection API.) + +We can connect to this server through the Lotus CLI. Virtually any other command other than `daemon` will run a client that will connect (by default) to the address specified in the `api` file in the repo associated with the node (by default in `~/.lotus`), e.g., + +```sh +cat ~/.lotus/api && echo +# /ip4/127.0.0.1/tcp/1234/http + +# With `lotus daemon` running in another terminal. +nc -v -z 127.0.0.1 1234 + +# Start daemon and turn off the logs to not clutter the command line. +bash -c "lotus daemon &" && + lotus wait-api && + lotus log set-level error # Or a env.var in the daemon command. + +nc -v -z 127.0.0.1 1234 +# Connection to 127.0.0.1 1234 port [tcp/*] succeeded! + +killall lotus +# FIXME: We need a lotus stop command: +# https://github.com/filecoin-project/lotus/issues/1827 +``` + +FIXME: Link to more in-depth documentation of the CLI architecture, maybe some IPFS documentation (since they share some common logic). + +## Node API + +The JSON-RPC server exposes the node API, the `FullNode` interface (defined in `api/api_full.go`). When we issue a command like `lotus sync status` to query the progress of the node sync we don't access the node's internals, those are decoupled in a separate daemon process, we call the `SyncState` function (of the `FullNode` API interface) through the RPC client started by our own command (see `NewFullNodeRPC` in `api/client/client.go` for more details). + +FIXME: Link to (and create) documentation about API fulfillment. + +Because we rely heavily on reflection for this part of the code the call chain is not easily visible by just following the references through the symbolic analysis of the IDE. If we start by the `lotus sync` command definition (in `cli/sync.go`), we eventually end up in the method interface `SyncState`, and when we look for its implementation we will find two functions: + +* `(*SyncAPI).SyncState()` (in `node/impl/full/sync.go`): this is the actual implementation of the API function that shows what the node (here acting as the RPC server) will execute when it receives the RPC request issued from the CLI acting as the client. + +* `(*FullNodeStruct).SyncState()`: this is an "empty placeholder" structure that will get later connected to the JSON-RPC client logic (see `NewMergeClient` in `lib/jsonrpc/client.go`, which is called by `NewFullNodeRPC`). (FIXME: check if this is accurate). The CLI (JSON-RPC client) will actually execute this function which will connect to the server and send the corresponding JSON request that will trigger the call of `(*SyncAPI).SyncState()` with the node implementation. + +This means that when we are tracking the logic of a CLI command we will eventually find this bifurcation and need to study the code of the server-side implementation in `node/impl/full` (mostly in the `common/` and `full/` directories). If we understand this architecture going directly to that part of the code abstracts away the JSON-RPC client/server logic and we can think that the CLI is actually running the node's logic. + +FIXME: Explain that "*the* node" is actually an API structure like `impl.FullNodeAPI` with the different API subcomponents like `full.SyncAPI`. We won't see a *single* node structure, each API (full node, minder, etc) will gather the necessary subcomponents it needs to service its calls. \ No newline at end of file From 6ba5c5d0e151f5351dbf444ff103db447f170e33 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 9 Jun 2020 18:19:33 -0400 Subject: [PATCH 22/81] Doc fix: use correct file name --- documentation/en/.library.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/en/.library.json b/documentation/en/.library.json index f637c813f..545eb4a5b 100644 --- a/documentation/en/.library.json +++ b/documentation/en/.library.json @@ -58,8 +58,8 @@ }, { "title": "Use Lotus with systemd", - "slug": "en+install-system-services", - "github": "en/install-system-services.md", + "slug": "en+install-systemd-services", + "github": "en/install-systemd-services.md", "value": null }, { From 27858506fff9df92f0fc5a51d31ed92a2774a5ae Mon Sep 17 00:00:00 2001 From: waynewyang Date: Wed, 10 Jun 2020 20:34:04 +0800 Subject: [PATCH 23/81] error check --- cli/chain.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cli/chain.go b/cli/chain.go index 01714d850..da88df657 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -889,6 +889,9 @@ var slashConsensusFault = &cli.Command{ BlockHeader1: bh1, BlockHeader2: bh2, }) + if err != nil { + return err + } if cctx.String("miner") == "" { return xerrors.Errorf("--miner flag is required") From 2ce3a5f887034e8d4ba8f5b2859dfc6ab19807a0 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 11 Jun 2020 00:09:48 +0200 Subject: [PATCH 24/81] docs: add a note about open files limit --- documentation/en/join-testnet.md | 2 ++ documentation/en/setup-troubleshooting.md | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/documentation/en/join-testnet.md b/documentation/en/join-testnet.md index bdd1b4a56..d01d7eafa 100644 --- a/documentation/en/join-testnet.md +++ b/documentation/en/join-testnet.md @@ -28,6 +28,8 @@ lotus net peers | wc -l In order to connect to the network, you need to be connected to at least 1 peer. If you’re seeing 0 peers, read our [troubleshooting notes](https://docs.lotu.sh/en+setup-troubleshooting). +Make sure that you have a reasonable "open files limit" set on your machine, such as 10000. If you're seeing a lower value, such as 256 (default on macOS), read our [troubleshooting notes](https://docs.lotu.sh/en+setup-troubleshooting) on how to update it prior to starting the Lotus daemon. + ## Chain sync While the daemon is running, the next requirement is to sync the chain. Run the command below to view the chain sync progress. To see current chain height, visit the [network stats page](https://stats.testnet.filecoin.io/). diff --git a/documentation/en/setup-troubleshooting.md b/documentation/en/setup-troubleshooting.md index 055a9c501..8a23544d9 100644 --- a/documentation/en/setup-troubleshooting.md +++ b/documentation/en/setup-troubleshooting.md @@ -29,4 +29,18 @@ ERROR hello hello/hello.go:81 other peer has different genesis! - repo is already locked ``` -- You already have another lotus daemon running. \ No newline at end of file +- You already have another lotus daemon running. + +## Config: Open files limit + +On most systems you can check the open files limit with: + +```sh +ulimit -n +``` + +You can also modify this number by using the `ulimit` command. It gives you the ability to control the resources available for the shell or process started by it. If the number is below 10000, you can change it with the following command prior to starting the Lotus daemon: + +```sh +ulimit -n 10000 +``` From 14ea847ef63eb1b05625c2ebb9d61443ec059429 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 10 Jun 2020 22:15:46 -0700 Subject: [PATCH 25/81] fix chain index seeking through long ranges of null rounds --- chain/store/index.go | 12 +++++-- chain/store/index_test.go | 76 +++++++++++++++++++++++++++++++++++++++ chain/types/mock/chain.go | 7 +++- 3 files changed, 91 insertions(+), 4 deletions(-) create mode 100644 chain/store/index_test.go diff --git a/chain/store/index.go b/chain/store/index.go index 15d5d7025..8747c79c6 100644 --- a/chain/store/index.go +++ b/chain/store/index.go @@ -97,9 +97,15 @@ func (ci *ChainIndex) fillCache(tsk types.TipSetKey) (*lbEntry, error) { rheight -= ci.skipLength - skipTarget, err := ci.walkBack(parent, rheight) - if err != nil { - return nil, err + var skipTarget *types.TipSet + if parent.Height() < rheight { + skipTarget = parent + + } else { + skipTarget, err = ci.walkBack(parent, rheight) + if err != nil { + return nil, xerrors.Errorf("fillCache walkback: %w", err) + } } lbe := &lbEntry{ diff --git a/chain/store/index_test.go b/chain/store/index_test.go new file mode 100644 index 000000000..6b97a8614 --- /dev/null +++ b/chain/store/index_test.go @@ -0,0 +1,76 @@ +package store_test + +import ( + "bytes" + "context" + "testing" + + "github.com/filecoin-project/lotus/chain/gen" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types/mock" + "github.com/filecoin-project/specs-actors/actors/abi" + datastore "github.com/ipfs/go-datastore" + syncds "github.com/ipfs/go-datastore/sync" + blockstore "github.com/ipfs/go-ipfs-blockstore" + "github.com/stretchr/testify/assert" +) + +func TestIndexSeeks(t *testing.T) { + cg, err := gen.NewGenerator() + if err != nil { + t.Fatal(err) + } + + gencar, err := cg.GenesisCar() + if err != nil { + t.Fatal(err) + } + + gen := cg.Genesis() + + ctx := context.TODO() + + nbs := blockstore.NewBlockstore(syncds.MutexWrap(datastore.NewMapDatastore())) + cs := store.NewChainStore(nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil) + + _, err = cs.Import(bytes.NewReader(gencar)) + if err != nil { + t.Fatal(err) + } + + cur := mock.TipSet(gen) + if err := cs.PutTipSet(ctx, mock.TipSet(gen)); err != nil { + t.Fatal(err) + } + cs.SetGenesis(gen) + + for i := 0; i < 100; i++ { + nextts := mock.TipSet(mock.MkBlock(cur, 1, 1)) + + if err := cs.PutTipSet(ctx, nextts); err != nil { + t.Fatal(err) + } + cur = nextts + } + + skip := mock.MkBlock(cur, 1, 1) + skip.Height += 50 + + skipts := mock.TipSet(skip) + + if err := cs.PutTipSet(ctx, skipts); err != nil { + t.Fatal(err) + } + + ts, err := cs.GetTipsetByHeight(ctx, skip.Height-10, skipts, false) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, abi.ChainEpoch(151), ts.Height()) + + ts2, err := cs.GetTipsetByHeight(ctx, 90, skipts, false) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, abi.ChainEpoch(90), ts2.Height()) +} diff --git a/chain/types/mock/chain.go b/chain/types/mock/chain.go index 5154ab115..00d0eecc9 100644 --- a/chain/types/mock/chain.go +++ b/chain/types/mock/chain.go @@ -49,6 +49,11 @@ func MkBlock(parents *types.TipSet, weightInc uint64, ticketNonce uint64) *types panic(err) } + pstateRoot := c + if parents != nil { + pstateRoot = parents.Blocks()[0].ParentStateRoot + } + var pcids []cid.Cid var height abi.ChainEpoch weight := types.NewInt(weightInc) @@ -72,7 +77,7 @@ func MkBlock(parents *types.TipSet, weightInc uint64, ticketNonce uint64) *types ParentWeight: weight, Messages: c, Height: height, - ParentStateRoot: c, + ParentStateRoot: pstateRoot, BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS, Data: []byte("boo! im a signature")}, } } From ac1048fcd7c3e1e616f7fedb4470bbd0ba159bae Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 10 Jun 2020 22:17:29 -0700 Subject: [PATCH 26/81] test a few more cases --- chain/store/index_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/chain/store/index_test.go b/chain/store/index_test.go index 6b97a8614..b1a85996f 100644 --- a/chain/store/index_test.go +++ b/chain/store/index_test.go @@ -68,9 +68,11 @@ func TestIndexSeeks(t *testing.T) { } assert.Equal(t, abi.ChainEpoch(151), ts.Height()) - ts2, err := cs.GetTipsetByHeight(ctx, 90, skipts, false) - if err != nil { - t.Fatal(err) + for i := 0; i <= 100; i++ { + ts3, err := cs.GetTipsetByHeight(ctx, abi.ChainEpoch(i), skipts, false) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, abi.ChainEpoch(i), ts3.Height()) } - assert.Equal(t, abi.ChainEpoch(90), ts2.Height()) } From 41066bd45e3bd73f2ee2c34e5681ec9241d97262 Mon Sep 17 00:00:00 2001 From: waynewyang Date: Thu, 11 Jun 2020 19:19:17 +0800 Subject: [PATCH 27/81] Add flag for cli slash-consensus --- cli/chain.go | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/cli/chain.go b/cli/chain.go index 0bf7741cb..cbdd957e9 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -835,6 +835,10 @@ var slashConsensusFault = &cli.Command{ Name: "miner", Usage: "Miner address", }, + &cli.StringFlag{ + Name: "extra", + Usage: "Extra block cid", + }, }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -879,10 +883,31 @@ var slashConsensusFault = &cli.Command{ return err } - params, err := actors.SerializeParams(&miner.ReportConsensusFaultParams{ + params := miner.ReportConsensusFaultParams{ BlockHeader1: bh1, BlockHeader2: bh2, - }) + } + + if cctx.String("extra") != "" { + cExtra, err := cid.Parse(cctx.String("extra")) + if err != nil { + return xerrors.Errorf("parsing cid extra: %w", err) + } + + bExtra, err := api.ChainGetBlock(ctx, cExtra) + if err != nil { + return xerrors.Errorf("getting block extra: %w", err) + } + + be, err := cborutil.Dump(bExtra) + if err != nil { + return err + } + + params.BlockHeaderExtra = be + } + + enc, err := actors.SerializeParams(¶ms) if err != nil { return err } @@ -903,7 +928,7 @@ var slashConsensusFault = &cli.Command{ GasPrice: types.NewInt(1), GasLimit: 10000000, Method: builtin.MethodsMiner.ReportConsensusFault, - Params: params, + Params: enc, } smsg, err := api.MpoolPushMessage(ctx, msg) From 8b308972a4ae483c48a9080dd66b8562b97ff41d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 11 Jun 2020 14:49:48 +0200 Subject: [PATCH 28/81] chainstore index: Drop wrong check --- chain/store/index.go | 10 ++++------ chain/store/index_test.go | 8 +++++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/chain/store/index.go b/chain/store/index.go index 8747c79c6..bb363ec18 100644 --- a/chain/store/index.go +++ b/chain/store/index.go @@ -91,16 +91,11 @@ func (ci *ChainIndex) fillCache(tsk types.TipSetKey) (*lbEntry, error) { return nil, err } - if parent.Height() > rheight { - return nil, xerrors.Errorf("cache is inconsistent") - } - rheight -= ci.skipLength var skipTarget *types.TipSet if parent.Height() < rheight { skipTarget = parent - } else { skipTarget, err = ci.walkBack(parent, rheight) if err != nil { @@ -119,8 +114,9 @@ func (ci *ChainIndex) fillCache(tsk types.TipSetKey) (*lbEntry, error) { return lbe, nil } +// floors to nearest skipLength multiple func (ci *ChainIndex) roundHeight(h abi.ChainEpoch) abi.ChainEpoch { - return abi.ChainEpoch(h/ci.skipLength) * ci.skipLength + return (h / ci.skipLength) * ci.skipLength } func (ci *ChainIndex) roundDown(ts *types.TipSet) (*types.TipSet, error) { @@ -152,6 +148,8 @@ func (ci *ChainIndex) walkBack(from *types.TipSet, to abi.ChainEpoch) (*types.Ti } if to > pts.Height() { + // in case pts is lower than the epoch we're looking for (null blocks) + // return a tipset above that height return ts, nil } if to == pts.Height() { diff --git a/chain/store/index_test.go b/chain/store/index_test.go index b1a85996f..73f4901f0 100644 --- a/chain/store/index_test.go +++ b/chain/store/index_test.go @@ -44,7 +44,8 @@ func TestIndexSeeks(t *testing.T) { } cs.SetGenesis(gen) - for i := 0; i < 100; i++ { + // Put 113 blocks from genesis + for i := 0; i < 113; i++ { nextts := mock.TipSet(mock.MkBlock(cur, 1, 1)) if err := cs.PutTipSet(ctx, nextts); err != nil { @@ -53,6 +54,7 @@ func TestIndexSeeks(t *testing.T) { cur = nextts } + // Put 50 null epochs + 1 block skip := mock.MkBlock(cur, 1, 1) skip.Height += 50 @@ -66,9 +68,9 @@ func TestIndexSeeks(t *testing.T) { if err != nil { t.Fatal(err) } - assert.Equal(t, abi.ChainEpoch(151), ts.Height()) + assert.Equal(t, abi.ChainEpoch(164), ts.Height()) - for i := 0; i <= 100; i++ { + for i := 0; i <= 113; i++ { ts3, err := cs.GetTipsetByHeight(ctx, abi.ChainEpoch(i), skipts, false) if err != nil { t.Fatal(err) From c3f23ddce829ddb9bac444f019d2e47d5ba85ac7 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 11 Jun 2020 02:47:28 +0200 Subject: [PATCH 29/81] Refactor ExecutionResult to ExecutionTrace Signed-off-by: Jakub Sztandera --- api/api_full.go | 10 ++--- api/docgen/docgen.go | 2 +- chain/stmgr/call.go | 10 ++--- chain/stmgr/stmgr.go | 8 ++-- chain/types/execresult.go | 4 +- chain/vm/runtime.go | 37 +++++------------ chain/vm/vm.go | 86 +++++++++++++++++++++++---------------- cli/state.go | 14 +++---- go.mod | 1 + go.sum | 3 ++ node/impl/full/state.go | 10 ++--- 11 files changed, 95 insertions(+), 90 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 63d9a5e5d..e9370a0b8 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -350,11 +350,11 @@ type RetrievalOrder struct { } type InvocResult struct { - Msg *types.Message - MsgRct *types.MessageReceipt - InternalExecutions []*types.ExecutionResult - Error string - Duration time.Duration + Msg *types.Message + MsgRct *types.MessageReceipt + ExecutionTrace types.ExecutionTrace + Error string + Duration time.Duration } type MethodCall struct { diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index eb4ab5fd9..d0b5d8558 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -93,7 +93,7 @@ func init() { addExample(build.APIVersion) addExample(api.PCHInbound) addExample(time.Minute) - addExample(&types.ExecutionResult{ + addExample(&types.ExecutionTrace{ Msg: exampleValue(reflect.TypeOf(&types.Message{})).(*types.Message), MsgRct: exampleValue(reflect.TypeOf(&types.MessageReceipt{})).(*types.MessageReceipt), }) diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index b9635696f..32e502e95 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -62,11 +62,11 @@ func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate } return &api.InvocResult{ - Msg: msg, - MsgRct: &ret.MessageReceipt, - InternalExecutions: ret.InternalExecutions, - Error: errs, - Duration: ret.Duration, + Msg: msg, + MsgRct: &ret.MessageReceipt, + ExecutionTrace: ret.ExecutionTrace, + Error: errs, + Duration: ret.Duration, }, nil } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index d6aaebab4..d4c12dffe 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -121,10 +121,10 @@ func (sm *StateManager) ExecutionTrace(ctx context.Context, ts *types.TipSet) (c var trace []*api.InvocResult st, _, err := sm.computeTipSetState(ctx, ts.Blocks(), func(mcid cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { ir := &api.InvocResult{ - Msg: msg, - MsgRct: &ret.MessageReceipt, - InternalExecutions: ret.InternalExecutions, - Duration: ret.Duration, + Msg: msg, + MsgRct: &ret.MessageReceipt, + ExecutionTrace: ret.ExecutionTrace, + Duration: ret.Duration, } if ret.ActorErr != nil { ir.Error = ret.ActorErr.Error() diff --git a/chain/types/execresult.go b/chain/types/execresult.go index 56f2ef143..dd5c8fa3e 100644 --- a/chain/types/execresult.go +++ b/chain/types/execresult.go @@ -2,11 +2,11 @@ package types import "time" -type ExecutionResult struct { +type ExecutionTrace struct { Msg *Message MsgRct *MessageReceipt Error string Duration time.Duration - Subcalls []*ExecutionResult + Subcalls []ExecutionTrace } diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index d9268afe4..0067055d1 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -5,7 +5,6 @@ import ( "context" "encoding/binary" "fmt" - "time" "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi" @@ -51,10 +50,10 @@ type Runtime struct { origin address.Address originNonce uint64 - internalExecutions []*types.ExecutionResult - numActorsCreated uint64 - allowInternal bool - callerValidated bool + executionTrace types.ExecutionTrace + numActorsCreated uint64 + allowInternal bool + callerValidated bool } func (rt *Runtime) TotalFilCircSupply() abi.TokenAmount { @@ -368,7 +367,6 @@ func (rt *Runtime) Send(to address.Address, method abi.MethodNum, m vmr.CBORMars } func (rt *Runtime) internalSend(from, to address.Address, method abi.MethodNum, value types.BigInt, params []byte) ([]byte, aerrors.ActorError) { - start := time.Now() ctx, span := trace.StartSpan(rt.ctx, "vmc.Send") defer span.End() if span.IsRecordingEvents() { @@ -401,27 +399,10 @@ func (rt *Runtime) internalSend(from, to address.Address, method abi.MethodNum, } } - mr := types.MessageReceipt{ - ExitCode: aerrors.RetCode(errSend), - Return: ret, - GasUsed: 0, - } - - er := types.ExecutionResult{ - Msg: msg, - MsgRct: &mr, - Duration: time.Since(start), - } - - if errSend != nil { - er.Error = errSend.Error() - } - if subrt != nil { - er.Subcalls = subrt.internalExecutions rt.numActorsCreated = subrt.numActorsCreated } - rt.internalExecutions = append(rt.internalExecutions, &er) + rt.executionTrace.Subcalls = append(rt.executionTrace.Subcalls, subrt.executionTrace) //&er) return ret, errSend } @@ -504,13 +485,13 @@ func (rt *Runtime) stateCommit(oldh, newh cid.Cid) aerrors.ActorError { } func (rt *Runtime) ChargeGas(toUse int64) { - err := rt.chargeGasSafe(toUse) + err := rt.chargeGasInternal(toUse) if err != nil { panic(err) } } -func (rt *Runtime) chargeGasSafe(toUse int64) aerrors.ActorError { +func (rt *Runtime) chargeGasInternal(toUse int64) aerrors.ActorError { if rt.gasUsed+toUse > rt.gasAvailable { rt.gasUsed = rt.gasAvailable return aerrors.Newf(exitcode.SysErrOutOfGas, "not enough gas: used=%d, available=%d", rt.gasUsed, rt.gasAvailable) @@ -519,6 +500,10 @@ func (rt *Runtime) chargeGasSafe(toUse int64) aerrors.ActorError { return nil } +func (rt *Runtime) chargeGasSafe(toUse int64) aerrors.ActorError { + return rt.chargeGasInternal(toUse) +} + func (rt *Runtime) Pricelist() Pricelist { return rt.pricelist } diff --git a/chain/vm/vm.go b/chain/vm/vm.go index da38fcbf3..42d6d373e 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -102,6 +102,7 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, origin addres pricelist: PricelistByEpoch(vm.blockHeight), allowInternal: true, callerValidated: false, + executionTrace: types.ExecutionTrace{Msg: msg}, } rt.cst = &cbor.BasicIpldStore{ @@ -163,14 +164,16 @@ type Rand interface { type ApplyRet struct { types.MessageReceipt - ActorErr aerrors.ActorError - Penalty types.BigInt - InternalExecutions []*types.ExecutionResult - Duration time.Duration + ActorErr aerrors.ActorError + Penalty types.BigInt + ExecutionTrace types.ExecutionTrace + Duration time.Duration } func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, gasCharge int64) ([]byte, aerrors.ActorError, *Runtime) { + start := time.Now() + st := vm.cstate gasUsed := gasCharge @@ -191,35 +194,50 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, }() } - if aerr := rt.chargeGasSafe(rt.Pricelist().OnMethodInvocation(msg.Value, msg.Method)); aerr != nil { - return nil, aerrors.Wrap(aerr, "not enough gas for method invocation"), rt - } + ret, err := func() ([]byte, aerrors.ActorError) { + if aerr := rt.chargeGasSafe(rt.Pricelist().OnMethodInvocation(msg.Value, msg.Method)); aerr != nil { + return nil, aerrors.Wrap(aerr, "not enough gas for method invocation") + } - toActor, err := st.GetActor(msg.To) - if err != nil { - if xerrors.Is(err, init_.ErrAddressNotFound) { - a, err := TryCreateAccountActor(rt, msg.To) - if err != nil { - return nil, aerrors.Wrapf(err, "could not create account"), rt + toActor, err := st.GetActor(msg.To) + if err != nil { + if xerrors.Is(err, init_.ErrAddressNotFound) { + a, err := TryCreateAccountActor(rt, msg.To) + if err != nil { + return nil, aerrors.Wrapf(err, "could not create account") + } + toActor = a + } else { + return nil, aerrors.Escalate(err, "getting actor") } - toActor = a - } else { - return nil, aerrors.Escalate(err, "getting actor"), rt } - } - if types.BigCmp(msg.Value, types.NewInt(0)) != 0 { - if err := vm.transfer(msg.From, msg.To, msg.Value); err != nil { - return nil, aerrors.Wrap(err, "failed to transfer funds"), nil + if types.BigCmp(msg.Value, types.NewInt(0)) != 0 { + if err := vm.transfer(msg.From, msg.To, msg.Value); err != nil { + return nil, aerrors.Wrap(err, "failed to transfer funds") + } } + + if msg.Method != 0 { + var ret []byte + ret, err := vm.Invoke(toActor, rt, msg.Method, msg.Params) + return ret, err + } + return nil, nil + }() + + mr := types.MessageReceipt{ + ExitCode: aerrors.RetCode(err), + Return: ret, + GasUsed: rt.gasUsed, + } + rt.executionTrace.MsgRct = &mr + rt.executionTrace.Duration = time.Since(start) + if err != nil { + rt.executionTrace.Error = err.Error() } - if msg.Method != 0 { - ret, err := vm.Invoke(toActor, rt, msg.Method, msg.Params) - return ret, err, rt - } - - return nil, nil, rt + return ret, err, rt } func checkMessage(msg *types.Message) error { @@ -250,10 +268,10 @@ func (vm *VM) ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*Ap Return: ret, GasUsed: 0, }, - ActorErr: actorErr, - InternalExecutions: rt.internalExecutions, - Penalty: types.NewInt(0), - Duration: time.Since(start), + ActorErr: actorErr, + ExecutionTrace: rt.executionTrace, + Penalty: types.NewInt(0), + Duration: time.Since(start), }, actorErr } @@ -419,10 +437,10 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, Return: ret, GasUsed: gasUsed, }, - ActorErr: actorErr, - InternalExecutions: rt.internalExecutions, - Penalty: types.NewInt(0), - Duration: time.Since(start), + ActorErr: actorErr, + ExecutionTrace: rt.executionTrace, + Penalty: types.NewInt(0), + Duration: time.Since(start), }, nil } diff --git a/cli/state.go b/cli/state.go index 6720ceacd..5b13c43b9 100644 --- a/cli/state.go +++ b/cli/state.go @@ -931,14 +931,14 @@ var stateComputeStateCmd = &cli.Command{ if cctx.Bool("show-trace") { for _, ir := range stout.Trace { fmt.Printf("%s\t%s\t%s\t%d\t%x\t%d\t%x\n", ir.Msg.From, ir.Msg.To, ir.Msg.Value, ir.Msg.Method, ir.Msg.Params, ir.MsgRct.ExitCode, ir.MsgRct.Return) - printInternalExecutions("\t", ir.InternalExecutions) + printInternalExecutions("\t", ir.ExecutionTrace.Subcalls) } } return nil }, } -func printInternalExecutions(prefix string, trace []*types.ExecutionResult) { +func printInternalExecutions(prefix string, trace []types.ExecutionTrace) { for _, im := range trace { fmt.Printf("%s%s\t%s\t%s\t%d\t%x\t%d\t%x\n", prefix, im.Msg.From, im.Msg.To, im.Msg.Value, im.Msg.Method, im.Msg.Params, im.MsgRct.ExitCode, im.MsgRct.Return) printInternalExecutions(prefix+"\t", im.Subcalls) @@ -1028,11 +1028,9 @@ func computeStateHtml(ts *types.TipSet, o *api.ComputeStateOutput, getCode func( fmt.Printf(`
Error: 
%s
`, ir.Error) } - if len(ir.InternalExecutions) > 0 { - fmt.Println("
Internal executions:
") - if err := printInternalExecutionsHtml(cid.String(), ir.InternalExecutions, getCode); err != nil { - return err - } + fmt.Println("
Execution trace:
") + if err := printInternalExecutionsHtml(cid.String(), ir.ExecutionTrace.Subcalls, getCode); err != nil { + return err } fmt.Println("") } @@ -1042,7 +1040,7 @@ func computeStateHtml(ts *types.TipSet, o *api.ComputeStateOutput, getCode func( return nil } -func printInternalExecutionsHtml(hashName string, trace []*types.ExecutionResult, getCode func(addr address.Address) (cid.Cid, error)) error { +func printInternalExecutionsHtml(hashName string, trace []types.ExecutionTrace, getCode func(addr address.Address) (cid.Cid, error)) error { for i, im := range trace { hashName := fmt.Sprintf("%s-r%d", hashName, i) diff --git a/go.mod b/go.mod index 21fea095f..5a3cf4a07 100644 --- a/go.mod +++ b/go.mod @@ -104,6 +104,7 @@ require ( github.com/multiformats/go-multibase v0.0.2 github.com/multiformats/go-multihash v0.0.13 github.com/opentracing/opentracing-go v1.1.0 + github.com/prometheus/common v0.9.1 github.com/stretchr/objx v0.2.0 // indirect github.com/stretchr/testify v1.5.1 github.com/syndtr/goleveldb v1.0.0 diff --git a/go.sum b/go.sum index 35395be66..e10972bb6 100644 --- a/go.sum +++ b/go.sum @@ -42,9 +42,11 @@ github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75/go.mod h1:uAXEEpARkRhCZfEvy/y0Jcc888f9tHCc1W7/UeEtreE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= @@ -1575,6 +1577,7 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/node/impl/full/state.go b/node/impl/full/state.go index fae2a7b67..1d68a1858 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -242,11 +242,11 @@ func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid. } return &api.InvocResult{ - Msg: m, - MsgRct: &r.MessageReceipt, - InternalExecutions: r.InternalExecutions, - Error: errstr, - Duration: r.Duration, + Msg: m, + MsgRct: &r.MessageReceipt, + ExecutionTrace: r.ExecutionTrace, + Error: errstr, + Duration: r.Duration, }, nil } From 500ddd65859c224dce2fb39a00bf9e151e1d4073 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 11 Jun 2020 16:07:04 +0200 Subject: [PATCH 30/81] go mod tidy Signed-off-by: Jakub Sztandera --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index 5a3cf4a07..21fea095f 100644 --- a/go.mod +++ b/go.mod @@ -104,7 +104,6 @@ require ( github.com/multiformats/go-multibase v0.0.2 github.com/multiformats/go-multihash v0.0.13 github.com/opentracing/opentracing-go v1.1.0 - github.com/prometheus/common v0.9.1 github.com/stretchr/objx v0.2.0 // indirect github.com/stretchr/testify v1.5.1 github.com/syndtr/goleveldb v1.0.0 From d4dbfdbe9191a2b17a660d97060522dbe749d708 Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 9 Jun 2020 15:04:11 -0700 Subject: [PATCH 31/81] rename Config to GetConfig --- node/builder.go | 2 +- node/repo/interface.go | 2 +- node/repo/memrepo.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/node/builder.go b/node/builder.go index f503bff5c..87b39ba41 100644 --- a/node/builder.go +++ b/node/builder.go @@ -412,7 +412,7 @@ func Repo(r repo.Repo) Option { if err != nil { return err } - c, err := lr.Config() + c, err := lr.GetConfig() if err != nil { return err } diff --git a/node/repo/interface.go b/node/repo/interface.go index d81d644c7..002373e0e 100644 --- a/node/repo/interface.go +++ b/node/repo/interface.go @@ -37,7 +37,7 @@ type LockedRepo interface { Datastore(namespace string) (datastore.Batching, error) // Returns config in this repo - Config() (interface{}, error) + GetConfig() (interface{}, error) GetStorage() (stores.StorageConfig, error) SetStorage(func(*stores.StorageConfig)) error diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index 17eeb0253..1b1c716c2 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -217,7 +217,7 @@ func (lmem *lockedMemRepo) Datastore(ns string) (datastore.Batching, error) { return namespace.Wrap(lmem.mem.datastore, datastore.NewKey(ns)), nil } -func (lmem *lockedMemRepo) Config() (interface{}, error) { +func (lmem *lockedMemRepo) GetConfig() (interface{}, error) { if err := lmem.checkToken(); err != nil { return nil, err } From 2d6b2e381148241e3def4ce1f7058fc04bb08fc6 Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 9 Jun 2020 16:30:30 -0700 Subject: [PATCH 32/81] fix a spelling error --- node/config/load.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/config/load.go b/node/config/load.go index 1590efedb..b0786643f 100644 --- a/node/config/load.go +++ b/node/config/load.go @@ -12,7 +12,7 @@ import ( ) // FromFile loads config from a specified file overriding defaults specified in -// the def parameter. If file does not exist or is empty defaults are asummed. +// the def parameter. If file does not exist or is empty defaults are assumed. func FromFile(path string, def interface{}) (interface{}, error) { file, err := os.Open(path) switch { From cf321f76678ab3850c4d1471c10ce3dfc9cedab7 Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 9 Jun 2020 16:37:18 -0700 Subject: [PATCH 33/81] add support for replacing config after node starts - TODO: does a "locked repo" need fine-grained (i.e. field-level) locking? --- node/builder.go | 2 +- node/repo/fsrepo.go | 24 ++++++++++++++++++++++++ node/repo/interface.go | 3 ++- node/repo/memrepo.go | 22 +++++++++++++++++++--- node/repo/repo_test.go | 18 ++++++++++++++++-- 5 files changed, 62 insertions(+), 7 deletions(-) diff --git a/node/builder.go b/node/builder.go index 87b39ba41..f503bff5c 100644 --- a/node/builder.go +++ b/node/builder.go @@ -412,7 +412,7 @@ func Repo(r repo.Repo) Option { if err != nil { return err } - c, err := lr.GetConfig() + c, err := lr.Config() if err != nil { return err } diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 6733b0868..a0530f0f7 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -11,6 +11,7 @@ import ( "strings" "sync" + "github.com/BurntSushi/toml" "github.com/ipfs/go-datastore" fslock "github.com/ipfs/go-fs-lock" logging "github.com/ipfs/go-log/v2" @@ -271,6 +272,29 @@ func (fsr *fsLockedRepo) Config() (interface{}, error) { return config.FromFile(fsr.join(fsConfig), defConfForType(fsr.repoType)) } +func (fsr *fsLockedRepo) SetConfig(cfg interface{}) error { + if err := fsr.stillValid(); err != nil { + return err + } + + tmp, err := ioutil.TempFile("", "lotus-config-temp") + if err != nil { + return err + } + + err = toml.NewEncoder(tmp).Encode(cfg) + if err != nil { + return err + } + + err = os.Rename(tmp.Name(), fsr.join(fsConfig)) + if err != nil { + return err + } + + return nil +} + func (fsr *fsLockedRepo) GetStorage() (stores.StorageConfig, error) { fsr.storageLk.Lock() defer fsr.storageLk.Unlock() diff --git a/node/repo/interface.go b/node/repo/interface.go index 002373e0e..560cd969c 100644 --- a/node/repo/interface.go +++ b/node/repo/interface.go @@ -37,7 +37,8 @@ type LockedRepo interface { Datastore(namespace string) (datastore.Batching, error) // Returns config in this repo - GetConfig() (interface{}, error) + Config() (interface{}, error) + SetConfig(interface{}) error GetStorage() (stores.StorageConfig, error) SetStorage(func(*stores.StorageConfig)) error diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index 1b1c716c2..d942b6bd9 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -30,8 +30,13 @@ type MemRepo struct { token *byte datastore datastore.Datastore - configF func(t RepoType) interface{} keystore map[string]types.KeyInfo + + // given a repo type, produce the default config + configF func(t RepoType) interface{} + + // holds the current config value + config interface{} } type lockedMemRepo struct { @@ -217,11 +222,22 @@ func (lmem *lockedMemRepo) Datastore(ns string) (datastore.Batching, error) { return namespace.Wrap(lmem.mem.datastore, datastore.NewKey(ns)), nil } -func (lmem *lockedMemRepo) GetConfig() (interface{}, error) { +func (lmem *lockedMemRepo) Config() (interface{}, error) { if err := lmem.checkToken(); err != nil { return nil, err } - return lmem.mem.configF(lmem.t), nil + + if lmem.mem.config == nil { + lmem.mem.config = lmem.mem.configF(lmem.t) + } + + return lmem.mem.config, nil +} + +func (lmem *lockedMemRepo) SetConfig(cfg interface{}) error { + lmem.mem.config = cfg + + return nil } func (lmem *lockedMemRepo) Storage() (stores.StorageConfig, error) { diff --git a/node/repo/repo_test.go b/node/repo/repo_test.go index 9c43e8f4b..5da4bcadc 100644 --- a/node/repo/repo_test.go +++ b/node/repo/repo_test.go @@ -9,6 +9,8 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/config" + + "github.com/stretchr/testify/require" ) func basicTest(t *testing.T, repo Repo) { @@ -47,10 +49,22 @@ func basicTest(t *testing.T, repo Repo) { assert.NoError(t, err, "setting multiaddr shouldn't error") assert.Equal(t, ma, apima, "returned API multiaddr should be the same") - cfg, err := lrepo.Config() - assert.Equal(t, config.DefaultFullNode(), cfg, "there should be a default config") + c1, err := lrepo.Config() + assert.Equal(t, config.DefaultFullNode(), c1, "there should be a default config") assert.NoError(t, err, "config should not error") + // mutate config and persist back to repo + cfg1 := c1.(*config.FullNode) + cfg1.Client.IpfsMAddr = "duvall" + err = lrepo.SetConfig(cfg1) + assert.NoError(t, err) + + // load config and verify changes + c2, err := lrepo.Config() + require.NoError(t, err) + cfg2 := c2.(*config.FullNode) + require.Equal(t, cfg2.Client.IpfsMAddr, "duvall") + err = lrepo.Close() assert.NoError(t, err, "should be able to close") From fb1d5197fa8a39cae762c43e2c72566d455794ff Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 10 Jun 2020 09:07:47 -0700 Subject: [PATCH 34/81] pass SetConfig a mutator func, as per PR feedback - fsLockedRepo.config gets a mutex - add missing checkToken call to lockedMemRepo#GetStorage and lockedMemRepo#SetStorage - LockedRepo#SetConfig accepts a mutating function as per @magik --- node/repo/fsrepo.go | 30 +++++++++++++++++++++++------- node/repo/interface.go | 2 +- node/repo/memrepo.go | 41 +++++++++++++++++++++++++++++++---------- node/repo/repo_test.go | 7 ++++--- 4 files changed, 59 insertions(+), 21 deletions(-) diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index a0530f0f7..b223731d9 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -230,6 +230,7 @@ type fsLockedRepo struct { dsOnce sync.Once storageLk sync.Mutex + configLk sync.Mutex } func (fsr *fsLockedRepo) Path() string { @@ -266,28 +267,43 @@ func (fsr *fsLockedRepo) stillValid() error { } func (fsr *fsLockedRepo) Config() (interface{}, error) { - if err := fsr.stillValid(); err != nil { - return nil, err - } + fsr.configLk.Lock() + defer fsr.configLk.Unlock() + + return fsr.loadConfigFromDisk() +} + +func (fsr *fsLockedRepo) loadConfigFromDisk() (interface{}, error) { return config.FromFile(fsr.join(fsConfig), defConfForType(fsr.repoType)) } -func (fsr *fsLockedRepo) SetConfig(cfg interface{}) error { +func (fsr *fsLockedRepo) SetConfig(c func(interface{})) error { if err := fsr.stillValid(); err != nil { return err } - tmp, err := ioutil.TempFile("", "lotus-config-temp") + fsr.configLk.Lock() + defer fsr.configLk.Unlock() + + cfg, err := fsr.loadConfigFromDisk() if err != nil { return err } - err = toml.NewEncoder(tmp).Encode(cfg) + // mutate in-memory representation of config + c(cfg) + + // buffer into which we write TOML bytes + buf := new(bytes.Buffer) + + // encode now-mutated config as TOML and write to buffer + err = toml.NewEncoder(buf).Encode(cfg) if err != nil { return err } - err = os.Rename(tmp.Name(), fsr.join(fsConfig)) + // write buffer of TOML bytes to config file + err = ioutil.WriteFile(fsr.join(fsConfig), buf.Bytes(), 0644) if err != nil { return err } diff --git a/node/repo/interface.go b/node/repo/interface.go index 560cd969c..5950f813f 100644 --- a/node/repo/interface.go +++ b/node/repo/interface.go @@ -38,7 +38,7 @@ type LockedRepo interface { // Returns config in this repo Config() (interface{}, error) - SetConfig(interface{}) error + SetConfig(func(interface{})) error GetStorage() (stores.StorageConfig, error) SetStorage(func(*stores.StorageConfig)) error diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index d942b6bd9..399b239c1 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -36,7 +36,10 @@ type MemRepo struct { configF func(t RepoType) interface{} // holds the current config value - config interface{} + config struct { + sync.Mutex + val interface{} + } } type lockedMemRepo struct { @@ -50,6 +53,10 @@ type lockedMemRepo struct { } func (lmem *lockedMemRepo) GetStorage() (stores.StorageConfig, error) { + if err := lmem.checkToken(); err != nil { + return stores.StorageConfig{}, err + } + if lmem.sc == nil { lmem.sc = &stores.StorageConfig{StoragePaths: []stores.LocalPath{ {Path: lmem.Path()}, @@ -60,6 +67,10 @@ func (lmem *lockedMemRepo) GetStorage() (stores.StorageConfig, error) { } func (lmem *lockedMemRepo) SetStorage(c func(*stores.StorageConfig)) error { + if err := lmem.checkToken(); err != nil { + return err + } + _, _ = lmem.GetStorage() c(lmem.sc) @@ -227,23 +238,33 @@ func (lmem *lockedMemRepo) Config() (interface{}, error) { return nil, err } - if lmem.mem.config == nil { - lmem.mem.config = lmem.mem.configF(lmem.t) + lmem.mem.config.Lock() + defer lmem.mem.config.Unlock() + + if lmem.mem.config.val == nil { + lmem.mem.config.val = lmem.mem.configF(lmem.t) } - return lmem.mem.config, nil + return lmem.mem.config.val, nil } -func (lmem *lockedMemRepo) SetConfig(cfg interface{}) error { - lmem.mem.config = cfg +func (lmem *lockedMemRepo) SetConfig(c func(interface{})) error { + if err := lmem.checkToken(); err != nil { + return err + } + + lmem.mem.config.Lock() + defer lmem.mem.config.Unlock() + + if lmem.mem.config.val == nil { + lmem.mem.config.val = lmem.mem.configF(lmem.t) + } + + c(lmem.mem.config.val) return nil } -func (lmem *lockedMemRepo) Storage() (stores.StorageConfig, error) { - panic("implement me") -} - func (lmem *lockedMemRepo) SetAPIEndpoint(ma multiaddr.Multiaddr) error { if err := lmem.checkToken(); err != nil { return err diff --git a/node/repo/repo_test.go b/node/repo/repo_test.go index 5da4bcadc..444fab267 100644 --- a/node/repo/repo_test.go +++ b/node/repo/repo_test.go @@ -54,9 +54,10 @@ func basicTest(t *testing.T, repo Repo) { assert.NoError(t, err, "config should not error") // mutate config and persist back to repo - cfg1 := c1.(*config.FullNode) - cfg1.Client.IpfsMAddr = "duvall" - err = lrepo.SetConfig(cfg1) + err = lrepo.SetConfig(func(c interface{}) { + cfg := c.(*config.FullNode) + cfg.Client.IpfsMAddr = "duvall" + }) assert.NoError(t, err) // load config and verify changes From 8354bd91db08764128f8d0d2abba576a5c3d9f9c Mon Sep 17 00:00:00 2001 From: waynewyang Date: Thu, 11 Jun 2020 23:52:44 +0800 Subject: [PATCH 35/81] fix:the condition of time-offset mining fault --- chain/vm/syscalls.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index 143807ab4..2349bc324 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -115,7 +115,7 @@ func (ss *syscallShim) VerifyConsensusFault(a, b, extra []byte) (*runtime.Consen // (b) time-offset mining fault // strictly speaking no need to compare heights based on double fork mining check above, // but at same height this would be a different fault. - if !types.CidArrsEqual(blockA.Parents, blockB.Parents) && blockA.Height != blockB.Height { + if types.CidArrsEqual(blockA.Parents, blockB.Parents) && blockA.Height != blockB.Height { consensusFault = &runtime.ConsensusFault{ Target: blockA.Miner, Epoch: blockB.Height, @@ -159,7 +159,7 @@ func (ss *syscallShim) VerifyConsensusFault(a, b, extra []byte) (*runtime.Consen } if sigErr := ss.VerifyBlockSig(&blockB); sigErr != nil { - return nil, xerrors.Errorf("cannot verify first block sig: %w", sigErr) + return nil, xerrors.Errorf("cannot verify second block sig: %w", sigErr) } return consensusFault, nil From d6b2519843ffe12657b8b9c2d7f5fd89cb57223c Mon Sep 17 00:00:00 2001 From: laser Date: Thu, 11 Jun 2020 11:29:59 -0700 Subject: [PATCH 36/81] config-driven IsAcceptingStorageDeals flag Makes incremental progress towards #1920. --- node/builder.go | 1 + node/config/def.go | 7 ++++++- node/modules/dtypes/miner.go | 4 ++++ node/modules/storageminer.go | 34 ++++++++++++++++++++++++++++++++-- 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/node/builder.go b/node/builder.go index f503bff5c..adbb25efe 100644 --- a/node/builder.go +++ b/node/builder.go @@ -313,6 +313,7 @@ func Online() Option { Override(HandleDealsKey, modules.HandleDeals), Override(new(gen.WinningPoStProver), storage.NewWinningPoStProver), Override(new(*miner.Miner), modules.SetupBlockProducer), + Override(new(dtypes.IsAcceptingStorageDealsFunc), modules.NewIsAcceptingStorageDealsFunc), ), ) } diff --git a/node/config/def.go b/node/config/def.go index 9ca97bb47..a1b03b899 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -27,7 +27,12 @@ type FullNode struct { type StorageMiner struct { Common - Storage sectorstorage.SealerConfig + StorageDeals StorageDealConfig + Storage sectorstorage.SealerConfig +} + +type StorageDealConfig struct { + IsAcceptingStorageDeals bool } // API contains configs for API endpoint diff --git a/node/modules/dtypes/miner.go b/node/modules/dtypes/miner.go index c872fdf69..9a3fcc2cd 100644 --- a/node/modules/dtypes/miner.go +++ b/node/modules/dtypes/miner.go @@ -7,3 +7,7 @@ import ( type MinerAddress address.Address type MinerID abi.ActorID + +// IsAcceptingStorageDealsFunc is a function which reads from miner config to +// determine if the user has disabled storage deals (or not). +type IsAcceptingStorageDealsFunc func() (bool, error) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 1d358628b..4fd1443d1 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -35,6 +35,7 @@ import ( paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/go-storedcounter" + "github.com/filecoin-project/lotus/node/config" sectorstorage "github.com/filecoin-project/sector-storage" "github.com/filecoin-project/sector-storage/ffiwrapper" "github.com/filecoin-project/sector-storage/stores" @@ -304,14 +305,27 @@ func NewStorageAsk(ctx helpers.MetricsCtx, fapi lapi.FullNode, ds dtypes.Metadat return storedAsk, nil } -func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Config, storedAsk *storedask.StoredAsk, h host.Host, ds dtypes.MetadataDS, ibs dtypes.StagingBlockstore, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode) (storagemarket.StorageProvider, error) { +func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Config, storedAsk *storedask.StoredAsk, h host.Host, ds dtypes.MetadataDS, ibs dtypes.StagingBlockstore, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode, isAcceptingStorageDealsFunc dtypes.IsAcceptingStorageDealsFunc) (storagemarket.StorageProvider, error) { net := smnet.NewFromLibp2pHost(h) store, err := piecefilestore.NewLocalFileStore(piecefilestore.OsPath(r.Path())) if err != nil { return nil, err } - p, err := storageimpl.NewProvider(net, namespace.Wrap(ds, datastore.NewKey("/deals/provider")), ibs, store, pieceStore, dataTransfer, spn, address.Address(minerAddress), ffiConfig.SealProofType, storedAsk) + opt := storageimpl.CustomDealDecisionLogic(func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) { + willEntertainProposals, err := isAcceptingStorageDealsFunc() + if err != nil { + return false, "IsAcceptingStorageDealsFunc error", err + } + + if !willEntertainProposals { + return false, "user has disabled storage deals", nil + } + + return true, "", nil + }) + + p, err := storageimpl.NewProvider(net, namespace.Wrap(ds, datastore.NewKey("/deals/provider")), ibs, store, pieceStore, dataTransfer, spn, address.Address(minerAddress), ffiConfig.SealProofType, storedAsk, opt) if err != nil { return p, err } @@ -361,3 +375,19 @@ func StorageAuth(ctx helpers.MetricsCtx, ca lapi.Common) (sectorstorage.StorageA headers.Add("Authorization", "Bearer "+string(token)) return sectorstorage.StorageAuth(headers), nil } + +func NewIsAcceptingStorageDealsFunc(r repo.LockedRepo) (dtypes.IsAcceptingStorageDealsFunc, error) { + return func() (bool, error) { + raw, err := r.Config() + if err != nil { + return false, err + } + + cfg, ok := raw.(*config.StorageMiner) + if !ok { + return false, xerrors.New("expected address of config.StorageMiner") + } + + return cfg.StorageDeals.IsAcceptingStorageDeals, nil + }, nil +} From 6acc9a62f83450eb51e22c5062b82730d482f41b Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 11 Jun 2020 20:37:14 +0200 Subject: [PATCH 37/81] Split gas internally into compute gas and storage gas Signed-off-by: Jakub Sztandera --- chain/vm/gas.go | 53 ++++++++++++++++++++++++++++++------------ chain/vm/gas_v0.go | 56 +++++++++++++++++++++++---------------------- chain/vm/runtime.go | 11 +++++---- chain/vm/vm.go | 4 ++-- 4 files changed, 75 insertions(+), 49 deletions(-) diff --git a/chain/vm/gas.go b/chain/vm/gas.go index f1cf89845..c030621f6 100644 --- a/chain/vm/gas.go +++ b/chain/vm/gas.go @@ -12,34 +12,57 @@ import ( "github.com/ipfs/go-cid" ) +const ( + GasStorageMulti = 1 + GasComputeMulti = 1 +) + +type GasCharge struct { + Name string + ComputeGas int64 + StorageGas int64 +} + +func (g GasCharge) Total() int64 { + return g.ComputeGas*GasComputeMulti + g.StorageGas*GasStorageMulti +} + +func newGasCharge(name string, computeGas int64, storageGas int64) GasCharge { + return GasCharge{ + Name: name, + ComputeGas: computeGas, + StorageGas: storageGas, + } +} + // Pricelist provides prices for operations in the VM. // // Note: this interface should be APPEND ONLY since last chain checkpoint type Pricelist interface { // OnChainMessage returns the gas used for storing a message of a given size in the chain. - OnChainMessage(msgSize int) int64 + OnChainMessage(msgSize int) GasCharge // OnChainReturnValue returns the gas used for storing the response of a message in the chain. - OnChainReturnValue(dataSize int) int64 + OnChainReturnValue(dataSize int) GasCharge // OnMethodInvocation returns the gas used when invoking a method. - OnMethodInvocation(value abi.TokenAmount, methodNum abi.MethodNum) int64 + OnMethodInvocation(value abi.TokenAmount, methodNum abi.MethodNum) GasCharge // OnIpldGet returns the gas used for storing an object - OnIpldGet(dataSize int) int64 + OnIpldGet(dataSize int) GasCharge // OnIpldPut returns the gas used for storing an object - OnIpldPut(dataSize int) int64 + OnIpldPut(dataSize int) GasCharge // OnCreateActor returns the gas used for creating an actor - OnCreateActor() int64 + OnCreateActor() GasCharge // OnDeleteActor returns the gas used for deleting an actor - OnDeleteActor() int64 + OnDeleteActor() GasCharge - OnVerifySignature(sigType crypto.SigType, planTextSize int) (int64, error) - OnHashing(dataSize int) int64 - OnComputeUnsealedSectorCid(proofType abi.RegisteredProof, pieces []abi.PieceInfo) int64 - OnVerifySeal(info abi.SealVerifyInfo) int64 - OnVerifyPost(info abi.WindowPoStVerifyInfo) int64 - OnVerifyConsensusFault() int64 + OnVerifySignature(sigType crypto.SigType, planTextSize int) (GasCharge, error) + OnHashing(dataSize int) GasCharge + OnComputeUnsealedSectorCid(proofType abi.RegisteredProof, pieces []abi.PieceInfo) GasCharge + OnVerifySeal(info abi.SealVerifyInfo) GasCharge + OnVerifyPost(info abi.WindowPoStVerifyInfo) GasCharge + OnVerifyConsensusFault() GasCharge } var prices = map[abi.ChainEpoch]Pricelist{ @@ -93,7 +116,7 @@ func PricelistByEpoch(epoch abi.ChainEpoch) Pricelist { type pricedSyscalls struct { under vmr.Syscalls pl Pricelist - chargeGas func(int64) + chargeGas func(GasCharge) } // Verifies that a signature is valid for an address and plaintext. @@ -146,6 +169,6 @@ func (ps pricedSyscalls) VerifyConsensusFault(h1 []byte, h2 []byte, extra []byte } func (ps pricedSyscalls) BatchVerifySeals(inp map[address.Address][]abi.SealVerifyInfo) (map[address.Address][]bool, error) { - ps.chargeGas(0) // TODO: this is only called by the cron actor. Should we even charge gas? + ps.chargeGas(newGasCharge("BatchVerifySeals", 0, 0)) // TODO: this is only called by the cron actor. Should we even charge gas? return ps.under.BatchVerifySeals(inp) } diff --git a/chain/vm/gas_v0.go b/chain/vm/gas_v0.go index a20ac927d..0b6196377 100644 --- a/chain/vm/gas_v0.go +++ b/chain/vm/gas_v0.go @@ -2,6 +2,7 @@ package vm import ( "fmt" + "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/crypto" @@ -84,17 +85,17 @@ type pricelistV0 struct { var _ Pricelist = (*pricelistV0)(nil) // OnChainMessage returns the gas used for storing a message of a given size in the chain. -func (pl *pricelistV0) OnChainMessage(msgSize int) int64 { - return pl.onChainMessageBase + pl.onChainMessagePerByte*int64(msgSize) +func (pl *pricelistV0) OnChainMessage(msgSize int) GasCharge { + return newGasCharge("OnChainMessage", 0, pl.onChainMessageBase+pl.onChainMessagePerByte*int64(msgSize)) } // OnChainReturnValue returns the gas used for storing the response of a message in the chain. -func (pl *pricelistV0) OnChainReturnValue(dataSize int) int64 { - return int64(dataSize) * pl.onChainReturnValuePerByte +func (pl *pricelistV0) OnChainReturnValue(dataSize int) GasCharge { + return newGasCharge("OnChainReturnValue", 0, int64(dataSize)*pl.onChainReturnValuePerByte) } // OnMethodInvocation returns the gas used when invoking a method. -func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.MethodNum) int64 { +func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.MethodNum) GasCharge { ret := pl.sendBase if value != abi.NewTokenAmount(0) { ret += pl.sendTransferFunds @@ -102,62 +103,63 @@ func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.M if methodNum != builtin.MethodSend { ret += pl.sendInvokeMethod } - return ret + return newGasCharge("OnMethodInvocation", ret, 0) } // OnIpldGet returns the gas used for storing an object -func (pl *pricelistV0) OnIpldGet(dataSize int) int64 { - return pl.ipldGetBase + int64(dataSize)*pl.ipldGetPerByte +func (pl *pricelistV0) OnIpldGet(dataSize int) GasCharge { + return newGasCharge("OnIpldGet", pl.ipldGetBase+int64(dataSize)*pl.ipldGetPerByte, 0) } // OnIpldPut returns the gas used for storing an object -func (pl *pricelistV0) OnIpldPut(dataSize int) int64 { - return pl.ipldPutBase + int64(dataSize)*pl.ipldPutPerByte +func (pl *pricelistV0) OnIpldPut(dataSize int) GasCharge { + return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte) } // OnCreateActor returns the gas used for creating an actor -func (pl *pricelistV0) OnCreateActor() int64 { - return pl.createActorBase + pl.createActorExtra +func (pl *pricelistV0) OnCreateActor() GasCharge { + return newGasCharge("OnCreateActor", pl.createActorBase, pl.createActorExtra) } // OnDeleteActor returns the gas used for deleting an actor -func (pl *pricelistV0) OnDeleteActor() int64 { - return pl.deleteActor +func (pl *pricelistV0) OnDeleteActor() GasCharge { + return newGasCharge("OnDeleteActor", 0, pl.deleteActor) } // OnVerifySignature -func (pl *pricelistV0) OnVerifySignature(sigType crypto.SigType, planTextSize int) (int64, error) { +func (pl *pricelistV0) OnVerifySignature(sigType crypto.SigType, planTextSize int) (GasCharge, error) { costFn, ok := pl.verifySignature[sigType] if !ok { - return 0, fmt.Errorf("cost function for signature type %d not supported", sigType) + return GasCharge{}, fmt.Errorf("cost function for signature type %d not supported", sigType) } - return costFn(int64(planTextSize)), nil + sigName, _ := sigType.Name() + return newGasCharge("OnVerifySignature/"+sigName, costFn(int64(planTextSize)), 0), nil } // OnHashing -func (pl *pricelistV0) OnHashing(dataSize int) int64 { - return pl.hashingBase + int64(dataSize)*pl.hashingPerByte +func (pl *pricelistV0) OnHashing(dataSize int) GasCharge { + return newGasCharge("OnHashing", pl.hashingBase+int64(dataSize)*pl.hashingPerByte, 0) } // OnComputeUnsealedSectorCid -func (pl *pricelistV0) OnComputeUnsealedSectorCid(proofType abi.RegisteredProof, pieces []abi.PieceInfo) int64 { +func (pl *pricelistV0) OnComputeUnsealedSectorCid(proofType abi.RegisteredProof, pieces []abi.PieceInfo) GasCharge { // TODO: this needs more cost tunning, check with @lotus - return pl.computeUnsealedSectorCidBase + return newGasCharge("OnComputeUnsealedSectorCid", pl.computeUnsealedSectorCidBase, 0) } // OnVerifySeal -func (pl *pricelistV0) OnVerifySeal(info abi.SealVerifyInfo) int64 { +func (pl *pricelistV0) OnVerifySeal(info abi.SealVerifyInfo) GasCharge { // TODO: this needs more cost tunning, check with @lotus - return pl.verifySealBase + return newGasCharge("OnVerifySeal", pl.verifySealBase, 0) } // OnVerifyPost -func (pl *pricelistV0) OnVerifyPost(info abi.WindowPoStVerifyInfo) int64 { +func (pl *pricelistV0) OnVerifyPost(info abi.WindowPoStVerifyInfo) GasCharge { // TODO: this needs more cost tunning, check with @lotus - return pl.verifyPostBase + return newGasCharge("OnVerifyPost", pl.verifyPostBase, 0) } // OnVerifyConsensusFault -func (pl *pricelistV0) OnVerifyConsensusFault() int64 { - return pl.verifyConsensusFault +func (pl *pricelistV0) OnVerifyConsensusFault() GasCharge { + return newGasCharge("OnVerifyConsensusFault", pl.verifyConsensusFault, 0) } diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 70583e47f..ae3a60616 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -484,14 +484,15 @@ func (rt *Runtime) stateCommit(oldh, newh cid.Cid) aerrors.ActorError { return nil } -func (rt *Runtime) ChargeGas(toUse int64) { - err := rt.chargeGasInternal(toUse) +func (rt *Runtime) ChargeGas(gas GasCharge) { + err := rt.chargeGasInternal(gas) if err != nil { panic(err) } } -func (rt *Runtime) chargeGasInternal(toUse int64) aerrors.ActorError { +func (rt *Runtime) chargeGasInternal(gas GasCharge) aerrors.ActorError { + toUse := gas.Total() if rt.gasUsed+toUse > rt.gasAvailable { rt.gasUsed = rt.gasAvailable return aerrors.Newf(exitcode.SysErrOutOfGas, "not enough gas: used=%d, available=%d", rt.gasUsed, rt.gasAvailable) @@ -500,8 +501,8 @@ func (rt *Runtime) chargeGasInternal(toUse int64) aerrors.ActorError { return nil } -func (rt *Runtime) chargeGasSafe(toUse int64) aerrors.ActorError { - return rt.chargeGasInternal(toUse) +func (rt *Runtime) chargeGasSafe(gas GasCharge) aerrors.ActorError { + return rt.chargeGasInternal(gas) } func (rt *Runtime) Pricelist() Pricelist { diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 42d6d373e..11db0d419 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -62,7 +62,7 @@ func ResolveToKeyAddr(state types.StateTree, cst cbor.IpldStore, addr address.Ad var _ cbor.IpldBlockstore = (*gasChargingBlocks)(nil) type gasChargingBlocks struct { - chargeGas func(int64) + chargeGas func(GasCharge) pricelist Pricelist under cbor.IpldBlockstore } @@ -294,7 +294,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, pl := PricelistByEpoch(vm.blockHeight) - msgGasCost := pl.OnChainMessage(cmsg.ChainLength()) + msgGasCost := pl.OnChainMessage(cmsg.ChainLength()).Total() // this should never happen, but is currently still exercised by some tests if msgGasCost > msg.GasLimit { return &ApplyRet{ From 0200e3def59c0f4bd7a942694503b9368d5b647c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 11 Jun 2020 20:33:15 +0200 Subject: [PATCH 38/81] wdpost: Submit recoveries for sectors within a deadline --- storage/wdpost_run.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 8cd9fdc5f..3a55356ba 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -111,7 +111,7 @@ func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check *abi.BitFi return &sbf, nil } -func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, deadline uint64, ts *types.TipSet) error { +func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, deadline uint64, deadlineSectors *abi.BitField, ts *types.TipSet) error { faults, err := s.api.StateMinerFaults(ctx, s.actor, ts.Key()) if err != nil { return xerrors.Errorf("getting on-chain faults: %w", err) @@ -136,6 +136,11 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, deadline return xerrors.Errorf("subtracting recovered set from fault set: %w", err) } + unrecovered, err = bitfield.IntersectBitField(unrecovered, deadlineSectors) + if err != nil { + return xerrors.Errorf("intersect unrecovered set with deadlineSectors: %w", err) + } + uc, err := unrecovered.Count() if err != nil { return xerrors.Errorf("counting unrecovered sectors: %w", err) @@ -326,7 +331,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo // late to declare them for this deadline declDeadline := (di.Index + 1) % miner.WPoStPeriodDeadlines - if err := s.checkNextRecoveries(ctx, declDeadline, ts); err != nil { + if err := s.checkNextRecoveries(ctx, declDeadline, deadlines.Due[declDeadline], ts); err != nil { // TODO: This is potentially quite bad, but not even trying to post when this fails is objectively worse log.Errorf("checking sector recoveries: %v", err) } From c458b4712a806639ba9dcdb3c64c0133d19f31a2 Mon Sep 17 00:00:00 2001 From: laser Date: Thu, 11 Jun 2020 12:06:38 -0700 Subject: [PATCH 39/81] better rejection messages --- node/modules/storageminer.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 4fd1443d1..0b3f12586 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -315,11 +315,12 @@ func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Con opt := storageimpl.CustomDealDecisionLogic(func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) { willEntertainProposals, err := isAcceptingStorageDealsFunc() if err != nil { - return false, "IsAcceptingStorageDealsFunc error", err + return false, "miner error", err } if !willEntertainProposals { - return false, "user has disabled storage deals", nil + log.Warnf("storage deal acceptance disabled; rejecting storage deal proposal from client: %s", deal.Client.String()) + return false, "miner is not accepting storage deals", nil } return true, "", nil From 5421d0d1c58093ba5fb2b6e20ce522dce1627994 Mon Sep 17 00:00:00 2001 From: laser Date: Thu, 11 Jun 2020 12:15:28 -0700 Subject: [PATCH 40/81] better field and struct names + set default appropriately Storage miners, by default, should be configured to accept storage deals. --- node/config/def.go | 10 +++++++--- node/modules/storageminer.go | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/node/config/def.go b/node/config/def.go index a1b03b899..3debfd64e 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -27,11 +27,11 @@ type FullNode struct { type StorageMiner struct { Common - StorageDeals StorageDealConfig - Storage sectorstorage.SealerConfig + Dealmaking DealmakingConfig + Storage sectorstorage.SealerConfig } -type StorageDealConfig struct { +type DealmakingConfig struct { IsAcceptingStorageDeals bool } @@ -114,6 +114,10 @@ func DefaultStorageMiner() *StorageMiner { AllowCommit: true, AllowUnseal: true, }, + + Dealmaking: DealmakingConfig{ + IsAcceptingStorageDeals: true, + }, } cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" cfg.Common.API.RemoteListenAddress = "127.0.0.1:2345" diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 0b3f12586..3719f621a 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -389,6 +389,6 @@ func NewIsAcceptingStorageDealsFunc(r repo.LockedRepo) (dtypes.IsAcceptingStorag return false, xerrors.New("expected address of config.StorageMiner") } - return cfg.StorageDeals.IsAcceptingStorageDeals, nil + return cfg.Dealmaking.IsAcceptingStorageDeals, nil }, nil } From 67110ce739a11f9e6e35598ea156bd7381a90938 Mon Sep 17 00:00:00 2001 From: laser Date: Thu, 11 Jun 2020 12:20:11 -0700 Subject: [PATCH 41/81] stub enable/disable storage deal commands --- cmd/lotus-storage-miner/main.go | 2 ++ cmd/lotus-storage-miner/market.go | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index ba7722e4e..9ade0a41b 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -33,6 +33,8 @@ func main() { setPriceCmd, workersCmd, provingCmd, + enableCmd, + disableCmd, } jaeger := tracing.SetupJaegerTracing("lotus") defer func() { diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 68b626360..cc0b84298 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -10,6 +10,24 @@ import ( "github.com/urfave/cli/v2" ) +var enableCmd = &cli.Command{ + Name: "enable", + Usage: "Configure the miner to consider storage deal proposals", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + panic("enable storage deals") + }, +} + +var disableCmd = &cli.Command{ + Name: "disable", + Usage: "Configure the miner to reject all storage deal proposals", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + panic("disable storage deals") + }, +} + var setPriceCmd = &cli.Command{ Name: "set-price", Usage: "Set price that miner will accept storage deals at (FIL / GiB / Epoch)", From c90d35869bd74ab1992971a9f1ed62419313838b Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 11 Jun 2020 21:59:39 +0200 Subject: [PATCH 42/81] Add gas tracing Signed-off-by: Jakub Sztandera --- chain/sync.go | 2 +- chain/types/execresult.go | 55 +++++++++++++++++++++++++++++++++++---- chain/vm/runtime.go | 38 +++++++++++++++++++++++---- chain/vm/vm.go | 40 +++++++++++++++++++--------- cli/state.go | 15 +++++++++++ miner/miner.go | 2 +- 6 files changed, 127 insertions(+), 25 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index 03ccc3ab7..f4b32f2bc 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -857,7 +857,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock // Phase 1: syntactic validation, as defined in the spec minGas := vm.PricelistByEpoch(baseTs.Height()).OnChainMessage(msg.ChainLength()) - if err := m.ValidForBlockInclusion(minGas); err != nil { + if err := m.ValidForBlockInclusion(minGas.Total()); err != nil { return err } diff --git a/chain/types/execresult.go b/chain/types/execresult.go index dd5c8fa3e..41372b832 100644 --- a/chain/types/execresult.go +++ b/chain/types/execresult.go @@ -1,12 +1,57 @@ package types -import "time" +import ( + "encoding/json" + "fmt" + "runtime" + "strings" + "time" +) type ExecutionTrace struct { - Msg *Message - MsgRct *MessageReceipt - Error string - Duration time.Duration + Msg *Message + MsgRct *MessageReceipt + Error string + Duration time.Duration + GasCharges []*GasTrace Subcalls []ExecutionTrace } + +type GasTrace struct { + Name string + Location string + TotalGas int64 + ComputeGas int64 + StorageGas int64 + + TimeTaken time.Duration + + Callers []uintptr `json:"-"` +} + +func (gt *GasTrace) MarshalJSON() ([]byte, error) { + type GasTraceCopy GasTrace + if gt.Location == "" { + if len(gt.Callers) != 0 { + frames := runtime.CallersFrames(gt.Callers) + for { + frame, more := frames.Next() + fn := strings.Split(frame.Function, "/") + + split := strings.Split(frame.File, "/") + file := strings.Join(split[len(split)-2:], "/") + gt.Location += fmt.Sprintf("%s@%s:%d", fn[len(fn)-1], file, frame.Line) + if !more { + break + } + gt.Location += "|" + } + } else { + gt.Location = "n/a" + } + } + + cpy := (*GasTraceCopy)(gt) + return json.Marshal(cpy) +} diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index ae3a60616..902ef94bb 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -5,6 +5,8 @@ import ( "context" "encoding/binary" "fmt" + gruntime "runtime" + "time" "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi" @@ -50,10 +52,12 @@ type Runtime struct { origin address.Address originNonce uint64 - executionTrace types.ExecutionTrace - numActorsCreated uint64 - allowInternal bool - callerValidated bool + executionTrace types.ExecutionTrace + numActorsCreated uint64 + allowInternal bool + callerValidated bool + lastGasChargeTime time.Time + lastGasCharge *types.GasTrace } func (rt *Runtime) TotalFilCircSupply() abi.TokenAmount { @@ -392,7 +396,7 @@ func (rt *Runtime) internalSend(from, to address.Address, method abi.MethodNum, } defer st.ClearSnapshot() - ret, errSend, subrt := rt.vm.send(ctx, msg, rt, 0) + ret, errSend, subrt := rt.vm.send(ctx, msg, rt, nil) if errSend != nil { if errRevert := st.Revert(); errRevert != nil { return nil, aerrors.Escalate(errRevert, "failed to revert state tree after failed subcall") @@ -490,9 +494,33 @@ func (rt *Runtime) ChargeGas(gas GasCharge) { panic(err) } } +func (rt *Runtime) finilizeGasTracing() { + if rt.lastGasCharge != nil { + rt.lastGasCharge.TimeTaken = time.Since(rt.lastGasChargeTime) + } +} func (rt *Runtime) chargeGasInternal(gas GasCharge) aerrors.ActorError { toUse := gas.Total() + var callers [3]uintptr + cout := gruntime.Callers(3, callers[:]) + + now := time.Now() + if rt.lastGasCharge != nil { + rt.lastGasCharge.TimeTaken = now.Sub(rt.lastGasChargeTime) + } + + gasTrace := types.GasTrace{ + Name: gas.Name, + TotalGas: toUse, + ComputeGas: gas.ComputeGas, + StorageGas: gas.StorageGas, + Callers: callers[:cout], + } + rt.executionTrace.GasCharges = append(rt.executionTrace.GasCharges, &gasTrace) + rt.lastGasChargeTime = now + rt.lastGasCharge = &gasTrace + if rt.gasUsed+toUse > rt.gasAvailable { rt.gasUsed = rt.gasAvailable return aerrors.Newf(exitcode.SysErrOutOfGas, "not enough gas: used=%d, available=%d", rt.gasUsed, rt.gasAvailable) diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 11db0d419..75bbdfedb 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -96,13 +96,14 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, origin addres originNonce: originNonce, height: vm.blockHeight, - gasUsed: usedGas, - gasAvailable: msg.GasLimit, - numActorsCreated: nac, - pricelist: PricelistByEpoch(vm.blockHeight), - allowInternal: true, - callerValidated: false, - executionTrace: types.ExecutionTrace{Msg: msg}, + gasUsed: usedGas, + gasAvailable: msg.GasLimit, + numActorsCreated: nac, + pricelist: PricelistByEpoch(vm.blockHeight), + allowInternal: true, + callerValidated: false, + executionTrace: types.ExecutionTrace{Msg: msg}, + lastGasChargeTime: time.Now(), } rt.cst = &cbor.BasicIpldStore{ @@ -171,17 +172,17 @@ type ApplyRet struct { } func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, - gasCharge int64) ([]byte, aerrors.ActorError, *Runtime) { + gasCharge *GasCharge) ([]byte, aerrors.ActorError, *Runtime) { start := time.Now() st := vm.cstate - gasUsed := gasCharge origin := msg.From on := msg.Nonce var nac uint64 = 0 + var gasUsed int64 if parent != nil { - gasUsed = parent.gasUsed + gasUsed + gasUsed = parent.gasUsed origin = parent.origin on = parent.originNonce nac = parent.numActorsCreated @@ -189,11 +190,22 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, rt := vm.makeRuntime(ctx, msg, origin, on, gasUsed, nac) if parent != nil { + rt.lastGasChargeTime = parent.lastGasChargeTime + rt.lastGasCharge = parent.lastGasCharge defer func() { parent.gasUsed = rt.gasUsed + parent.lastGasChargeTime = rt.lastGasChargeTime + parent.lastGasCharge = rt.lastGasCharge }() } + if gasCharge != nil { + if err := rt.chargeGasSafe(*gasCharge); err != nil { + // this should never happen + return nil, aerrors.Wrap(err, "not enough gas for initial message charge, this should not happen"), rt + } + } + ret, err := func() ([]byte, aerrors.ActorError) { if aerr := rt.chargeGasSafe(rt.Pricelist().OnMethodInvocation(msg.Value, msg.Method)); aerr != nil { return nil, aerrors.Wrap(aerr, "not enough gas for method invocation") @@ -261,7 +273,7 @@ func checkMessage(msg *types.Message) error { func (vm *VM) ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*ApplyRet, error) { start := time.Now() - ret, actorErr, rt := vm.send(ctx, msg, nil, 0) + ret, actorErr, rt := vm.send(ctx, msg, nil, nil) return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: aerrors.RetCode(actorErr), @@ -294,7 +306,8 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, pl := PricelistByEpoch(vm.blockHeight) - msgGasCost := pl.OnChainMessage(cmsg.ChainLength()).Total() + msgGas := pl.OnChainMessage(cmsg.ChainLength()) + msgGasCost := msgGas.Total() // this should never happen, but is currently still exercised by some tests if msgGasCost > msg.GasLimit { return &ApplyRet{ @@ -377,7 +390,8 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, } defer st.ClearSnapshot() - ret, actorErr, rt := vm.send(ctx, msg, nil, msgGasCost) + ret, actorErr, rt := vm.send(ctx, msg, nil, &msgGas) + rt.finilizeGasTracing() if aerrors.IsFatal(actorErr) { return nil, xerrors.Errorf("[from=%s,to=%s,n=%d,m=%d,h=%d] fatal error: %w", msg.From, msg.To, msg.Nonce, msg.Method, vm.blockHeight, actorErr) } diff --git a/cli/state.go b/cli/state.go index 1b6f676f2..c65eb1713 100644 --- a/cli/state.go +++ b/cli/state.go @@ -975,6 +975,7 @@ func computeStateHtml(ts *types.TipSet, o *api.ComputeStateOutput, getCode func( } .slow-true-false { color: #660; } .slow-true-true { color: #f80; } + table { font-size: 12px; } @@ -1026,6 +1027,13 @@ func computeStateHtml(ts *types.TipSet, o *api.ComputeStateOutput, getCode func( if ir.MsgRct.ExitCode != 0 { fmt.Printf(`
Error: 
%s
`, ir.Error) } + fmt.Printf("\n
Gas Trace" + + "") + for _, gc := range ir.ExecutionTrace.GasCharges { + fmt.Printf("", + gc.Name, gc.TotalGas, gc.ComputeGas, gc.StorageGas, gc.TimeTaken, gc.Location) + } + fmt.Printf("
NameTotal/Compute/StorageTime TakenLocation
%s%d/%d/%d%s%s
\n") fmt.Println("
Execution trace:
") if err := printInternalExecutionsHtml(cid.String(), ir.ExecutionTrace.Subcalls, getCode); err != nil { @@ -1082,6 +1090,13 @@ func printInternalExecutionsHtml(hashName string, trace []types.ExecutionTrace, if im.MsgRct.ExitCode != 0 { fmt.Printf(`
Error: 
%s
`, im.Error) } + fmt.Printf("\n
Gas Trace" + + "") + for _, gc := range im.GasCharges { + fmt.Printf("", + gc.Name, gc.TotalGas, gc.ComputeGas, gc.StorageGas, gc.TimeTaken, gc.Location) + } + fmt.Printf("
NameTotal/Compute/StorageTime TakenLocation
%s%d/%d/%d%s%s
\n") if len(im.Subcalls) > 0 { fmt.Println("
Subcalls:
") if err := printInternalExecutionsHtml(hashName, im.Subcalls, getCode); err != nil { diff --git a/miner/miner.go b/miner/miner.go index f2059ef7f..23f40338d 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -485,7 +485,7 @@ func SelectMessages(ctx context.Context, al ActorLookup, ts *types.TipSet, msgs vmstart := time.Now() minGas := vm.PricelistByEpoch(ts.Height()).OnChainMessage(msg.ChainLength()) // TODO: really should be doing just msg.ChainLength() but the sync side of this code doesnt seem to have access to that - if err := msg.VMMessage().ValidForBlockInclusion(minGas); err != nil { + if err := msg.VMMessage().ValidForBlockInclusion(minGas.Total()); err != nil { log.Warnf("invalid message in message pool: %s", err) continue } From 7587e6c08bc8048802616212dfc14bb8c0d396fe Mon Sep 17 00:00:00 2001 From: laser Date: Thu, 11 Jun 2020 12:59:50 -0700 Subject: [PATCH 43/81] get and set storage deal acceptance through CLI --- api/api_storage.go | 1 + api/apistruct/struct.go | 9 +++++++-- cmd/lotus-storage-miner/main.go | 2 -- cmd/lotus-storage-miner/market.go | 18 ++++++++++++++++-- node/builder.go | 1 + node/impl/storminer.go | 7 +++++++ node/modules/dtypes/miner.go | 4 ++++ node/modules/storageminer.go | 20 ++++++++++++++++++++ 8 files changed, 56 insertions(+), 6 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index 7d3c78b48..e414ed353 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -54,6 +54,7 @@ type StorageMiner interface { DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) + DealsSetIsAcceptingStorageDeals(ctx context.Context, b bool) error StorageAddLocal(ctx context.Context, path string) error } diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 31cd6badd..23164a993 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -220,8 +220,9 @@ type StorageMinerStruct struct { StorageLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error `perm:"admin"` StorageTryLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) (bool, error) `perm:"admin"` - DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` - DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` + DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` + DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` + DealsSetIsAcceptingStorageDeals func(ctx context.Context, b bool) error `perm:"admin"` StorageAddLocal func(ctx context.Context, path string) error `perm:"admin"` } @@ -852,6 +853,10 @@ func (c *StorageMinerStruct) DealsList(ctx context.Context) ([]storagemarket.Sto return c.Internal.DealsList(ctx) } +func (c *StorageMinerStruct) DealsSetIsAcceptingStorageDeals(ctx context.Context, b bool) error { + return c.Internal.DealsSetIsAcceptingStorageDeals(ctx, b) +} + func (c *StorageMinerStruct) StorageAddLocal(ctx context.Context, path string) error { return c.Internal.StorageAddLocal(ctx, path) } diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index 9ade0a41b..ba7722e4e 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -33,8 +33,6 @@ func main() { setPriceCmd, workersCmd, provingCmd, - enableCmd, - disableCmd, } jaeger := tracing.SetupJaegerTracing("lotus") defer func() { diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index cc0b84298..4b9f903c8 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -15,7 +15,13 @@ var enableCmd = &cli.Command{ Usage: "Configure the miner to consider storage deal proposals", Flags: []cli.Flag{}, Action: func(cctx *cli.Context) error { - panic("enable storage deals") + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + return api.DealsSetIsAcceptingStorageDeals(lcli.DaemonContext(cctx), true) }, } @@ -24,7 +30,13 @@ var disableCmd = &cli.Command{ Usage: "Configure the miner to reject all storage deal proposals", Flags: []cli.Flag{}, Action: func(cctx *cli.Context) error { - panic("disable storage deals") + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + return api.DealsSetIsAcceptingStorageDeals(lcli.DaemonContext(cctx), false) }, } @@ -60,6 +72,8 @@ var dealsCmd = &cli.Command{ Subcommands: []*cli.Command{ dealsImportDataCmd, dealsListCmd, + enableCmd, + disableCmd, }, } diff --git a/node/builder.go b/node/builder.go index adbb25efe..bf42eabd6 100644 --- a/node/builder.go +++ b/node/builder.go @@ -314,6 +314,7 @@ func Online() Option { Override(new(gen.WinningPoStProver), storage.NewWinningPoStProver), Override(new(*miner.Miner), modules.SetupBlockProducer), Override(new(dtypes.IsAcceptingStorageDealsFunc), modules.NewIsAcceptingStorageDealsFunc), + Override(new(dtypes.SetAcceptingStorageDealsFunc), modules.NewSetAcceptingStorageDealsFunc), ), ) } diff --git a/node/impl/storminer.go b/node/impl/storminer.go index e48c9fcad..b88541886 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -25,6 +25,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/impl/common" + "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -41,6 +42,8 @@ type StorageMinerAPI struct { Full api.FullNode StorageMgr *sectorstorage.Manager `optional:"true"` *stores.Index + + SetAcceptingStorageDealsFunc dtypes.SetAcceptingStorageDealsFunc } func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) { @@ -206,6 +209,10 @@ func (sm *StorageMinerAPI) DealsList(ctx context.Context) ([]storagemarket.Stora return sm.StorageProvider.ListDeals(ctx) } +func (sm *StorageMinerAPI) DealsSetIsAcceptingStorageDeals(ctx context.Context, b bool) error { + return sm.SetAcceptingStorageDealsFunc(b) +} + func (sm *StorageMinerAPI) DealsImportData(ctx context.Context, deal cid.Cid, fname string) error { fi, err := os.Open(fname) if err != nil { diff --git a/node/modules/dtypes/miner.go b/node/modules/dtypes/miner.go index 9a3fcc2cd..d14a8775e 100644 --- a/node/modules/dtypes/miner.go +++ b/node/modules/dtypes/miner.go @@ -11,3 +11,7 @@ type MinerID abi.ActorID // IsAcceptingStorageDealsFunc is a function which reads from miner config to // determine if the user has disabled storage deals (or not). type IsAcceptingStorageDealsFunc func() (bool, error) + +// SetAcceptingStorageDealsFunc is a function which is used to disable or enable +// storage deal acceptance. +type SetAcceptingStorageDealsFunc func(bool) error diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 3719f621a..9847f7cc2 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -2,6 +2,7 @@ package modules import ( "context" + "errors" "net/http" "github.com/ipfs/go-bitswap" @@ -17,6 +18,7 @@ import ( "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/routing" "go.uber.org/fx" + "go.uber.org/multierr" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -392,3 +394,21 @@ func NewIsAcceptingStorageDealsFunc(r repo.LockedRepo) (dtypes.IsAcceptingStorag return cfg.Dealmaking.IsAcceptingStorageDeals, nil }, nil } + +func NewSetAcceptingStorageDealsFunc(r repo.LockedRepo) (dtypes.SetAcceptingStorageDealsFunc, error) { + return func(b bool) error { + var typeErr error + + setConfigErr := r.SetConfig(func(raw interface{}) { + cfg, ok := raw.(*config.StorageMiner) + if !ok { + typeErr = errors.New("expected storage miner config") + return + } + + cfg.Dealmaking.IsAcceptingStorageDeals = b + }) + + return multierr.Combine(typeErr, setConfigErr) + }, nil +} From 36b327b57b320d48db272812d186252c4d27695a Mon Sep 17 00:00:00 2001 From: laser Date: Thu, 11 Jun 2020 13:18:18 -0700 Subject: [PATCH 44/81] various symbol renames --- api/api_storage.go | 2 +- api/apistruct/struct.go | 10 +++++----- cmd/lotus-storage-miner/market.go | 4 ++-- node/builder.go | 5 +++-- node/config/def.go | 4 ++-- node/impl/storminer.go | 6 +++--- node/modules/dtypes/miner.go | 6 +++--- node/modules/storageminer.go | 14 +++++++------- 8 files changed, 26 insertions(+), 25 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index e414ed353..04ff8311c 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -54,7 +54,7 @@ type StorageMiner interface { DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) - DealsSetIsAcceptingStorageDeals(ctx context.Context, b bool) error + DealsSetAcceptingStorageDeals(context.Context, bool) error StorageAddLocal(ctx context.Context, path string) error } diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 23164a993..199ad2357 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -220,9 +220,9 @@ type StorageMinerStruct struct { StorageLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error `perm:"admin"` StorageTryLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) (bool, error) `perm:"admin"` - DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` - DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` - DealsSetIsAcceptingStorageDeals func(ctx context.Context, b bool) error `perm:"admin"` + DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` + DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` + DealsSetAcceptingStorageDeals func(context.Context, bool) error `perm:"admin"` StorageAddLocal func(ctx context.Context, path string) error `perm:"admin"` } @@ -853,8 +853,8 @@ func (c *StorageMinerStruct) DealsList(ctx context.Context) ([]storagemarket.Sto return c.Internal.DealsList(ctx) } -func (c *StorageMinerStruct) DealsSetIsAcceptingStorageDeals(ctx context.Context, b bool) error { - return c.Internal.DealsSetIsAcceptingStorageDeals(ctx, b) +func (c *StorageMinerStruct) DealsSetAcceptingStorageDeals(ctx context.Context, b bool) error { + return c.Internal.DealsSetAcceptingStorageDeals(ctx, b) } func (c *StorageMinerStruct) StorageAddLocal(ctx context.Context, path string) error { diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 4b9f903c8..c2f2555fa 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -21,7 +21,7 @@ var enableCmd = &cli.Command{ } defer closer() - return api.DealsSetIsAcceptingStorageDeals(lcli.DaemonContext(cctx), true) + return api.DealsSetAcceptingStorageDeals(lcli.DaemonContext(cctx), true) }, } @@ -36,7 +36,7 @@ var disableCmd = &cli.Command{ } defer closer() - return api.DealsSetIsAcceptingStorageDeals(lcli.DaemonContext(cctx), false) + return api.DealsSetAcceptingStorageDeals(lcli.DaemonContext(cctx), false) }, } diff --git a/node/builder.go b/node/builder.go index bf42eabd6..e628e999b 100644 --- a/node/builder.go +++ b/node/builder.go @@ -313,8 +313,9 @@ func Online() Option { Override(HandleDealsKey, modules.HandleDeals), Override(new(gen.WinningPoStProver), storage.NewWinningPoStProver), Override(new(*miner.Miner), modules.SetupBlockProducer), - Override(new(dtypes.IsAcceptingStorageDealsFunc), modules.NewIsAcceptingStorageDealsFunc), - Override(new(dtypes.SetAcceptingStorageDealsFunc), modules.NewSetAcceptingStorageDealsFunc), + + Override(new(dtypes.AcceptingStorageDealsConfigFunc), modules.NewAcceptingStorageDealsConfigFunc), + Override(new(dtypes.SetAcceptingStorageDealsConfigFunc), modules.NewSetAcceptingStorageDealsConfigFunc), ), ) } diff --git a/node/config/def.go b/node/config/def.go index 3debfd64e..651e99aed 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -32,7 +32,7 @@ type StorageMiner struct { } type DealmakingConfig struct { - IsAcceptingStorageDeals bool + AcceptingStorageDeals bool } // API contains configs for API endpoint @@ -116,7 +116,7 @@ func DefaultStorageMiner() *StorageMiner { }, Dealmaking: DealmakingConfig{ - IsAcceptingStorageDeals: true, + AcceptingStorageDeals: true, }, } cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" diff --git a/node/impl/storminer.go b/node/impl/storminer.go index b88541886..de80eb4cd 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -43,7 +43,7 @@ type StorageMinerAPI struct { StorageMgr *sectorstorage.Manager `optional:"true"` *stores.Index - SetAcceptingStorageDealsFunc dtypes.SetAcceptingStorageDealsFunc + SetAcceptingStorageDealsConfigFunc dtypes.SetAcceptingStorageDealsConfigFunc } func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) { @@ -209,8 +209,8 @@ func (sm *StorageMinerAPI) DealsList(ctx context.Context) ([]storagemarket.Stora return sm.StorageProvider.ListDeals(ctx) } -func (sm *StorageMinerAPI) DealsSetIsAcceptingStorageDeals(ctx context.Context, b bool) error { - return sm.SetAcceptingStorageDealsFunc(b) +func (sm *StorageMinerAPI) DealsSetAcceptingStorageDeals(ctx context.Context, b bool) error { + return sm.SetAcceptingStorageDealsConfigFunc(b) } func (sm *StorageMinerAPI) DealsImportData(ctx context.Context, deal cid.Cid, fname string) error { diff --git a/node/modules/dtypes/miner.go b/node/modules/dtypes/miner.go index d14a8775e..5c761d3e5 100644 --- a/node/modules/dtypes/miner.go +++ b/node/modules/dtypes/miner.go @@ -8,10 +8,10 @@ import ( type MinerAddress address.Address type MinerID abi.ActorID -// IsAcceptingStorageDealsFunc is a function which reads from miner config to +// AcceptingStorageDealsFunc is a function which reads from miner config to // determine if the user has disabled storage deals (or not). -type IsAcceptingStorageDealsFunc func() (bool, error) +type AcceptingStorageDealsConfigFunc func() (bool, error) // SetAcceptingStorageDealsFunc is a function which is used to disable or enable // storage deal acceptance. -type SetAcceptingStorageDealsFunc func(bool) error +type SetAcceptingStorageDealsConfigFunc func(bool) error diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 9847f7cc2..440aa8593 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -307,7 +307,7 @@ func NewStorageAsk(ctx helpers.MetricsCtx, fapi lapi.FullNode, ds dtypes.Metadat return storedAsk, nil } -func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Config, storedAsk *storedask.StoredAsk, h host.Host, ds dtypes.MetadataDS, ibs dtypes.StagingBlockstore, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode, isAcceptingStorageDealsFunc dtypes.IsAcceptingStorageDealsFunc) (storagemarket.StorageProvider, error) { +func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Config, storedAsk *storedask.StoredAsk, h host.Host, ds dtypes.MetadataDS, ibs dtypes.StagingBlockstore, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode, isAcceptingFunc dtypes.AcceptingStorageDealsConfigFunc) (storagemarket.StorageProvider, error) { net := smnet.NewFromLibp2pHost(h) store, err := piecefilestore.NewLocalFileStore(piecefilestore.OsPath(r.Path())) if err != nil { @@ -315,12 +315,12 @@ func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Con } opt := storageimpl.CustomDealDecisionLogic(func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) { - willEntertainProposals, err := isAcceptingStorageDealsFunc() + b, err := isAcceptingFunc() if err != nil { return false, "miner error", err } - if !willEntertainProposals { + if !b { log.Warnf("storage deal acceptance disabled; rejecting storage deal proposal from client: %s", deal.Client.String()) return false, "miner is not accepting storage deals", nil } @@ -379,7 +379,7 @@ func StorageAuth(ctx helpers.MetricsCtx, ca lapi.Common) (sectorstorage.StorageA return sectorstorage.StorageAuth(headers), nil } -func NewIsAcceptingStorageDealsFunc(r repo.LockedRepo) (dtypes.IsAcceptingStorageDealsFunc, error) { +func NewAcceptingStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.AcceptingStorageDealsConfigFunc, error) { return func() (bool, error) { raw, err := r.Config() if err != nil { @@ -391,11 +391,11 @@ func NewIsAcceptingStorageDealsFunc(r repo.LockedRepo) (dtypes.IsAcceptingStorag return false, xerrors.New("expected address of config.StorageMiner") } - return cfg.Dealmaking.IsAcceptingStorageDeals, nil + return cfg.Dealmaking.AcceptingStorageDeals, nil }, nil } -func NewSetAcceptingStorageDealsFunc(r repo.LockedRepo) (dtypes.SetAcceptingStorageDealsFunc, error) { +func NewSetAcceptingStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.SetAcceptingStorageDealsConfigFunc, error) { return func(b bool) error { var typeErr error @@ -406,7 +406,7 @@ func NewSetAcceptingStorageDealsFunc(r repo.LockedRepo) (dtypes.SetAcceptingStor return } - cfg.Dealmaking.IsAcceptingStorageDeals = b + cfg.Dealmaking.AcceptingStorageDeals = b }) return multierr.Combine(typeErr, setConfigErr) From 071ddf6563f37ef1d0a361102ed95f8411153c8a Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 11 Jun 2020 22:25:24 +0200 Subject: [PATCH 45/81] Add aggregate number, add number of bytes for storage calls Signed-off-by: Jakub Sztandera --- chain/vm/gas_v0.go | 4 ++-- cli/state.go | 24 +++++++++++++++++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/chain/vm/gas_v0.go b/chain/vm/gas_v0.go index 0b6196377..c631beaee 100644 --- a/chain/vm/gas_v0.go +++ b/chain/vm/gas_v0.go @@ -108,12 +108,12 @@ func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.M // OnIpldGet returns the gas used for storing an object func (pl *pricelistV0) OnIpldGet(dataSize int) GasCharge { - return newGasCharge("OnIpldGet", pl.ipldGetBase+int64(dataSize)*pl.ipldGetPerByte, 0) + return newGasCharge(fmt.Sprintf("OnIpldGet:%db", dataSize), pl.ipldGetBase+int64(dataSize)*pl.ipldGetPerByte, 0) } // OnIpldPut returns the gas used for storing an object func (pl *pricelistV0) OnIpldPut(dataSize int) GasCharge { - return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte) + return newGasCharge(fmt.Sprintf("OnIpldPut:%db", dataSize), pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte) } // OnCreateActor returns the gas used for creating an actor diff --git a/cli/state.go b/cli/state.go index c65eb1713..acc648c23 100644 --- a/cli/state.go +++ b/cli/state.go @@ -975,7 +975,11 @@ func computeStateHtml(ts *types.TipSet, o *api.ComputeStateOutput, getCode func( } .slow-true-false { color: #660; } .slow-true-true { color: #f80; } - table { font-size: 12px; } + table { + font-size: 12px; + border-collapse: collapse; + } + tr.sum { border-top: 1px solid black; } @@ -1029,10 +1033,20 @@ func computeStateHtml(ts *types.TipSet, o *api.ComputeStateOutput, getCode func( } fmt.Printf("\n
Gas Trace" + "") + + var sumTotal, sumCompute, sumStorage int64 + var sumTime time.Duration for _, gc := range ir.ExecutionTrace.GasCharges { fmt.Printf("", gc.Name, gc.TotalGas, gc.ComputeGas, gc.StorageGas, gc.TimeTaken, gc.Location) + sumTotal += gc.TotalGas + sumCompute += gc.ComputeGas + sumStorage += gc.StorageGas + sumTime += gc.TimeTaken } + fmt.Printf("", + "Sum", sumTotal, sumCompute, sumStorage, sumTime, "") + fmt.Printf("
NameTotal/Compute/StorageTime TakenLocation
%s%d/%d/%d%s%s
%s%d/%d/%d%s%s
\n") fmt.Println("
Execution trace:
") @@ -1092,10 +1106,18 @@ func printInternalExecutionsHtml(hashName string, trace []types.ExecutionTrace, } fmt.Printf("\n
Gas Trace" + "") + var sumTotal, sumCompute, sumStorage int64 + var sumTime time.Duration for _, gc := range im.GasCharges { fmt.Printf("", gc.Name, gc.TotalGas, gc.ComputeGas, gc.StorageGas, gc.TimeTaken, gc.Location) + sumTotal += gc.TotalGas + sumCompute += gc.ComputeGas + sumStorage += gc.StorageGas + sumTime += gc.TimeTaken } + fmt.Printf("", + "Sum", sumTotal, sumCompute, sumStorage, sumTime, "") fmt.Printf("
NameTotal/Compute/StorageTime TakenLocation
%s%d/%d/%d%s%s
%s%d/%d/%d%s%s
\n") if len(im.Subcalls) > 0 { fmt.Println("
Subcalls:
") From c7e3a5cff4e3c3e0cec5027dde55610c054670bb Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 11 Jun 2020 17:11:38 -0700 Subject: [PATCH 46/81] fix null incrementing --- miner/miner.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/miner/miner.go b/miner/miner.go index 23f40338d..bdeed8ac5 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -188,6 +188,8 @@ func (m *Miner) mine(ctx context.Context) { log.Errorf("failed to submit newly mined block: %s", err) } } else { + base.NullRounds++ + // Wait until the next epoch, plus the propagation delay, so a new tipset // has enough time to form. // @@ -261,7 +263,6 @@ func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*types.BlockMsg, return nil, xerrors.Errorf("failed to get mining base info: %w", err) } if mbi == nil { - base.NullRounds++ return nil, nil } @@ -278,7 +279,6 @@ func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*types.BlockMsg, } if !hasPower { // slashed or just have no power yet - base.NullRounds++ return nil, nil } @@ -302,7 +302,6 @@ func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*types.BlockMsg, } if winner == nil { - base.NullRounds++ return nil, nil } From 2c401f90419535543e8f811c9ff4062959ff2e6c Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 12 Jun 2020 01:13:11 -0400 Subject: [PATCH 47/81] Improve UX of fetch-params --- .circleci/config.yml | 2 +- cli/params.go | 18 ++++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ddcbd5481..fc9f495a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,7 +45,7 @@ commands: - 'v25-2k-lotus-params' paths: - /var/tmp/filecoin-proof-parameters/ - - run: ./lotus fetch-params --proving-params 2048 + - run: ./lotus fetch-params 2048 - save_cache: name: Save parameters cache key: 'v25-2k-lotus-params' diff --git a/cli/params.go b/cli/params.go index 95596ff57..47e1e3988 100644 --- a/cli/params.go +++ b/cli/params.go @@ -10,18 +10,16 @@ import ( ) var fetchParamCmd = &cli.Command{ - Name: "fetch-params", - Usage: "Fetch proving parameters", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "proving-params", - Usage: "download params used creating proofs for given size, i.e. 32GiB", - }, - }, + Name: "fetch-params", + Usage: "Fetch proving parameters", + ArgsUsage: "[sectorSize]", Action: func(cctx *cli.Context) error { - sectorSizeInt, err := units.RAMInBytes(cctx.String("proving-params")) + if !cctx.Args().Present() { + return xerrors.Errorf("must pass sector size to fetch params for (specify as \"32GiB\", for instance)") + } + sectorSizeInt, err := units.RAMInBytes(cctx.Args().First()) if err != nil { - return err + return xerrors.Errorf("error parsing sector size (specify as \"32GiB\", for instance): %w", err) } sectorSize := uint64(sectorSizeInt) From 227819dc1e43fd9c64022fc3b79cb2ca907bc62c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 12 Jun 2020 10:03:31 +0200 Subject: [PATCH 48/81] docs: Update local-dev-net --- documentation/en/local-dev-net.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/en/local-dev-net.md b/documentation/en/local-dev-net.md index 0b352fe8a..e11d9b358 100644 --- a/documentation/en/local-dev-net.md +++ b/documentation/en/local-dev-net.md @@ -8,7 +8,7 @@ make 2k Download the 2048 byte parameters: ```sh -./lotus fetch-params --proving-params 2048 +./lotus fetch-params 2048 ``` Pre-seal some sectors: From bd25d6db06e5c7060a2884282c1ddbcd9a8cf959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 12 Jun 2020 13:24:38 +0200 Subject: [PATCH 49/81] compute-state html: Convert to templates --- cli/state.go | 285 ++++++++++++++++++++++++--------------------------- 1 file changed, 134 insertions(+), 151 deletions(-) diff --git a/cli/state.go b/cli/state.go index acc648c23..47a1624c4 100644 --- a/cli/state.go +++ b/cli/state.go @@ -5,10 +5,12 @@ import ( "context" "encoding/json" "fmt" + "os" "reflect" "sort" "strconv" "strings" + "text/template" "time" "github.com/ipfs/go-cid" @@ -31,6 +33,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + "github.com/filecoin-project/specs-actors/actors/runtime/exitcode" "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" @@ -40,7 +43,7 @@ import ( ) type methodMeta struct { - name string + Name string params reflect.Type ret reflect.Type @@ -68,7 +71,7 @@ func init() { nf := rt.NumField() methods[c] = append(methods[c], methodMeta{ - name: "Send", + Name: "Send", params: reflect.TypeOf(new(adt.EmptyValue)), ret: reflect.TypeOf(new(adt.EmptyValue)), }) @@ -78,7 +81,7 @@ func init() { export := reflect.TypeOf(exports[i+1]) methods[c] = append(methods[c], methodMeta{ - name: rt.Field(i).Name, + Name: rt.Field(i).Name, params: export.In(1), ret: export.Out(0), }) @@ -923,7 +926,7 @@ var stateComputeStateCmd = &cli.Command{ return c.Code, nil } - return computeStateHtml(ts, stout, getCode) + return computeStateHtmlT(ts, stout, getCode) } fmt.Println("computed state cid: ", stout.Root) @@ -944,16 +947,8 @@ func printInternalExecutions(prefix string, trace []types.ExecutionTrace) { } } -func codeStr(c cid.Cid) string { - cmh, err := multihash.Decode(c.Hash()) - if err != nil { - panic(err) - } - return string(cmh.Digest) -} - -func computeStateHtml(ts *types.TipSet, o *api.ComputeStateOutput, getCode func(addr address.Address) (cid.Cid, error)) error { - fmt.Printf(` +var compStateTemplate = ` + -
Tipset: %s
-
Height: %d
-
State CID: %s
-
Calls
`, ts.Key(), ts.Height(), o.Root) +
Tipset: {{.TipSet.Key}}
+
Epoch: {{.TipSet.Height}}
+
State CID: {{.Comp.Root}}
+
Calls
+ {{range .Comp.Trace}} + {{template "message" (Call .ExecutionTrace false .Msg.Cid.String)}} + {{end}} + + +` - for _, ir := range o.Trace { - toCode, err := getCode(ir.Msg.To) - if err != nil { - return xerrors.Errorf("getting code for %s: %w", toCode, err) - } +var compStateMsg = ` +
+ {{$code := GetCode .Msg.To}} + {{$h := "h2"}}{{if .Subcall}}{{$h = "h4"}}{{end}} - params, err := jsonParams(toCode, ir.Msg.Method, ir.Msg.Params) - if err != nil { - return xerrors.Errorf("decoding params: %w", err) - } + +
{{.Msg.From}} -> {{.Msg.To}} ({{ToFil .Msg.Value}} FIL), M{{.Msg.Method}}
+ {{if not .Subcall}}
Msg CID: {{.Msg.Cid}}
{{end}} + {{if gt (len .Msg.Params) 0}} + 
{{JsonParams ($code) (.Msg.Method) (.Msg.Params) | html}}
+ {{end}} +
Took {{.Duration}}, Exit: {{.MsgRct.ExitCode}}{{if gt (len .MsgRct.Return) 0}}, Return{{end}}
+ + {{if gt (len .MsgRct.Return) 0}} + 
{{JsonReturn ($code) (.Msg.Method) (.MsgRct.Return) | html}}
+ {{end}} - if len(ir.Msg.Params) != 0 { - params = `
` + params + `
` - } else { - params = "" - } + {{if ne .MsgRct.ExitCode 0}} +
Error: 
{{.Error}}
+ {{end}} - ret, err := jsonReturn(toCode, ir.Msg.Method, ir.MsgRct.Return) - if err != nil { - return xerrors.Errorf("decoding return value: %w", err) - } +
+ Gas Trace + + + {{range .GasCharges}} + + {{end}} + {{with SumGas .GasCharges}} + + {{end}} +
NameTotal/Compute/StorageTime TakenLocation
{{.Name}}{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}}{{.Location}}
Sum{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}}
+
+ {{if gt (len .Subcalls) 0}} +
Subcalls:
+ {{$hash := .Hash}} + {{range .Subcalls}} + {{template "message" (Call . true (printf "%s-%s" $hash .Msg.Cid.String))}} + {{end}} + {{end}} +
` - if len(ir.MsgRct.Return) == 0 { - ret = "" - } else { - ret = `, Return
` + ret + `
` - } - - slow := ir.Duration > 10*time.Millisecond - veryslow := ir.Duration > 50*time.Millisecond - - cid := ir.Msg.Cid() - - fmt.Printf(`
- -
%s -> %s (%s FIL), M%d
-
Msg CID: %s
-%s -
Took %s, Exit: %d%s -`, cid, cid, codeStr(toCode), methods[toCode][ir.Msg.Method].name, ir.Msg.From, ir.Msg.To, types.FIL(ir.Msg.Value), ir.Msg.Method, cid, params, slow, veryslow, ir.Duration, ir.MsgRct.ExitCode, ir.MsgRct.ExitCode, ret) - if ir.MsgRct.ExitCode != 0 { - fmt.Printf(`
Error: 
%s
`, ir.Error) - } - fmt.Printf("\n
Gas Trace" + - "") - - var sumTotal, sumCompute, sumStorage int64 - var sumTime time.Duration - for _, gc := range ir.ExecutionTrace.GasCharges { - fmt.Printf("", - gc.Name, gc.TotalGas, gc.ComputeGas, gc.StorageGas, gc.TimeTaken, gc.Location) - sumTotal += gc.TotalGas - sumCompute += gc.ComputeGas - sumStorage += gc.StorageGas - sumTime += gc.TimeTaken - } - fmt.Printf("", - "Sum", sumTotal, sumCompute, sumStorage, sumTime, "") - - fmt.Printf("
NameTotal/Compute/StorageTime TakenLocation
%s%d/%d/%d%s%s
%s%d/%d/%d%s%s
\n") - - fmt.Println("
Execution trace:
") - if err := printInternalExecutionsHtml(cid.String(), ir.ExecutionTrace.Subcalls, getCode); err != nil { - return err - } - fmt.Println("
") - } - - fmt.Printf(` -`) - return nil +type compStateHtmlIn struct { + TipSet *types.TipSet + Comp *api.ComputeStateOutput } -func printInternalExecutionsHtml(hashName string, trace []types.ExecutionTrace, getCode func(addr address.Address) (cid.Cid, error)) error { - for i, im := range trace { - hashName := fmt.Sprintf("%s-r%d", hashName, i) - - toCode, err := getCode(im.Msg.To) - if err != nil { - return xerrors.Errorf("getting code for %s: %w", toCode, err) - } - - params, err := jsonParams(toCode, im.Msg.Method, im.Msg.Params) - if err != nil { - return xerrors.Errorf("decoding params: %w", err) - } - - if len(im.Msg.Params) != 0 { - params = `
` + params + `
` - } else { - params = "" - } - - ret, err := jsonReturn(toCode, im.Msg.Method, im.MsgRct.Return) - if err != nil { - return xerrors.Errorf("decoding return value: %w", err) - } - - if len(im.MsgRct.Return) == 0 { - ret = "
" - } else { - ret = `, Return
` + ret + `
` - } - - slow := im.Duration > 10*time.Millisecond - veryslow := im.Duration > 50*time.Millisecond - - fmt.Printf(`
- -
%s -> %s (%s FIL), M%d
-%s -
Took %s, Exit: %d%s -`, hashName, hashName, codeStr(toCode), methods[toCode][im.Msg.Method].name, im.Msg.From, im.Msg.To, types.FIL(im.Msg.Value), im.Msg.Method, params, slow, veryslow, im.Duration, im.MsgRct.ExitCode, im.MsgRct.ExitCode, ret) - if im.MsgRct.ExitCode != 0 { - fmt.Printf(`
Error: 
%s
`, im.Error) - } - fmt.Printf("\n
Gas Trace" + - "") - var sumTotal, sumCompute, sumStorage int64 - var sumTime time.Duration - for _, gc := range im.GasCharges { - fmt.Printf("", - gc.Name, gc.TotalGas, gc.ComputeGas, gc.StorageGas, gc.TimeTaken, gc.Location) - sumTotal += gc.TotalGas - sumCompute += gc.ComputeGas - sumStorage += gc.StorageGas - sumTime += gc.TimeTaken - } - fmt.Printf("", - "Sum", sumTotal, sumCompute, sumStorage, sumTime, "") - fmt.Printf("
NameTotal/Compute/StorageTime TakenLocation
%s%d/%d/%d%s%s
%s%d/%d/%d%s%s
\n") - if len(im.Subcalls) > 0 { - fmt.Println("
Subcalls:
") - if err := printInternalExecutionsHtml(hashName, im.Subcalls, getCode); err != nil { - return err - } - } - fmt.Println("
") +func computeStateHtmlT(ts *types.TipSet, o *api.ComputeStateOutput, getCode func(addr address.Address) (cid.Cid, error)) error { + t, err := template.New("compute_state").Funcs(map[string]interface{}{ + "GetCode": getCode, + "GetMethod": getMethod, + "ToFil": toFil, + "JsonParams": jsonParams, + "JsonReturn": jsonReturn, + "IsSlow": isSlow, + "IsVerySlow": isVerySlow, + "IntExit": func(i exitcode.ExitCode) int64 { return int64(i) }, + "SumGas": sumGas, + "CodeStr": codeStr, + "Call": call, + }).Parse(compStateTemplate) + if err != nil { + return err + } + t, err = t.New("message").Parse(compStateMsg) + if err != nil { + return err } - return nil + return t.ExecuteTemplate(os.Stdout, "compute_state", &compStateHtmlIn{ + TipSet: ts, + Comp: o, + }) +} + +type callMeta struct { + types.ExecutionTrace + Subcall bool + Hash string +} + +func call(e types.ExecutionTrace, subcall bool, hash string) callMeta { + return callMeta{ + ExecutionTrace: e, + Subcall: subcall, + Hash: hash, + } +} + +func codeStr(c cid.Cid) string { + cmh, err := multihash.Decode(c.Hash()) + if err != nil { + panic(err) + } + return string(cmh.Digest) +} + +func getMethod(code cid.Cid, method abi.MethodNum) string { + return methods[code][method].Name +} + +func toFil(f types.BigInt) types.FIL { + return types.FIL(f) +} + +func isSlow(t time.Duration) bool { + return t > 10*time.Millisecond +} + +func isVerySlow(t time.Duration) bool { + return t > 50*time.Millisecond +} + +func sumGas(changes []*types.GasTrace) types.GasTrace { + var out types.GasTrace + for _, gc := range changes { + out.TotalGas += gc.TotalGas + out.ComputeGas += gc.ComputeGas + out.StorageGas += gc.StorageGas + out.TimeTaken += gc.TimeTaken + } + + return out } func jsonParams(code cid.Cid, method abi.MethodNum, params []byte) (string, error) { From 813940762c58e26393e343f7f1f7b40fdfa03748 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 12 Jun 2020 14:46:50 +0200 Subject: [PATCH 50/81] Use html/template Signed-off-by: Jakub Sztandera --- cli/state.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/cli/state.go b/cli/state.go index 47a1624c4..3d92f9000 100644 --- a/cli/state.go +++ b/cli/state.go @@ -5,12 +5,12 @@ import ( "context" "encoding/json" "fmt" + "html/template" "os" "reflect" "sort" "strconv" "strings" - "text/template" "time" "github.com/ipfs/go-cid" @@ -992,9 +992,22 @@ var compStateTemplate = ` var compStateMsg = `
{{$code := GetCode .Msg.To}} - {{$h := "h2"}}{{if .Subcall}}{{$h = "h4"}}{{end}} + -
{{.Msg.From}} -> {{.Msg.To}} ({{ToFil .Msg.Value}} FIL), M{{.Msg.Method}}
{{if not .Subcall}}
Msg CID: {{.Msg.Cid}}
{{end}} {{if gt (len .Msg.Params) 0}} @@ -1049,6 +1062,9 @@ func computeStateHtmlT(ts *types.TipSet, o *api.ComputeStateOutput, getCode func "SumGas": sumGas, "CodeStr": codeStr, "Call": call, + "htmlSafeAttr": func(html string) template.HTMLAttr { + return template.HTMLAttr(html) + }, }).Parse(compStateTemplate) if err != nil { return err From 00385acc1d2561e49ec395086f7780c8e66706ab Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 12 Jun 2020 15:26:58 +0200 Subject: [PATCH 51/81] Remove htmlSafeAttr Signed-off-by: Jakub Sztandera --- cli/state.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cli/state.go b/cli/state.go index 3d92f9000..e2ec3b6d1 100644 --- a/cli/state.go +++ b/cli/state.go @@ -1062,9 +1062,6 @@ func computeStateHtmlT(ts *types.TipSet, o *api.ComputeStateOutput, getCode func "SumGas": sumGas, "CodeStr": codeStr, "Call": call, - "htmlSafeAttr": func(html string) template.HTMLAttr { - return template.HTMLAttr(html) - }, }).Parse(compStateTemplate) if err != nil { return err From 82eb7786d464d963c9e2be23672608e09d07d918 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 12 Jun 2020 16:25:55 +0200 Subject: [PATCH 52/81] Cleanup lint warnings Signed-off-by: Jakub Sztandera --- cli/state.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/state.go b/cli/state.go index e2ec3b6d1..96114864f 100644 --- a/cli/state.go +++ b/cli/state.go @@ -926,7 +926,7 @@ var stateComputeStateCmd = &cli.Command{ return c.Code, nil } - return computeStateHtmlT(ts, stout, getCode) + return computeStateHTMLTempl(ts, stout, getCode) } fmt.Println("computed state cid: ", stout.Root) @@ -1044,12 +1044,12 @@ var compStateMsg = ` {{end}}
` -type compStateHtmlIn struct { +type compStateHTMLIn struct { TipSet *types.TipSet Comp *api.ComputeStateOutput } -func computeStateHtmlT(ts *types.TipSet, o *api.ComputeStateOutput, getCode func(addr address.Address) (cid.Cid, error)) error { +func computeStateHTMLTempl(ts *types.TipSet, o *api.ComputeStateOutput, getCode func(addr address.Address) (cid.Cid, error)) error { t, err := template.New("compute_state").Funcs(map[string]interface{}{ "GetCode": getCode, "GetMethod": getMethod, @@ -1071,7 +1071,7 @@ func computeStateHtmlT(ts *types.TipSet, o *api.ComputeStateOutput, getCode func return err } - return t.ExecuteTemplate(os.Stdout, "compute_state", &compStateHtmlIn{ + return t.ExecuteTemplate(os.Stdout, "compute_state", &compStateHTMLIn{ TipSet: ts, Comp: o, }) From ca8a00f8a15d2277c142310a448ec069ecbd6ad0 Mon Sep 17 00:00:00 2001 From: Rob Quist Date: Fri, 12 Jun 2020 18:19:04 +0200 Subject: [PATCH 53/81] Update hardware-mining.md --- documentation/en/hardware-mining.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/en/hardware-mining.md b/documentation/en/hardware-mining.md index a7c410184..0379cc4e1 100644 --- a/documentation/en/hardware-mining.md +++ b/documentation/en/hardware-mining.md @@ -16,7 +16,7 @@ The setup below is a minimal example for sealing 32 GiB sectors on Lotus: Note that 1GB sectors don't require as high of specs, but are likely to be removed as we improve the performance of 32GB sector sealing. -AMD CPU's are __highly recommended__, because of the `Intel SHA Extensions` instruction set that is available there since the `Zen` microarchitecture. Hence, AMD CPU's seem to perform much better on the testnet than other CPU's. Contrary to what the name implies, this extended instruction set is not available on recent Intel desktop/server chips. +For the first part of the sealing process, AMD CPU's are __highly recommended__, because of the `Intel SHA Extensions` instruction set that is available there ever since the `Zen` microarchitecture. Hence, AMD CPU's seem to perform much better on the testnet than other CPU's. Contrary to what the name implies, this extended instruction set is not available on recent Intel desktop/server chips. ## Testnet discoveries @@ -35,7 +35,7 @@ GPUs are a must for getting **block rewards**. Here are a few that have been con ## Testing other GPUs -If you want to test a GPU that is not explicitly supported, use the following global**environment variable**: +If you want to test a GPU that is not explicitly supported, use the following global **environment variable**: ```sh BELLMAN_CUSTOM_GPU=":" From cd69e57a33513089aaf142cd0a4fbab0c95cefdf Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 12 Jun 2020 18:49:29 +0200 Subject: [PATCH 54/81] Expose more callers, ellipsis unimportant ones Signed-off-by: Jakub Sztandera --- chain/types/execresult.go | 68 +++++++++++++++++++++++++++++++++------ chain/vm/runtime.go | 31 ++++++++++++------ chain/vm/vm.go | 4 +-- cli/state.go | 48 ++++++++++++++++++++++++--- 4 files changed, 125 insertions(+), 26 deletions(-) diff --git a/chain/types/execresult.go b/chain/types/execresult.go index 41372b832..57e14dcf9 100644 --- a/chain/types/execresult.go +++ b/chain/types/execresult.go @@ -19,36 +19,84 @@ type ExecutionTrace struct { } type GasTrace struct { - Name string - Location string + Name string + + Location []Loc TotalGas int64 ComputeGas int64 StorageGas int64 TimeTaken time.Duration + Extra interface{} Callers []uintptr `json:"-"` } +type Loc struct { + File string + Line int + Function string +} + +func (l Loc) Show() bool { + ignorePrefix := []string{ + "reflect.", + "github.com/filecoin-project/lotus/chain/vm.(*Invoker).transform", + "github.com/filecoin-project/go-amt-ipld/", + } + for _, pre := range ignorePrefix { + if strings.HasPrefix(l.Function, pre) { + return false + } + } + return true +} +func (l Loc) String() string { + file := strings.Split(l.File, "/") + + fn := strings.Split(l.Function, "/") + var fnpkg string + if len(fn) > 2 { + fnpkg = strings.Join(fn[len(fn)-2:], "/") + } else { + fnpkg = l.Function + } + + return fmt.Sprintf("%s@%s:%d", fnpkg, file[len(file)-1], l.Line) +} + +func (l Loc) Important() bool { + if strings.HasPrefix(l.Function, "github.com/filecoin-project/specs-actors/actors/builtin") { + return true + } + return false +} + func (gt *GasTrace) MarshalJSON() ([]byte, error) { type GasTraceCopy GasTrace - if gt.Location == "" { + if len(gt.Location) == 0 { if len(gt.Callers) != 0 { frames := runtime.CallersFrames(gt.Callers) for { frame, more := frames.Next() - fn := strings.Split(frame.Function, "/") + if frame.Function == "github.com/filecoin-project/lotus/chain/vm.(*VM).ApplyMessage" { + break + } + l := Loc{ + File: frame.File, + Line: frame.Line, + Function: frame.Function, + } + //fn := strings.Split(frame.Function, "/") - split := strings.Split(frame.File, "/") - file := strings.Join(split[len(split)-2:], "/") - gt.Location += fmt.Sprintf("%s@%s:%d", fn[len(fn)-1], file, frame.Line) + //split := strings.Split(frame.File, "/") + //file := strings.Join(split[len(split)-2:], "/") + //gt.Location += fmt.Sprintf("%s@%s:%d", fn[len(fn)-1], file, frame.Line) + gt.Location = append(gt.Location, l) if !more { break } - gt.Location += "|" } - } else { - gt.Location = "n/a" } } diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 902ef94bb..67f7adc03 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -488,22 +488,33 @@ func (rt *Runtime) stateCommit(oldh, newh cid.Cid) aerrors.ActorError { return nil } -func (rt *Runtime) ChargeGas(gas GasCharge) { - err := rt.chargeGasInternal(gas) - if err != nil { - panic(err) - } -} func (rt *Runtime) finilizeGasTracing() { if rt.lastGasCharge != nil { rt.lastGasCharge.TimeTaken = time.Since(rt.lastGasChargeTime) } } -func (rt *Runtime) chargeGasInternal(gas GasCharge) aerrors.ActorError { +func (rt *Runtime) ChargeGas(gas GasCharge) { + err := rt.chargeGasInternal(gas, 1) + if err != nil { + panic(err) + } +} + +func (rt *Runtime) chargeGasFunc(skip int) func(GasCharge) { + return func(gas GasCharge) { + err := rt.chargeGasInternal(gas, 1+skip) + if err != nil { + panic(err) + } + } + +} + +func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError { toUse := gas.Total() - var callers [3]uintptr - cout := gruntime.Callers(3, callers[:]) + var callers [10]uintptr + cout := gruntime.Callers(2+skip, callers[:]) now := time.Now() if rt.lastGasCharge != nil { @@ -530,7 +541,7 @@ func (rt *Runtime) chargeGasInternal(gas GasCharge) aerrors.ActorError { } func (rt *Runtime) chargeGasSafe(gas GasCharge) aerrors.ActorError { - return rt.chargeGasInternal(gas) + return rt.chargeGasInternal(gas, 1) } func (rt *Runtime) Pricelist() Pricelist { diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 75bbdfedb..acae3cde0 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -107,12 +107,12 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, origin addres } rt.cst = &cbor.BasicIpldStore{ - Blocks: &gasChargingBlocks{rt.ChargeGas, rt.pricelist, vm.cst.Blocks}, + Blocks: &gasChargingBlocks{rt.chargeGasFunc(2), rt.pricelist, vm.cst.Blocks}, Atlas: vm.cst.Atlas, } rt.sys = pricedSyscalls{ under: vm.Syscalls, - chargeGas: rt.ChargeGas, + chargeGas: rt.chargeGasFunc(1), pl: rt.pricelist, } diff --git a/cli/state.go b/cli/state.go index 96114864f..f2185c4ab 100644 --- a/cli/state.go +++ b/cli/state.go @@ -972,9 +972,35 @@ var compStateTemplate = ` .slow-true-true { color: #f80; } table { font-size: 12px; - border-collapse: collapse; - } - tr.sum { border-top: 1px solid black; } + border-collapse: collapse; + } + tr { + border-top: 1px solid black; + border-bottom: 1px solid black; + } + tr.sum { border-top: 2px solid black; } + tr:first-child { border-top: none; } + tr:last-child { border-bottom: none; } + + + .ellipsis-content, + .ellipsis-toggle input { + display: none; + } + .ellipsis-toggle { + cursor: pointer; + } + /** + Checked State + **/ + + .ellipsis-toggle input:checked + .ellipsis { + display: none; + } + .ellipsis-toggle input:checked ~ .ellipsis-content { + display: inline; + background-color: #ddd; + } @@ -1028,7 +1054,21 @@ var compStateMsg = ` {{range .GasCharges}} - + + {{end}} {{with SumGas .GasCharges}} From 8ad2b3c3d946428bd10fb8f0cca2851601752838 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 12 Jun 2020 18:58:55 +0200 Subject: [PATCH 55/81] Removed commented out code Signed-off-by: Jakub Sztandera --- chain/types/execresult.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/chain/types/execresult.go b/chain/types/execresult.go index 57e14dcf9..7ffa8c219 100644 --- a/chain/types/execresult.go +++ b/chain/types/execresult.go @@ -87,11 +87,6 @@ func (gt *GasTrace) MarshalJSON() ([]byte, error) { Line: frame.Line, Function: frame.Function, } - //fn := strings.Split(frame.Function, "/") - - //split := strings.Split(frame.File, "/") - //file := strings.Join(split[len(split)-2:], "/") - //gt.Location += fmt.Sprintf("%s@%s:%d", fn[len(fn)-1], file, frame.Line) gt.Location = append(gt.Location, l) if !more { break From a2943a6b7ff963388c57e02ed2ad68806397b810 Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Fri, 12 Jun 2020 13:57:30 -0300 Subject: [PATCH 56/81] doc: active development branches --- README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b0d9867d9..cd64073a6 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,19 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more details about Filecoin, check out the [Filecoin Spec](https://github.com/filecoin-project/specs). -## Development - -All work is tracked via issues. An attempt at keeping an up-to-date view on remaining work is in the [lotus testnet github project board](https://github.com/filecoin-project/lotus/projects/1). - ## Building & Documentation For instructions on how to build lotus from source, please visit [https://docs.lotu.sh](https://docs.lotu.sh) or read the source [here](https://github.com/filecoin-project/lotus/tree/master/documentation). +## Development + +All work is tracked via issues. An attempt at keeping an up-to-date view on remaining work is in the [lotus testnet github project board](https://github.com/filecoin-project/lotus/projects/1). + +The main branches under development at the moment are: +* [`master`](https://github.com/filecoin-project/lotus): current testnet. +* [`next`](https://github.com/filecoin-project/lotus/tree/next): working branch with chain-breaking changes. +* [`interopnet`](https://github.com/filecoin-project/lotus/tree/interopnet): devnet running one of `next` commits. + ## License Dual-licensed under [MIT](https://github.com/filecoin-project/lotus/blob/master/LICENSE-MIT) + [Apache 2.0](https://github.com/filecoin-project/lotus/blob/master/LICENSE-APACHE) From 4e9293ba04946ab3b3ef7bb8b11af55c07b17878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 12 Jun 2020 19:16:54 +0100 Subject: [PATCH 57/81] fix a potential race with chain reorgs notifees. --- chain/store/store.go | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index dba5995de..f0fb2a611 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -48,6 +48,9 @@ var log = logging.Logger("chainstore") var chainHeadKey = dstore.NewKey("head") +// ReorgNotifee represents a callback that gets called upon reorgs. +type ReorgNotifee func(rev, app []*types.TipSet) error + type ChainStore struct { bs bstore.Blockstore ds dstore.Datastore @@ -63,8 +66,8 @@ type ChainStore struct { cindex *ChainIndex - reorgCh chan<- reorg - headChangeNotifs []func(rev, app []*types.TipSet) error + reorgCh chan<- reorg + reorgNotifeeCh chan ReorgNotifee mmCache *lru.ARCCache tsCache *lru.ARCCache @@ -89,8 +92,6 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls runtime.Sys cs.cindex = ci - cs.reorgCh = cs.reorgWorker(context.TODO()) - hcnf := func(rev, app []*types.TipSet) error { cs.pubLk.Lock() defer cs.pubLk.Unlock() @@ -122,7 +123,8 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls runtime.Sys return nil } - cs.headChangeNotifs = append(cs.headChangeNotifs, hcnf, hcmetric) + cs.reorgNotifeeCh = make(chan ReorgNotifee) + cs.reorgCh = cs.reorgWorker(context.TODO(), []ReorgNotifee{hcnf, hcmetric}) return cs } @@ -211,8 +213,8 @@ func (cs *ChainStore) SubHeadChanges(ctx context.Context) chan []*api.HeadChange return out } -func (cs *ChainStore) SubscribeHeadChanges(f func(rev, app []*types.TipSet) error) { - cs.headChangeNotifs = append(cs.headChangeNotifs, f) +func (cs *ChainStore) SubscribeHeadChanges(f ReorgNotifee) { + cs.reorgNotifeeCh <- f } func (cs *ChainStore) SetGenesis(b *types.BlockHeader) error { @@ -273,13 +275,19 @@ type reorg struct { new *types.TipSet } -func (cs *ChainStore) reorgWorker(ctx context.Context) chan<- reorg { +func (cs *ChainStore) reorgWorker(ctx context.Context, initialNotifees []ReorgNotifee) chan<- reorg { out := make(chan reorg, 32) + notifees := make([]ReorgNotifee, len(initialNotifees)) + copy(notifees, initialNotifees) + go func() { defer log.Warn("reorgWorker quit") for { select { + case n := <-cs.reorgNotifeeCh: + notifees = append(notifees, n) + case r := <-out: revert, apply, err := cs.ReorgOps(r.old, r.new) if err != nil { @@ -293,7 +301,7 @@ func (cs *ChainStore) reorgWorker(ctx context.Context) chan<- reorg { apply[i], apply[opp] = apply[opp], apply[i] } - for _, hcf := range cs.headChangeNotifs { + for _, hcf := range notifees { if err := hcf(revert, apply); err != nil { log.Error("head change func errored (BAD): ", err) } From 91ea03907709e923ac9499144c40d6e44be37cab Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 12 Jun 2020 20:46:04 +0200 Subject: [PATCH 58/81] Change ellipsis, more accurate timing Signed-off-by: Jakub Sztandera --- chain/vm/runtime.go | 4 +- chain/vm/vm.go | 27 +++++++------- cli/state.go | 89 ++++++++++++++++++++++++++++++++------------- 3 files changed, 81 insertions(+), 39 deletions(-) diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 67f7adc03..be34f075c 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -371,6 +371,8 @@ func (rt *Runtime) Send(to address.Address, method abi.MethodNum, m vmr.CBORMars } func (rt *Runtime) internalSend(from, to address.Address, method abi.MethodNum, value types.BigInt, params []byte) ([]byte, aerrors.ActorError) { + + start := time.Now() ctx, span := trace.StartSpan(rt.ctx, "vmc.Send") defer span.End() if span.IsRecordingEvents() { @@ -396,7 +398,7 @@ func (rt *Runtime) internalSend(from, to address.Address, method abi.MethodNum, } defer st.ClearSnapshot() - ret, errSend, subrt := rt.vm.send(ctx, msg, rt, nil) + ret, errSend, subrt := rt.vm.send(ctx, msg, rt, nil, start) if errSend != nil { if errRevert := st.Revert(); errRevert != nil { return nil, aerrors.Escalate(errRevert, "failed to revert state tree after failed subcall") diff --git a/chain/vm/vm.go b/chain/vm/vm.go index acae3cde0..9a4c9991d 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -96,14 +96,13 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, origin addres originNonce: originNonce, height: vm.blockHeight, - gasUsed: usedGas, - gasAvailable: msg.GasLimit, - numActorsCreated: nac, - pricelist: PricelistByEpoch(vm.blockHeight), - allowInternal: true, - callerValidated: false, - executionTrace: types.ExecutionTrace{Msg: msg}, - lastGasChargeTime: time.Now(), + gasUsed: usedGas, + gasAvailable: msg.GasLimit, + numActorsCreated: nac, + pricelist: PricelistByEpoch(vm.blockHeight), + allowInternal: true, + callerValidated: false, + executionTrace: types.ExecutionTrace{Msg: msg}, } rt.cst = &cbor.BasicIpldStore{ @@ -172,8 +171,7 @@ type ApplyRet struct { } func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, - gasCharge *GasCharge) ([]byte, aerrors.ActorError, *Runtime) { - start := time.Now() + gasCharge *GasCharge, start time.Time) ([]byte, aerrors.ActorError, *Runtime) { st := vm.cstate @@ -189,6 +187,7 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, } rt := vm.makeRuntime(ctx, msg, origin, on, gasUsed, nac) + rt.lastGasChargeTime = start if parent != nil { rt.lastGasChargeTime = parent.lastGasChargeTime rt.lastGasCharge = parent.lastGasCharge @@ -273,7 +272,8 @@ func checkMessage(msg *types.Message) error { func (vm *VM) ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*ApplyRet, error) { start := time.Now() - ret, actorErr, rt := vm.send(ctx, msg, nil, nil) + ret, actorErr, rt := vm.send(ctx, msg, nil, nil, start) + rt.finilizeGasTracing() return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: aerrors.RetCode(actorErr), @@ -390,8 +390,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, } defer st.ClearSnapshot() - ret, actorErr, rt := vm.send(ctx, msg, nil, &msgGas) - rt.finilizeGasTracing() + ret, actorErr, rt := vm.send(ctx, msg, nil, &msgGas, start) if aerrors.IsFatal(actorErr) { return nil, xerrors.Errorf("[from=%s,to=%s,n=%d,m=%d,h=%d] fatal error: %w", msg.From, msg.To, msg.Nonce, msg.Method, vm.blockHeight, actorErr) } @@ -445,6 +444,8 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, return nil, xerrors.Errorf("gas handling math is wrong") } + rt.finilizeGasTracing() + return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: errcode, diff --git a/cli/state.go b/cli/state.go index f2185c4ab..d3d95675c 100644 --- a/cli/state.go +++ b/cli/state.go @@ -1001,6 +1001,12 @@ var compStateTemplate = ` display: inline; background-color: #ddd; } + hr { + border: none; + height: 1px; + background-color: black; + margin: 0; + } @@ -1049,32 +1055,54 @@ var compStateMsg = `
Error: 
{{.Error}}
{{end}} -
- Gas Trace -
NameTotal/Compute/StorageTime TakenLocation
{{.Name}}{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}}{{.Location}}
{{.Name}}{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}} + {{ range $index, $ele := .Location }} + {{- if $index }}|​{{end -}} + {{- if .Show -}} + {{- if .Important }}{{end -}} + {{- . -}} + {{- if .Important }}{{end -}} + {{- else -}} + + {{- end -}} + {{end}} +
Sum{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}}
- - {{range .GasCharges}} - - + {{end}} + {{with SumGas .GasCharges}} + + {{end}} +
NameTotal/Compute/StorageTime TakenLocation
{{.Name}}{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}} - {{ range $index, $ele := .Location }} - {{- if $index }}|​{{end -}} - {{- if .Show -}} - {{- if .Important }}{{end -}} - {{- . -}} - {{- if .Important }}{{end -}} - {{- else -}} - - {{- end -}} +
+Gas Trace + + + {{range .GasCharges}} + + - {{end}} - {{with SumGas .GasCharges}} - - {{end}} -
NameTotal/Compute/StorageTime TakenLocation
{{.Name}}{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}} + {{ $fImp := FirstImportant .Location }} + {{ if $fImp }} +
+ {{ $fImp }}
+ {{ $elipOn := false }} + {{ range $index, $ele := .Location -}} + {{- if $index }}
{{end -}} + {{- if .Show -}} + {{ if $elipOn }} + {{ $elipOn = false }} + + {{end}} + + {{- if .Important }}{{end -}} + {{- . -}} + {{if .Important }}{{end}} + {{else}} + {{ if not $elipOn }} + {{ $elipOn = true }} + {{end}} -
Sum{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}}
-
+ + {{end}} +
Sum{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}}
+ + + {{if gt (len .Subcalls) 0}}
Subcalls:
{{$hash := .Hash}} @@ -1102,6 +1130,17 @@ func computeStateHTMLTempl(ts *types.TipSet, o *api.ComputeStateOutput, getCode "SumGas": sumGas, "CodeStr": codeStr, "Call": call, + "FirstImportant": func(locs []types.Loc) *types.Loc { + if len(locs) != 0 { + for _, l := range locs { + if l.Important() { + return &l + } + } + return &locs[0] + } + return nil + }, }).Parse(compStateTemplate) if err != nil { return err From 82c6be14f44bc14c9667e264a0d35123cd09a6de Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Fri, 12 Jun 2020 22:34:00 -0300 Subject: [PATCH 59/81] unchecked errs Signed-off-by: Ignacio Hagopian --- chain/gen/genesis/miners.go | 3 +++ cli/chain.go | 3 +++ cli/state.go | 11 ++++++----- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index 51a128d54..26f52eae0 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -239,6 +239,9 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid st.TotalQualityAdjPower = big.Sub(st.TotalQualityAdjPower, big.NewInt(1)) return nil }) + if err != nil { + return cid.Undef, xerrors.Errorf("mutating state: %w", err) + } c, err := vm.Flush(ctx) if err != nil { diff --git a/cli/chain.go b/cli/chain.go index cbdd957e9..0dfcb926a 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -221,6 +221,9 @@ var chainStatObjCmd = &cli.Command{ base := cid.Undef if cctx.IsSet("base") { base, err = cid.Decode(cctx.String("base")) + if err != nil { + return err + } } stats, err := api.ChainStatObj(ctx, obj, base) diff --git a/cli/state.go b/cli/state.go index 6b5c9035b..910b9557a 100644 --- a/cli/state.go +++ b/cli/state.go @@ -359,7 +359,7 @@ var stateReplaySetCmd = &cli.Command{ return fmt.Errorf("message cid was invalid: %s", err) } - api, closer, err := GetFullNodeAPI(cctx) + fapi, closer, err := GetFullNodeAPI(cctx) if err != nil { return err } @@ -381,7 +381,7 @@ var stateReplaySetCmd = &cli.Command{ if len(tscids) > 0 { var headers []*types.BlockHeader for _, c := range tscids { - h, err := api.ChainGetBlock(ctx, c) + h, err := fapi.ChainGetBlock(ctx, c) if err != nil { return err } @@ -391,12 +391,13 @@ var stateReplaySetCmd = &cli.Command{ ts, err = types.NewTipSet(headers) } else { - r, err := api.StateWaitMsg(ctx, mcid) + var r *api.MsgLookup + r, err = fapi.StateWaitMsg(ctx, mcid) if err != nil { return xerrors.Errorf("finding message in chain: %w", err) } - ts, err = api.ChainGetTipSet(ctx, r.TipSet.Parents()) + ts, err = fapi.ChainGetTipSet(ctx, r.TipSet.Parents()) } if err != nil { return err @@ -404,7 +405,7 @@ var stateReplaySetCmd = &cli.Command{ } - res, err := api.StateReplay(ctx, ts.Key(), mcid) + res, err := fapi.StateReplay(ctx, ts.Key(), mcid) if err != nil { return xerrors.Errorf("replay call failed: %w", err) } From 6d36c030e0cafcf99c17c2c77846a60ee9568981 Mon Sep 17 00:00:00 2001 From: chunqizhi <1558763837@qq.com> Date: Sat, 13 Jun 2020 18:05:30 +0800 Subject: [PATCH 60/81] Add usage information for provingInfoCmd --- cmd/lotus-storage-miner/proving.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/lotus-storage-miner/proving.go b/cmd/lotus-storage-miner/proving.go index 7cfa010b9..60e85e0d8 100644 --- a/cmd/lotus-storage-miner/proving.go +++ b/cmd/lotus-storage-miner/proving.go @@ -16,6 +16,7 @@ import ( var provingCmd = &cli.Command{ Name: "proving", + Usage:"View proving information", Subcommands: []*cli.Command{ provingInfoCmd, provingDeadlinesCmd, @@ -24,6 +25,7 @@ var provingCmd = &cli.Command{ var provingInfoCmd = &cli.Command{ Name: "info", + Usage:"View current state information", Action: func(cctx *cli.Context) error { nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { @@ -146,6 +148,7 @@ func epochTime(curr, e abi.ChainEpoch) string { var provingDeadlinesCmd = &cli.Command{ Name: "deadlines", + Usage:"View the current proving period deadlines information", Action: func(cctx *cli.Context) error { nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { From c822816f8ee3805c5ed620cbd098782c8fa20aac Mon Sep 17 00:00:00 2001 From: Howard Yeh Date: Sun, 14 Jun 2020 17:49:20 +0800 Subject: [PATCH 61/81] Cache tipset validation progress --- chain/store/store.go | 17 +++++++++++++++++ chain/sync.go | 25 +++++++++++++++++++++++-- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index dba5995de..e3d6506e4 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -47,6 +47,7 @@ import ( var log = logging.Logger("chainstore") var chainHeadKey = dstore.NewKey("head") +var blockValidationCacheKeyPrefix = dstore.NewKey("blockValidation") type ChainStore struct { bs bstore.Blockstore @@ -215,6 +216,22 @@ func (cs *ChainStore) SubscribeHeadChanges(f func(rev, app []*types.TipSet) erro cs.headChangeNotifs = append(cs.headChangeNotifs, f) } +func (cs *ChainStore) IsBlockValidated(ctx context.Context, blkid cid.Cid) (bool, error) { + key := blockValidationCacheKeyPrefix.Instance(blkid.String()) + + return cs.ds.Has(key) +} + +func (cs *ChainStore) MarkBlockAsValidated(ctx context.Context, blkid cid.Cid) error { + key := blockValidationCacheKeyPrefix.Instance(blkid.String()) + + if err := cs.ds.Put(key, []byte{0}); err != nil { + return xerrors.Errorf("cache block validation: %w", err) + } + + return nil +} + func (cs *ChainStore) SetGenesis(b *types.BlockHeader) error { ts, err := types.NewTipSet([]*types.BlockHeader{b}) if err != nil { diff --git a/chain/sync.go b/chain/sync.go index 07d470d28..a1315d82b 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -529,7 +529,24 @@ func blockSanityChecks(h *types.BlockHeader) error { } // Should match up with 'Semantical Validation' in validation.md in the spec -func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) error { +func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (err error) { + defer func() { + // b.Cid() could panic for empty blocks that are used in tests. + if rerr := recover(); rerr != nil { + err = xerrors.Errorf("validate block panic: %w", rerr) + return + } + }() + + isValidated, err := syncer.store.IsBlockValidated(ctx, b.Cid()) + if err != nil { + return xerrors.Errorf("check block validation cache %s: %w", b.Cid(), err) + } + + if isValidated { + return nil + } + validationStart := time.Now() defer func() { dur := time.Since(validationStart) @@ -759,7 +776,11 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err } } - return merr + if err := syncer.store.MarkBlockAsValidated(ctx, b.Cid()); err != nil { + return xerrors.Errorf("caching block validation %s: %w", b.Cid(), err) + } + + return nil } func (syncer *Syncer) VerifyWinningPoStProof(ctx context.Context, h *types.BlockHeader, prevBeacon types.BeaconEntry, lbst cid.Cid, waddr address.Address) error { From 12056c8904a41fcf398a5eb0dd299551ef9a44c4 Mon Sep 17 00:00:00 2001 From: chunqizhi <1558763837@qq.com> Date: Mon, 15 Jun 2020 17:43:42 +0800 Subject: [PATCH 62/81] go fmt --- cmd/lotus-storage-miner/proving.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/lotus-storage-miner/proving.go b/cmd/lotus-storage-miner/proving.go index 60e85e0d8..d419efd7b 100644 --- a/cmd/lotus-storage-miner/proving.go +++ b/cmd/lotus-storage-miner/proving.go @@ -15,8 +15,8 @@ import ( ) var provingCmd = &cli.Command{ - Name: "proving", - Usage:"View proving information", + Name: "proving", + Usage: "View proving information", Subcommands: []*cli.Command{ provingInfoCmd, provingDeadlinesCmd, @@ -24,8 +24,8 @@ var provingCmd = &cli.Command{ } var provingInfoCmd = &cli.Command{ - Name: "info", - Usage:"View current state information", + Name: "info", + Usage: "View current state information", Action: func(cctx *cli.Context) error { nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { @@ -147,8 +147,8 @@ func epochTime(curr, e abi.ChainEpoch) string { } var provingDeadlinesCmd = &cli.Command{ - Name: "deadlines", - Usage:"View the current proving period deadlines information", + Name: "deadlines", + Usage: "View the current proving period deadlines information", Action: func(cctx *cli.Context) error { nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { From 464f6a6b08f3267fcf3b5d5aea2bf190c6578c18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 15 Jun 2020 14:49:18 +0200 Subject: [PATCH 63/81] Update specs-actors to v0.6 --- extern/filecoin-ffi | 2 +- go.mod | 4 ++-- go.sum | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 61c02f6be..1bff7f456 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 61c02f6bea8d69bb79c70daa1d62f26c486643aa +Subproject commit 1bff7f4563370ada590a605b5459b91e1662ebaa diff --git a/go.mod b/go.mod index 21fea095f..5cfe55a8d 100644 --- a/go.mod +++ b/go.mod @@ -29,8 +29,8 @@ require ( github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/sector-storage v0.0.0-20200609231555-252c2b0c969d - github.com/filecoin-project/specs-actors v0.5.6 + github.com/filecoin-project/sector-storage v0.0.0-20200615123301-7d09fa88b4b0 + github.com/filecoin-project/specs-actors v0.6.0 github.com/filecoin-project/specs-storage v0.1.0 github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index e10972bb6..5366cf6b6 100644 --- a/go.sum +++ b/go.sum @@ -223,6 +223,8 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/ github.com/filecoin-project/sector-storage v0.0.0-20200508203401-a74812ba12f3/go.mod h1:B+xzopr/oWZJz2hBL5Ekb7Obcum5ntmfbaAUlaaho28= github.com/filecoin-project/sector-storage v0.0.0-20200609231555-252c2b0c969d h1:q7KC8/yIirwVX7JCLgjEmo5LX77V8tYV0EzVCnCwrE8= github.com/filecoin-project/sector-storage v0.0.0-20200609231555-252c2b0c969d/go.mod h1:SfaVNw/A6LwqqEt6wEMVBN5Grn3XC50j+yjmegHIe7Y= +github.com/filecoin-project/sector-storage v0.0.0-20200615123301-7d09fa88b4b0 h1:DpdlpS3G2XN+F68rxOTAXsGL3cV7UkFr0TxdmKSiaJg= +github.com/filecoin-project/sector-storage v0.0.0-20200615123301-7d09fa88b4b0/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= @@ -231,6 +233,8 @@ github.com/filecoin-project/specs-actors v0.5.5 h1:bDsowem6dLRc9B7g3sgFYvHgfWTH4 github.com/filecoin-project/specs-actors v0.5.5/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-actors v0.5.6 h1:WlhtoXwFoKlP1b06NI4NJaxC4m9EXNV+qFVl43/xRN8= github.com/filecoin-project/specs-actors v0.5.6/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= +github.com/filecoin-project/specs-actors v0.6.0 h1:IepUsmDGY60QliENVTkBTAkwqGWw9kNbbHOcU/9oiC0= +github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 h1:T3f/zkuvgtgqcXrb0NO3BicuveGOxxUAMPa/Yif2kuE= github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102/go.mod h1:xJ1/xl9+8zZeSSSFmDC3Wr6uusCTxyYPI0VeNVSFmPE= github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sgOVdJbvFjOnD5w94= From a23a87a17fedaed8f3ff77a17f02e55aca5c2515 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 15 Jun 2020 16:18:05 +0200 Subject: [PATCH 64/81] Add virtual gas Signed-off-by: Jakub Sztandera --- chain/types/execresult.go | 11 +++++++---- chain/vm/gas.go | 9 +++++++++ chain/vm/runtime.go | 16 ++++++++++------ cli/state.go | 25 ++++++++++++++++++++++--- 4 files changed, 48 insertions(+), 13 deletions(-) diff --git a/chain/types/execresult.go b/chain/types/execresult.go index 7ffa8c219..b0e1f760d 100644 --- a/chain/types/execresult.go +++ b/chain/types/execresult.go @@ -21,10 +21,13 @@ type ExecutionTrace struct { type GasTrace struct { Name string - Location []Loc - TotalGas int64 - ComputeGas int64 - StorageGas int64 + Location []Loc + TotalGas int64 + ComputeGas int64 + StorageGas int64 + TotalVirtualGas int64 + VirtualComputeGas int64 + VirtualStorageGas int64 TimeTaken time.Duration Extra interface{} diff --git a/chain/vm/gas.go b/chain/vm/gas.go index c030621f6..dc9966e57 100644 --- a/chain/vm/gas.go +++ b/chain/vm/gas.go @@ -21,11 +21,20 @@ type GasCharge struct { Name string ComputeGas int64 StorageGas int64 + + VirtualCompute int64 + VirtualStorage int64 } func (g GasCharge) Total() int64 { return g.ComputeGas*GasComputeMulti + g.StorageGas*GasStorageMulti } +func (g GasCharge) WithVirtual(compute, storage int64) GasCharge { + out := g + out.VirtualCompute = compute + out.VirtualStorage = storage + return out +} func newGasCharge(name string, computeGas int64, storageGas int64) GasCharge { return GasCharge{ diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index be34f075c..9b83593ab 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -524,11 +524,14 @@ func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError } gasTrace := types.GasTrace{ - Name: gas.Name, - TotalGas: toUse, - ComputeGas: gas.ComputeGas, - StorageGas: gas.StorageGas, - Callers: callers[:cout], + Name: gas.Name, + TotalGas: toUse, + ComputeGas: gas.ComputeGas, + StorageGas: gas.StorageGas, + TotalVirtualGas: gas.VirtualCompute*GasComputeMulti + gas.VirtualStorage*GasStorageMulti, + VirtualComputeGas: gas.VirtualCompute, + VirtualStorageGas: gas.VirtualStorage, + Callers: callers[:cout], } rt.executionTrace.GasCharges = append(rt.executionTrace.GasCharges, &gasTrace) rt.lastGasChargeTime = now @@ -536,7 +539,8 @@ func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError if rt.gasUsed+toUse > rt.gasAvailable { rt.gasUsed = rt.gasAvailable - return aerrors.Newf(exitcode.SysErrOutOfGas, "not enough gas: used=%d, available=%d", rt.gasUsed, rt.gasAvailable) + return aerrors.Newf(exitcode.SysErrOutOfGas, "not enough gas: used=%d, available=%d", + rt.gasUsed, rt.gasAvailable) } rt.gasUsed += toUse return nil diff --git a/cli/state.go b/cli/state.go index bb5dd4277..753b0b017 100644 --- a/cli/state.go +++ b/cli/state.go @@ -971,6 +971,7 @@ var compStateTemplate = ` } .slow-true-false { color: #660; } .slow-true-true { color: #f80; } + .deemp { color: #444; } table { font-size: 12px; border-collapse: collapse; @@ -1060,8 +1061,20 @@ var compStateMsg = ` Gas Trace + {{define "virt" -}} + {{- if . -}} + +({{.}}) + {{- end -}} + {{- end}} + + {{define "gasC" -}} + + {{- end}} + {{range .GasCharges}} - + + {{template "gasC" .}} + {{end}} {{with SumGas .GasCharges}} - + + {{template "gasC" .}} + + {{end}}
NameTotal/Compute/StorageTime TakenLocation
{{.TotalGas}}{{template "virt" .TotalVirtualGas }}/{{.ComputeGas}}{{template "virt" .VirtualComputeGas}}/{{.StorageGas}}{{template "virt" .VirtualStorageGas}}
{{.Name}}{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}}
{{.Name}}{{.TimeTaken}} {{ $fImp := FirstImportant .Location }} {{ if $fImp }} @@ -1098,7 +1111,10 @@ var compStateMsg = `
Sum{{.TotalGas}}/{{.ComputeGas}}/{{.StorageGas}}{{.TimeTaken}}
Sum{{.TimeTaken}}
@@ -1201,7 +1217,10 @@ func sumGas(changes []*types.GasTrace) types.GasTrace { out.TotalGas += gc.TotalGas out.ComputeGas += gc.ComputeGas out.StorageGas += gc.StorageGas - out.TimeTaken += gc.TimeTaken + + out.TotalVirtualGas += gc.TotalVirtualGas + out.VirtualComputeGas += gc.VirtualComputeGas + out.VirtualStorageGas += gc.VirtualStorageGas } return out From 2a78ebabd9117de39ac344cea12bd7845b528558 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 17 Apr 2020 20:16:36 +0200 Subject: [PATCH 65/81] Use HashVerify Signed-off-by: Jakub Sztandera --- chain/sync.go | 18 ++++-------------- lib/sigs/bls/init.go | 4 ++-- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index f4b32f2bc..bfbca0ec1 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -8,7 +8,6 @@ import ( "os" "sort" "strings" - "sync" "time" "github.com/Gurpartap/async" @@ -949,23 +948,14 @@ func (syncer *Syncer) verifyBlsAggregate(ctx context.Context, sig *crypto.Signat trace.Int64Attribute("msgCount", int64(len(msgs))), ) - var wg sync.WaitGroup - - digests := make([]bls.Digest, len(msgs)) - for i := 0; i < 10; i++ { - wg.Add(1) - go func(w int) { - defer wg.Done() - for j := 0; (j*10)+w < len(msgs); j++ { - digests[j*10+w] = bls.Hash(bls.Message(msgs[j*10+w].Bytes())) - } - }(i) + bmsgs := make([]bls.Message, len(msgs)) + for i, m := range msgs { + bmsgs[i] = m.Bytes() } - wg.Wait() var bsig bls.Signature copy(bsig[:], sig.Data) - if !bls.Verify(&bsig, digests, pubks) { + if !bls.HashVerify(&bsig, bmsgs, pubks) { return xerrors.New("bls aggregate signature failed to verify") } diff --git a/lib/sigs/bls/init.go b/lib/sigs/bls/init.go index fe916c446..66a5ade81 100644 --- a/lib/sigs/bls/init.go +++ b/lib/sigs/bls/init.go @@ -33,16 +33,16 @@ func (blsSigner) Sign(p []byte, msg []byte) ([]byte, error) { } func (blsSigner) Verify(sig []byte, a address.Address, msg []byte) error { - digests := []ffi.Digest{ffi.Hash(ffi.Message(msg))} var pubk ffi.PublicKey copy(pubk[:], a.Payload()) pubkeys := []ffi.PublicKey{pubk} + digests := []ffi.Message{msg} var s ffi.Signature copy(s[:], sig) - if !ffi.Verify(&s, digests, pubkeys) { + if !ffi.HashVerify(&s, digests, pubkeys) { return fmt.Errorf("bls signature failed to verify") } From 907364ce670c394ae3308e63728005f591468836 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 15 Jun 2020 18:30:49 +0200 Subject: [PATCH 66/81] Update deps, fix more tests --- api/apistruct/struct.go | 26 +++++++++++++------------- api/docgen/docgen.go | 2 +- api/types.go | 2 +- build/params_testnet.go | 8 ++++---- chain/gen/gen.go | 10 +++++----- chain/gen/gen_test.go | 4 ++-- chain/gen/genesis/miners.go | 12 ++++++------ chain/stmgr/forks_test.go | 4 ++-- chain/stmgr/stmgr.go | 7 +++---- chain/stmgr/utils.go | 6 +++--- chain/store/store_test.go | 4 ++-- chain/sync_test.go | 4 ++-- chain/types/blockheader_test.go | 9 ++++----- chain/vm/gas.go | 4 ++-- chain/vm/gas_v0.go | 2 +- chain/vm/syscalls.go | 2 +- cmd/lotus-bench/main.go | 23 ++++++++++++++--------- cmd/lotus-seed/seed/seed.go | 11 +++-------- cmd/lotus-shed/proofs.go | 2 +- extern/filecoin-ffi | 2 +- genesis/types.go | 2 +- go.mod | 10 ++++++---- go.sum | 10 ++++++++++ lotuspond/spawn.go | 6 +++--- node/impl/full/state.go | 2 +- node/node_test.go | 6 +++--- storage/adapter_storage_miner.go | 2 +- storage/miner.go | 2 +- storage/wdpost_run.go | 6 +++--- storage/wdpost_sched.go | 2 +- 30 files changed, 101 insertions(+), 91 deletions(-) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 31cd6badd..c607c13a4 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -207,18 +207,18 @@ type StorageMinerStruct struct { WorkerConnect func(context.Context, string) error `perm:"admin"` // TODO: worker perm WorkerStats func(context.Context) (map[uint64]storiface.WorkerStats, error) `perm:"admin"` - StorageList func(context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"` - StorageLocal func(context.Context) (map[stores.ID]string, error) `perm:"admin"` - StorageStat func(context.Context, stores.ID) (stores.FsStat, error) `perm:"admin"` - StorageAttach func(context.Context, stores.StorageInfo, stores.FsStat) error `perm:"admin"` - StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType, bool) error `perm:"admin"` - StorageDropSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType) error `perm:"admin"` - StorageFindSector func(context.Context, abi.SectorID, stores.SectorFileType, bool) ([]stores.SectorStorageInfo, error) `perm:"admin"` - StorageInfo func(context.Context, stores.ID) (stores.StorageInfo, error) `perm:"admin"` - StorageBestAlloc func(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredProof, sealing stores.PathType) ([]stores.StorageInfo, error) `perm:"admin"` - StorageReportHealth func(ctx context.Context, id stores.ID, report stores.HealthReport) error `perm:"admin"` - StorageLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error `perm:"admin"` - StorageTryLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) (bool, error) `perm:"admin"` + StorageList func(context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"` + StorageLocal func(context.Context) (map[stores.ID]string, error) `perm:"admin"` + StorageStat func(context.Context, stores.ID) (stores.FsStat, error) `perm:"admin"` + StorageAttach func(context.Context, stores.StorageInfo, stores.FsStat) error `perm:"admin"` + StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType, bool) error `perm:"admin"` + StorageDropSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType) error `perm:"admin"` + StorageFindSector func(context.Context, abi.SectorID, stores.SectorFileType, bool) ([]stores.SectorStorageInfo, error) `perm:"admin"` + StorageInfo func(context.Context, stores.ID) (stores.StorageInfo, error) `perm:"admin"` + StorageBestAlloc func(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredSealProof, sealing stores.PathType) ([]stores.StorageInfo, error) `perm:"admin"` + StorageReportHealth func(ctx context.Context, id stores.ID, report stores.HealthReport) error `perm:"admin"` + StorageLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error `perm:"admin"` + StorageTryLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) (bool, error) `perm:"admin"` DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` @@ -812,7 +812,7 @@ func (c *StorageMinerStruct) StorageInfo(ctx context.Context, id stores.ID) (sto return c.Internal.StorageInfo(ctx, id) } -func (c *StorageMinerStruct) StorageBestAlloc(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredProof, pt stores.PathType) ([]stores.StorageInfo, error) { +func (c *StorageMinerStruct) StorageBestAlloc(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredSealProof, pt stores.PathType) ([]stores.StorageInfo, error) { return c.Internal.StorageBestAlloc(ctx, allocate, spt, pt) } diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index d0b5d8558..4cd982aa7 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -72,7 +72,7 @@ func init() { addExample(pid) addExample(bitfield.NewFromSet([]uint64{5})) - addExample(abi.RegisteredProof_StackedDRG32GiBPoSt) + addExample(abi.RegisteredSealProof_StackedDrg32GiBV1) addExample(abi.ChainEpoch(10101)) addExample(crypto.SigTypeBLS) addExample(int64(9)) diff --git a/api/types.go b/api/types.go index f5a595353..29bd7401c 100644 --- a/api/types.go +++ b/api/types.go @@ -50,7 +50,7 @@ type MinerInfo struct { WorkerChangeEpoch abi.ChainEpoch PeerId peer.ID Multiaddrs []abi.Multiaddrs - SealProofType abi.RegisteredProof + SealProofType abi.RegisteredSealProof SectorSize abi.SectorSize WindowPoStPartitionSectors uint64 } diff --git a/build/params_testnet.go b/build/params_testnet.go index 917b8059e..c222862eb 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -13,10 +13,10 @@ import ( func init() { power.ConsensusMinerMinPower = big.NewInt(1024 << 20) - miner.SupportedProofTypes = map[abi.RegisteredProof]struct{}{ - abi.RegisteredProof_StackedDRG512MiBSeal: {}, - abi.RegisteredProof_StackedDRG32GiBSeal: {}, - abi.RegisteredProof_StackedDRG64GiBSeal: {}, + miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg512MiBV1: {}, + abi.RegisteredSealProof_StackedDrg32GiBV1: {}, + abi.RegisteredSealProof_StackedDrg64GiBV1: {}, } } diff --git a/chain/gen/gen.go b/chain/gen/gen.go index d58bbc28e..368240a04 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -89,8 +89,8 @@ func (m mybs) Get(c cid.Cid) (block.Block, error) { } func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { - saminer.SupportedProofTypes = map[abi.RegisteredProof]struct{}{ - abi.RegisteredProof_StackedDRG2KiBSeal: {}, + saminer.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } mr := repo.NewMemory(nil) @@ -141,7 +141,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { return nil, err } - genm1, k1, err := seed.PreSeal(maddr1, abi.RegisteredProof_StackedDRG2KiBPoSt, 0, numSectors, m1temp, []byte("some randomness"), nil) + genm1, k1, err := seed.PreSeal(maddr1, abi.RegisteredSealProof_StackedDrg2KiBV1, 0, numSectors, m1temp, []byte("some randomness"), nil) if err != nil { return nil, err } @@ -153,7 +153,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { return nil, err } - genm2, k2, err := seed.PreSeal(maddr2, abi.RegisteredProof_StackedDRG2KiBPoSt, 0, numSectors, m2temp, []byte("some randomness"), nil) + genm2, k2, err := seed.PreSeal(maddr2, abi.RegisteredSealProof_StackedDrg2KiBV1, 0, numSectors, m2temp, []byte("some randomness"), nil) if err != nil { return nil, err } @@ -611,6 +611,6 @@ func (m genFakeVerifier) VerifyWindowPoSt(ctx context.Context, info abi.WindowPo panic("not supported") } -func (m genFakeVerifier) GenerateWinningPoStSectorChallenge(ctx context.Context, proof abi.RegisteredProof, id abi.ActorID, randomness abi.PoStRandomness, u uint64) ([]uint64, error) { +func (m genFakeVerifier) GenerateWinningPoStSectorChallenge(ctx context.Context, proof abi.RegisteredPoStProof, id abi.ActorID, randomness abi.PoStRandomness, u uint64) ([]uint64, error) { panic("not supported") } diff --git a/chain/gen/gen_test.go b/chain/gen/gen_test.go index 69c8587ef..7a4d73031 100644 --- a/chain/gen/gen_test.go +++ b/chain/gen/gen_test.go @@ -14,8 +14,8 @@ import ( ) func init() { - miner.SupportedProofTypes = map[abi.RegisteredProof]struct{}{ - abi.RegisteredProof_StackedDRG2KiBSeal: {}, + miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } power.ConsensusMinerMinPower = big.NewInt(2048) verifreg.MinVerifiedDealSize = big.NewInt(256) diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index 685fc58cc..acc63c31c 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -204,12 +204,12 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid { newSectorInfo := &miner.SectorOnChainInfo{ Info: miner.SectorPreCommitInfo{ - RegisteredProof: preseal.ProofType, - SectorNumber: preseal.SectorID, - SealedCID: preseal.CommR, - SealRandEpoch: 0, - DealIDs: []abi.DealID{dealIDs[pi]}, - Expiration: preseal.Deal.EndEpoch, + SealProof: preseal.ProofType, + SectorNumber: preseal.SectorID, + SealedCID: preseal.CommR, + SealRandEpoch: 0, + DealIDs: []abi.DealID{dealIDs[pi]}, + Expiration: preseal.Deal.EndEpoch, }, ActivationEpoch: 0, DealWeight: dealWeight.DealWeight, diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index a3b01cd84..2fbcbbc99 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -37,8 +37,8 @@ import ( ) func init() { - miner.SupportedProofTypes = map[abi.RegisteredProof]struct{}{ - abi.RegisteredProof_StackedDRG2KiBSeal: {}, + miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } power.ConsensusMinerMinPower = big.NewInt(2048) verifreg.MinVerifiedDealSize = big.NewInt(256) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index d4c12dffe..917b5ca26 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -184,10 +184,9 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, pstate cid.Cid, bms []B var err error params, err := actors.SerializeParams(&reward.AwardBlockRewardParams{ - Miner: b.Miner, - Penalty: penalty, - GasReward: gasReward, - TicketCount: 1, // TODO: no longer need ticket count here. + Miner: b.Miner, + Penalty: penalty, + GasReward: gasReward, }) if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("failed to serialize award params: %w", err) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 308c30823..cf424b4eb 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -206,9 +206,9 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S out := make([]abi.SectorInfo, len(ids)) for i, n := range ids { out[i] = abi.SectorInfo{ - RegisteredProof: wpt, - SectorNumber: sectorSet[n].ID, - SealedCID: sectorSet[n].Info.Info.SealedCID, + SealProof: spt, + SectorNumber: sectorSet[n].ID, + SealedCID: sectorSet[n].Info.Info.SealedCID, } } diff --git a/chain/store/store_test.go b/chain/store/store_test.go index 41b875916..939c85d20 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -22,8 +22,8 @@ import ( ) func init() { - miner.SupportedProofTypes = map[abi.RegisteredProof]struct{}{ - abi.RegisteredProof_StackedDRG2KiBSeal: {}, + miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } power.ConsensusMinerMinPower = big.NewInt(2048) verifreg.MinVerifiedDealSize = big.NewInt(256) diff --git a/chain/sync_test.go b/chain/sync_test.go index e1c846d4e..92c0f72d7 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -34,8 +34,8 @@ import ( func init() { build.InsecurePoStValidation = true os.Setenv("TRUST_PARAMS", "1") - miner.SupportedProofTypes = map[abi.RegisteredProof]struct{}{ - abi.RegisteredProof_StackedDRG2KiBSeal: {}, + miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } power.ConsensusMinerMinPower = big.NewInt(2048) verifreg.MinVerifiedDealSize = big.NewInt(256) diff --git a/chain/types/blockheader_test.go b/chain/types/blockheader_test.go index 387fe86c2..a1ece308a 100644 --- a/chain/types/blockheader_test.go +++ b/chain/types/blockheader_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" "fmt" + "github.com/stretchr/testify/require" "reflect" "testing" @@ -79,7 +80,7 @@ func TestInteropBH(t *testing.T) { } posts := []abi.PoStProof{ - {abi.RegisteredProof_StackedDRG2KiBWinningPoSt, []byte{0x07}}, + {abi.RegisteredPoStProof_StackedDrgWinning2KiBV1, []byte{0x07}}, } bh := &BlockHeader{ @@ -116,10 +117,8 @@ func TestInteropBH(t *testing.T) { } // acquired from go-filecoin - gfc := "8f5501d04cb15021bf6bd003073d79e2238d4e61f1ad22814301020381420a0b818205410c818209410781d82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619cc430003e802d82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619ccd82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619ccd82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619cc410001f603" - if gfc != hex.EncodeToString(bhsb) { - t.Fatal("not equal!") - } + gfc := "8f5501d04cb15021bf6bd003073d79e2238d4e61f1ad22814301020381420a0b818205410c818200410781d82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619cc430003e802d82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619ccd82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619ccd82a5827000171a0e402202f84fef0d7cc2d7f9f00d22445f7bf7539fdd685fd9f284aa37f3822b57619cc410001f603" + require.Equal(t, gfc, hex.EncodeToString(bhsb)) } func BenchmarkBlockHeaderMarshal(b *testing.B) { diff --git a/chain/vm/gas.go b/chain/vm/gas.go index c030621f6..66d74fb70 100644 --- a/chain/vm/gas.go +++ b/chain/vm/gas.go @@ -59,7 +59,7 @@ type Pricelist interface { OnVerifySignature(sigType crypto.SigType, planTextSize int) (GasCharge, error) OnHashing(dataSize int) GasCharge - OnComputeUnsealedSectorCid(proofType abi.RegisteredProof, pieces []abi.PieceInfo) GasCharge + OnComputeUnsealedSectorCid(proofType abi.RegisteredSealProof, pieces []abi.PieceInfo) GasCharge OnVerifySeal(info abi.SealVerifyInfo) GasCharge OnVerifyPost(info abi.WindowPoStVerifyInfo) GasCharge OnVerifyConsensusFault() GasCharge @@ -136,7 +136,7 @@ func (ps pricedSyscalls) HashBlake2b(data []byte) [32]byte { } // Computes an unsealed sector CID (CommD) from its constituent piece CIDs (CommPs) and sizes. -func (ps pricedSyscalls) ComputeUnsealedSectorCID(reg abi.RegisteredProof, pieces []abi.PieceInfo) (cid.Cid, error) { +func (ps pricedSyscalls) ComputeUnsealedSectorCID(reg abi.RegisteredSealProof, pieces []abi.PieceInfo) (cid.Cid, error) { ps.chargeGas(ps.pl.OnComputeUnsealedSectorCid(reg, pieces)) return ps.under.ComputeUnsealedSectorCID(reg, pieces) } diff --git a/chain/vm/gas_v0.go b/chain/vm/gas_v0.go index c631beaee..b1b2c5c9d 100644 --- a/chain/vm/gas_v0.go +++ b/chain/vm/gas_v0.go @@ -142,7 +142,7 @@ func (pl *pricelistV0) OnHashing(dataSize int) GasCharge { } // OnComputeUnsealedSectorCid -func (pl *pricelistV0) OnComputeUnsealedSectorCid(proofType abi.RegisteredProof, pieces []abi.PieceInfo) GasCharge { +func (pl *pricelistV0) OnComputeUnsealedSectorCid(proofType abi.RegisteredSealProof, pieces []abi.PieceInfo) GasCharge { // TODO: this needs more cost tunning, check with @lotus return newGasCharge("OnComputeUnsealedSectorCid", pl.computeUnsealedSectorCidBase, 0) } diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index 2349bc324..a5910121d 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -43,7 +43,7 @@ type syscallShim struct { verifier ffiwrapper.Verifier } -func (ss *syscallShim) ComputeUnsealedSectorCID(st abi.RegisteredProof, pieces []abi.PieceInfo) (cid.Cid, error) { +func (ss *syscallShim) ComputeUnsealedSectorCID(st abi.RegisteredSealProof, pieces []abi.PieceInfo) (cid.Cid, error) { var sum abi.PaddedPieceSize for _, p := range pieces { sum += p.Size diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 7d7991636..57f5241df 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -74,7 +74,7 @@ func main() { log.Info("Starting lotus-bench") - miner.SupportedProofTypes[abi.RegisteredProof_StackedDRG2KiBSeal] = struct{}{} + miner.SupportedProofTypes[abi.RegisteredSealProof_StackedDrg2KiBV1] = struct{}{} app := &cli.App{ Name: "lotus-bench", @@ -270,9 +270,9 @@ var sealBenchCmd = &cli.Command{ for _, s := range genm.Sectors { sealedSectors = append(sealedSectors, abi.SectorInfo{ - SealedCID: s.CommR, - SectorNumber: s.SectorID, - RegisteredProof: s.ProofType, + SealedCID: s.CommR, + SectorNumber: s.SectorID, + SealProof: s.ProofType, }) } } @@ -284,7 +284,12 @@ var sealBenchCmd = &cli.Command{ if !c.Bool("skip-commit2") { log.Info("generating winning post candidates") - fcandidates, err := ffiwrapper.ProofVerifier.GenerateWinningPoStSectorChallenge(context.TODO(), spt, mid, challenge[:], uint64(len(sealedSectors))) + wpt, err := spt.RegisteredWindowPoStProof() + if err != nil { + return err + } + + fcandidates, err := ffiwrapper.ProofVerifier.GenerateWinningPoStSectorChallenge(context.TODO(), wpt, mid, challenge[:], uint64(len(sealedSectors))) if err != nil { return err } @@ -485,9 +490,9 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, mid precommit2 := time.Now() sealedSectors = append(sealedSectors, abi.SectorInfo{ - RegisteredProof: sb.SealProofType(), - SectorNumber: i, - SealedCID: cids.Sealed, + SealProof: sb.SealProofType(), + SectorNumber: i, + SealedCID: cids.Sealed, }) seed := lapi.SealSeed{ @@ -536,7 +541,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, mid svi := abi.SealVerifyInfo{ SectorID: abi.SectorID{Miner: mid, Number: i}, SealedCID: cids.Sealed, - RegisteredProof: sb.SealProofType(), + SealProof: sb.SealProofType(), Proof: proof, DealIDs: nil, Randomness: ticket, diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index 06cb011b4..ffb651c00 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -33,12 +33,7 @@ import ( var log = logging.Logger("preseal") -func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNumber, sectors int, sbroot string, preimage []byte, key *types.KeyInfo) (*genesis.Miner, *types.KeyInfo, error) { - spt, err := pt.RegisteredSealProof() - if err != nil { - return nil, nil, err - } - +func PreSeal(maddr address.Address, spt abi.RegisteredSealProof, offset abi.SectorNumber, sectors int, sbroot string, preimage []byte, key *types.KeyInfo) (*genesis.Miner, *types.KeyInfo, error) { mid, err := address.IDFromAddress(maddr) if err != nil { return nil, nil, err @@ -63,7 +58,7 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum return nil, nil, err } - ssize, err := pt.SectorSize() + ssize, err := spt.SectorSize() if err != nil { return nil, nil, err } @@ -102,7 +97,7 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum CommR: cids.Sealed, CommD: cids.Unsealed, SectorID: sid.Number, - ProofType: pt, + ProofType: spt, }) } diff --git a/cmd/lotus-shed/proofs.go b/cmd/lotus-shed/proofs.go index 3a4615658..f18dc93fb 100644 --- a/cmd/lotus-shed/proofs.go +++ b/cmd/lotus-shed/proofs.go @@ -88,7 +88,7 @@ var verifySealProofCmd = &cli.Command{ Number: snum, }, SealedCID: commr, - RegisteredProof: abi.RegisteredProof(cctx.Int64("proof-type")), + SealProof: abi.RegisteredSealProof(cctx.Int64("proof-type")), Proof: proof, DealIDs: nil, Randomness: abi.SealRandomness(ticket), diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 1bff7f456..5bb4a309b 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 1bff7f4563370ada590a605b5459b91e1662ebaa +Subproject commit 5bb4a309bce9d446ac618f34a8b9e2883af2002f diff --git a/genesis/types.go b/genesis/types.go index 4b95d3311..41d8a413c 100644 --- a/genesis/types.go +++ b/genesis/types.go @@ -22,7 +22,7 @@ type PreSeal struct { CommD cid.Cid SectorID abi.SectorNumber Deal market.DealProposal - ProofType abi.RegisteredProof + ProofType abi.RegisteredSealProof } type Miner struct { diff --git a/go.mod b/go.mod index 5cfe55a8d..df0c03c9f 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/drand/drand v0.9.2-0.20200529123141-6b138aefaef2 github.com/drand/kyber v1.0.2 github.com/fatih/color v1.8.0 - github.com/filecoin-project/chain-validation v0.0.6-0.20200605221044-7f78284bbc94 + github.com/filecoin-project/chain-validation v0.0.6-0.20200615142407-39c0fe23a3d5 github.com/filecoin-project/filecoin-ffi v0.26.1-0.20200508175440-05b30afeb00d github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2 @@ -23,16 +23,16 @@ require ( github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.3.0 github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 - github.com/filecoin-project/go-fil-markets v0.2.7 + github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615162707-5dd6b2eb253b github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/sector-storage v0.0.0-20200615123301-7d09fa88b4b0 + github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6 github.com/filecoin-project/specs-actors v0.6.0 github.com/filecoin-project/specs-storage v0.1.0 - github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa + github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect @@ -127,3 +127,5 @@ require ( replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v1.18.0 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi + +replace github.com/filecoin-project/go-fil-markets => /home/magik6k/gohack/github.com/filecoin-project/go-fil-markets diff --git a/go.sum b/go.sum index 5366cf6b6..c4a07cf66 100644 --- a/go.sum +++ b/go.sum @@ -184,6 +184,8 @@ github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGj github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/filecoin-project/chain-validation v0.0.6-0.20200605221044-7f78284bbc94 h1:svEal/usZ/REEjEz6ije3/ALn8duM2CckR0LjT1btx0= github.com/filecoin-project/chain-validation v0.0.6-0.20200605221044-7f78284bbc94/go.mod h1:aOmmdhO0xIRtWCwx3MyIv3tPCCorM1Bq7pNOJGMLndA= +github.com/filecoin-project/chain-validation v0.0.6-0.20200615142407-39c0fe23a3d5 h1:8Ytl1uk8/Sff7Z2vfb7MOEM3ZRlQKJPqn19FzDcr+bc= +github.com/filecoin-project/chain-validation v0.0.6-0.20200615142407-39c0fe23a3d5/go.mod h1:HEJn6kOXMNhCNBYNTO/lrEI7wSgqCOR6hN5ecfYUnC8= github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef h1:Wi5E+P1QfHP8IF27eUiTx5vYfqQZwfPxzq3oFEq8w8U= @@ -225,6 +227,10 @@ github.com/filecoin-project/sector-storage v0.0.0-20200609231555-252c2b0c969d h1 github.com/filecoin-project/sector-storage v0.0.0-20200609231555-252c2b0c969d/go.mod h1:SfaVNw/A6LwqqEt6wEMVBN5Grn3XC50j+yjmegHIe7Y= github.com/filecoin-project/sector-storage v0.0.0-20200615123301-7d09fa88b4b0 h1:DpdlpS3G2XN+F68rxOTAXsGL3cV7UkFr0TxdmKSiaJg= github.com/filecoin-project/sector-storage v0.0.0-20200615123301-7d09fa88b4b0/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= +github.com/filecoin-project/sector-storage v0.0.0-20200615142857-3d1225bf3449 h1:aFXA4iI8WumN1pgEv5gtAyZs3mEX7AdV2xqHf76Dn6o= +github.com/filecoin-project/sector-storage v0.0.0-20200615142857-3d1225bf3449/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= +github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6 h1:NIcubpeasVs++K5EFelMXeURRb8sWCuXQNOSWnvTc14= +github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= @@ -241,6 +247,10 @@ github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sg github.com/filecoin-project/specs-storage v0.1.0/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa h1:kTCIBKMhhhVcjCh1ws72vpdDr4cdZjCyIUDFijuWuvA= github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa/go.mod h1:S0u14Wr55mpe22lElCSKbXrhtWg/jquVTTMhefQ8f4Q= +github.com/filecoin-project/storage-fsm v0.0.0-20200615142724-95dbdc803c53 h1:mvvBaFPyYugJfmHgFHg0BgE0plwV2IhfgL6tLu02umM= +github.com/filecoin-project/storage-fsm v0.0.0-20200615142724-95dbdc803c53/go.mod h1:ZBID+J03DzANc0cv+sv03ROqfYYZbl62d3u8JftBH6k= +github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743 h1:a8f1p6UdeD+ZINBKJN4FhEos8uaKeASOAabq5RCpQdg= +github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743/go.mod h1:q1YCutTSMq/yGYvDPHReT37bPfDLHltnwJutzR9kOY0= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= diff --git a/lotuspond/spawn.go b/lotuspond/spawn.go index 32fc69b05..8100d784a 100644 --- a/lotuspond/spawn.go +++ b/lotuspond/spawn.go @@ -25,8 +25,8 @@ import ( ) func init() { - miner.SupportedProofTypes = map[abi.RegisteredProof]struct{}{ - abi.RegisteredProof_StackedDRG2KiBSeal: {}, + miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } } @@ -49,7 +49,7 @@ func (api *api) Spawn() (nodeInfo, error) { } sbroot := filepath.Join(dir, "preseal") - genm, ki, err := seed.PreSeal(genMiner, abi.RegisteredProof_StackedDRG2KiBSeal, 0, 2, sbroot, []byte("8"), nil) + genm, ki, err := seed.PreSeal(genMiner, abi.RegisteredSealProof_StackedDrg2KiBV1, 0, 2, sbroot, []byte("8"), nil) if err != nil { return nodeInfo{}, xerrors.Errorf("preseal failed: %w", err) } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 1d68a1858..e39527201 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -750,7 +750,7 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr initialPledge := big.Zero() { - ssize, err := precommit.Info.RegisteredProof.SectorSize() + ssize, err := precommit.Info.SealProof.SectorSize() if err != nil { return types.EmptyInt, err } diff --git a/node/node_test.go b/node/node_test.go index 8b17fac2e..a3a37bdd3 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -54,8 +54,8 @@ func init() { _ = logging.SetLogLevel("*", "INFO") power.ConsensusMinerMinPower = big.NewInt(2048) - saminer.SupportedProofTypes = map[abi.RegisteredProof]struct{}{ - abi.RegisteredProof_StackedDRG2KiBSeal: {}, + saminer.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } verifreg.MinVerifiedDealSize = big.NewInt(256) } @@ -189,7 +189,7 @@ func builder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestN if err != nil { t.Fatal(err) } - genm, k, err := seed.PreSeal(maddr, abi.RegisteredProof_StackedDRG2KiBPoSt, 0, test.GenesisPreseals, tdir, []byte("make genesis mem random"), nil) + genm, k, err := seed.PreSeal(maddr, abi.RegisteredSealProof_StackedDrg2KiBV1, 0, test.GenesisPreseals, tdir, []byte("make genesis mem random"), nil) if err != nil { t.Fatal(err) } diff --git a/storage/adapter_storage_miner.go b/storage/adapter_storage_miner.go index 7821b009c..5fc811997 100644 --- a/storage/adapter_storage_miner.go +++ b/storage/adapter_storage_miner.go @@ -98,7 +98,7 @@ func (s SealingAPIAdapter) StateWaitMsg(ctx context.Context, mcid cid.Cid) (seal }, nil } -func (s SealingAPIAdapter) StateComputeDataCommitment(ctx context.Context, maddr address.Address, sectorType abi.RegisteredProof, deals []abi.DealID, tok sealing.TipSetToken) (cid.Cid, error) { +func (s SealingAPIAdapter) StateComputeDataCommitment(ctx context.Context, maddr address.Address, sectorType abi.RegisteredSealProof, deals []abi.DealID, tok sealing.TipSetToken) (cid.Cid, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return cid.Undef, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) diff --git a/storage/miner.go b/storage/miner.go index 171c166aa..514af527a 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -135,7 +135,7 @@ type StorageWpp struct { prover storage.Prover verifier ffiwrapper.Verifier miner abi.ActorID - winnRpt abi.RegisteredProof + winnRpt abi.RegisteredPoStProof } func NewWinningPoStProver(api api.FullNode, prover storage.Prover, verifier ffiwrapper.Verifier, miner dtypes.MinerID) (*StorageWpp, error) { diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 8cd9fdc5f..01d840848 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -464,9 +464,9 @@ func (s *WindowPoStScheduler) sortedSectorInfo(ctx context.Context, deadlineSect sbsi := make([]abi.SectorInfo, len(sset)) for k, sector := range sset { sbsi[k] = abi.SectorInfo{ - SectorNumber: sector.ID, - SealedCID: sector.Info.Info.SealedCID, - RegisteredProof: sector.Info.Info.RegisteredProof, + SectorNumber: sector.ID, + SealedCID: sector.Info.Info.SealedCID, + SealProof: sector.Info.Info.SealProof, } } diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index 8b405747e..077209a4e 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -24,7 +24,7 @@ type WindowPoStScheduler struct { api storageMinerApi prover storage.Prover faultTracker sectorstorage.FaultTracker - proofType abi.RegisteredProof + proofType abi.RegisteredPoStProof partitionSectors uint64 actor address.Address From c832c0bf3fdfe56b87932fc3ddcc9988c4112765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 15 Jun 2020 18:31:18 +0200 Subject: [PATCH 67/81] mod tidy --- go.mod | 2 -- go.sum | 26 ++------------------------ 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index df0c03c9f..3b35c53d3 100644 --- a/go.mod +++ b/go.mod @@ -127,5 +127,3 @@ require ( replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v1.18.0 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi - -replace github.com/filecoin-project/go-fil-markets => /home/magik6k/gohack/github.com/filecoin-project/go-fil-markets diff --git a/go.sum b/go.sum index c4a07cf66..4ae1fb3af 100644 --- a/go.sum +++ b/go.sum @@ -182,8 +182,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= -github.com/filecoin-project/chain-validation v0.0.6-0.20200605221044-7f78284bbc94 h1:svEal/usZ/REEjEz6ije3/ALn8duM2CckR0LjT1btx0= -github.com/filecoin-project/chain-validation v0.0.6-0.20200605221044-7f78284bbc94/go.mod h1:aOmmdhO0xIRtWCwx3MyIv3tPCCorM1Bq7pNOJGMLndA= github.com/filecoin-project/chain-validation v0.0.6-0.20200615142407-39c0fe23a3d5 h1:8Ytl1uk8/Sff7Z2vfb7MOEM3ZRlQKJPqn19FzDcr+bc= github.com/filecoin-project/chain-validation v0.0.6-0.20200615142407-39c0fe23a3d5/go.mod h1:HEJn6kOXMNhCNBYNTO/lrEI7wSgqCOR6hN5ecfYUnC8= github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= @@ -193,7 +191,6 @@ github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef/go.m github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg= github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2 h1:jamfsxfK0Q9yCMHt8MPWx7Aa/O9k2Lve8eSc6FILYGQ= github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg= -github.com/filecoin-project/go-bitfield v0.0.0-20200309034705-8c7ac40bd550/go.mod h1:iodsLxOFZnqKtjj2zkgqzoGNrv6vUqj69AT/J8DKXEw= github.com/filecoin-project/go-bitfield v0.0.0-20200416002808-b3ee67ec9060/go.mod h1:iodsLxOFZnqKtjj2zkgqzoGNrv6vUqj69AT/J8DKXEw= github.com/filecoin-project/go-bitfield v0.0.1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e h1:gkG/7G+iKy4He+IiQNeQn+nndFznb/vCoOR8iRQsm60= @@ -206,8 +203,8 @@ github.com/filecoin-project/go-data-transfer v0.3.0 h1:BwBrrXu9Unh9JjjX4GAc5FfzU github.com/filecoin-project/go-data-transfer v0.3.0/go.mod h1:cONglGP4s/d+IUQw5mWZrQK+FQATQxr3AXzi4dRh0l4= github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1:yvQJCW9mmi9zy+51xA01Ea2X7/dL7r8eKDPuGUjRmbo= github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA= -github.com/filecoin-project/go-fil-markets v0.2.7 h1:bgdK/e+xW15aVZLtdFLzAHdrx1hqtGF9veg2lstLK6o= -github.com/filecoin-project/go-fil-markets v0.2.7/go.mod h1:LI3VFHse33aU0djAmFQ8+Hg39i0J8ibAoppGu6TbgkA= +github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615162707-5dd6b2eb253b h1:xhaFY+rdZMDErIkUHvkJWdN6H3h/dn/5U3GOX4HP/Cs= +github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615162707-5dd6b2eb253b/go.mod h1:nCjNrwgRsjX1TSiBUVgewa5PbP19x5Mq07uzlJD9YUw= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM= github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs= @@ -222,33 +219,14 @@ github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIi github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= -github.com/filecoin-project/sector-storage v0.0.0-20200508203401-a74812ba12f3/go.mod h1:B+xzopr/oWZJz2hBL5Ekb7Obcum5ntmfbaAUlaaho28= -github.com/filecoin-project/sector-storage v0.0.0-20200609231555-252c2b0c969d h1:q7KC8/yIirwVX7JCLgjEmo5LX77V8tYV0EzVCnCwrE8= -github.com/filecoin-project/sector-storage v0.0.0-20200609231555-252c2b0c969d/go.mod h1:SfaVNw/A6LwqqEt6wEMVBN5Grn3XC50j+yjmegHIe7Y= -github.com/filecoin-project/sector-storage v0.0.0-20200615123301-7d09fa88b4b0 h1:DpdlpS3G2XN+F68rxOTAXsGL3cV7UkFr0TxdmKSiaJg= -github.com/filecoin-project/sector-storage v0.0.0-20200615123301-7d09fa88b4b0/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= -github.com/filecoin-project/sector-storage v0.0.0-20200615142857-3d1225bf3449 h1:aFXA4iI8WumN1pgEv5gtAyZs3mEX7AdV2xqHf76Dn6o= -github.com/filecoin-project/sector-storage v0.0.0-20200615142857-3d1225bf3449/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6 h1:NIcubpeasVs++K5EFelMXeURRb8sWCuXQNOSWnvTc14= github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= -github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= -github.com/filecoin-project/specs-actors v0.5.4-0.20200521014528-0df536f7e461/go.mod h1:r5btrNzZD0oBkEz1pohv80gSCXQnqGrD0kYwOTiExyE= -github.com/filecoin-project/specs-actors v0.5.5 h1:bDsowem6dLRc9B7g3sgFYvHgfWTH4D9q5ehWLGLdKCw= -github.com/filecoin-project/specs-actors v0.5.5/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= -github.com/filecoin-project/specs-actors v0.5.6 h1:WlhtoXwFoKlP1b06NI4NJaxC4m9EXNV+qFVl43/xRN8= -github.com/filecoin-project/specs-actors v0.5.6/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-actors v0.6.0 h1:IepUsmDGY60QliENVTkBTAkwqGWw9kNbbHOcU/9oiC0= github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= -github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 h1:T3f/zkuvgtgqcXrb0NO3BicuveGOxxUAMPa/Yif2kuE= -github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102/go.mod h1:xJ1/xl9+8zZeSSSFmDC3Wr6uusCTxyYPI0VeNVSFmPE= github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sgOVdJbvFjOnD5w94= github.com/filecoin-project/specs-storage v0.1.0/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= -github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa h1:kTCIBKMhhhVcjCh1ws72vpdDr4cdZjCyIUDFijuWuvA= -github.com/filecoin-project/storage-fsm v0.0.0-20200605082304-aa405b2176aa/go.mod h1:S0u14Wr55mpe22lElCSKbXrhtWg/jquVTTMhefQ8f4Q= -github.com/filecoin-project/storage-fsm v0.0.0-20200615142724-95dbdc803c53 h1:mvvBaFPyYugJfmHgFHg0BgE0plwV2IhfgL6tLu02umM= -github.com/filecoin-project/storage-fsm v0.0.0-20200615142724-95dbdc803c53/go.mod h1:ZBID+J03DzANc0cv+sv03ROqfYYZbl62d3u8JftBH6k= github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743 h1:a8f1p6UdeD+ZINBKJN4FhEos8uaKeASOAabq5RCpQdg= github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743/go.mod h1:q1YCutTSMq/yGYvDPHReT37bPfDLHltnwJutzR9kOY0= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= From 58a79773ee8467fe3ac323543146554e5c69682a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 15 Jun 2020 18:34:36 +0200 Subject: [PATCH 68/81] fix debug build --- build/params_2k.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/params_2k.go b/build/params_2k.go index 56e1ca2f8..046753678 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -12,8 +12,8 @@ import ( func init() { power.ConsensusMinerMinPower = big.NewInt(2048) - miner.SupportedProofTypes = map[abi.RegisteredProof]struct{}{ - abi.RegisteredProof_StackedDRG2KiBSeal: {}, + miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } verifreg.MinVerifiedDealSize = big.NewInt(256) From 8039701f9ef8bbab42ceb9f4894c666cd320f123 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 15 Jun 2020 19:50:43 +0200 Subject: [PATCH 69/81] Use extra field in gastrace Signed-off-by: Jakub Sztandera --- chain/types/execresult.go | 2 +- chain/vm/gas.go | 10 +++++++++- chain/vm/gas_v0.go | 6 +++--- chain/vm/runtime.go | 14 +++++++++----- cli/state.go | 2 +- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/chain/types/execresult.go b/chain/types/execresult.go index b0e1f760d..443147f9e 100644 --- a/chain/types/execresult.go +++ b/chain/types/execresult.go @@ -30,7 +30,7 @@ type GasTrace struct { VirtualStorageGas int64 TimeTaken time.Duration - Extra interface{} + Extra interface{} `json:",omitempty"` Callers []uintptr `json:"-"` } diff --git a/chain/vm/gas.go b/chain/vm/gas.go index dc9966e57..2540afbf0 100644 --- a/chain/vm/gas.go +++ b/chain/vm/gas.go @@ -18,7 +18,9 @@ const ( ) type GasCharge struct { - Name string + Name string + Extra interface{} + ComputeGas int64 StorageGas int64 @@ -36,6 +38,12 @@ func (g GasCharge) WithVirtual(compute, storage int64) GasCharge { return out } +func (g GasCharge) WithExtra(extra interface{}) GasCharge { + out := g + out.Extra = extra + return out +} + func newGasCharge(name string, computeGas int64, storageGas int64) GasCharge { return GasCharge{ Name: name, diff --git a/chain/vm/gas_v0.go b/chain/vm/gas_v0.go index c631beaee..e2a908a23 100644 --- a/chain/vm/gas_v0.go +++ b/chain/vm/gas_v0.go @@ -108,12 +108,12 @@ func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.M // OnIpldGet returns the gas used for storing an object func (pl *pricelistV0) OnIpldGet(dataSize int) GasCharge { - return newGasCharge(fmt.Sprintf("OnIpldGet:%db", dataSize), pl.ipldGetBase+int64(dataSize)*pl.ipldGetPerByte, 0) + return newGasCharge("OnIpldGet", pl.ipldGetBase+int64(dataSize)*pl.ipldGetPerByte, 0).WithExtra(dataSize) } // OnIpldPut returns the gas used for storing an object func (pl *pricelistV0) OnIpldPut(dataSize int) GasCharge { - return newGasCharge(fmt.Sprintf("OnIpldPut:%db", dataSize), pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte) + return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte).WithExtra(dataSize) } // OnCreateActor returns the gas used for creating an actor @@ -133,7 +133,7 @@ func (pl *pricelistV0) OnVerifySignature(sigType crypto.SigType, planTextSize in return GasCharge{}, fmt.Errorf("cost function for signature type %d not supported", sigType) } sigName, _ := sigType.Name() - return newGasCharge("OnVerifySignature/"+sigName, costFn(int64(planTextSize)), 0), nil + return newGasCharge("OnVerifySignature", costFn(int64(planTextSize)), 0).WithExtra(sigName), nil } // OnHashing diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 9b83593ab..48563df64 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -524,14 +524,18 @@ func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError } gasTrace := types.GasTrace{ - Name: gas.Name, - TotalGas: toUse, - ComputeGas: gas.ComputeGas, - StorageGas: gas.StorageGas, + Name: gas.Name, + Extra: gas.Extra, + + TotalGas: toUse, + ComputeGas: gas.ComputeGas, + StorageGas: gas.StorageGas, + TotalVirtualGas: gas.VirtualCompute*GasComputeMulti + gas.VirtualStorage*GasStorageMulti, VirtualComputeGas: gas.VirtualCompute, VirtualStorageGas: gas.VirtualStorage, - Callers: callers[:cout], + + Callers: callers[:cout], } rt.executionTrace.GasCharges = append(rt.executionTrace.GasCharges, &gasTrace) rt.lastGasChargeTime = now diff --git a/cli/state.go b/cli/state.go index 753b0b017..1aad85299 100644 --- a/cli/state.go +++ b/cli/state.go @@ -1072,7 +1072,7 @@ var compStateMsg = ` {{- end}} {{range .GasCharges}} - {{.Name}} + {{.Name}}{{if .Extra}}:{{.Extra}}{{end}} {{template "gasC" .}} {{.TimeTaken}} From daca8651f41efacacd804b77db6c8b7959751f0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 15 Jun 2020 19:53:21 +0200 Subject: [PATCH 70/81] Fix bench --- cmd/lotus-bench/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 57f5241df..a10c3c00e 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -284,12 +284,12 @@ var sealBenchCmd = &cli.Command{ if !c.Bool("skip-commit2") { log.Info("generating winning post candidates") - wpt, err := spt.RegisteredWindowPoStProof() + wipt, err := spt.RegisteredWinningPoStProof() if err != nil { return err } - fcandidates, err := ffiwrapper.ProofVerifier.GenerateWinningPoStSectorChallenge(context.TODO(), wpt, mid, challenge[:], uint64(len(sealedSectors))) + fcandidates, err := ffiwrapper.ProofVerifier.GenerateWinningPoStSectorChallenge(context.TODO(), wipt, mid, challenge[:], uint64(len(sealedSectors))) if err != nil { return err } From f8669d4e9547d0f322cc735baf22529e58360e5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 15 Jun 2020 21:21:40 +0200 Subject: [PATCH 71/81] Update chain-val and sector-storage --- go.mod | 4 ++-- go.sum | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 3b35c53d3..89e2e2842 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/drand/drand v0.9.2-0.20200529123141-6b138aefaef2 github.com/drand/kyber v1.0.2 github.com/fatih/color v1.8.0 - github.com/filecoin-project/chain-validation v0.0.6-0.20200615142407-39c0fe23a3d5 + github.com/filecoin-project/chain-validation v0.0.6-0.20200615191232-6be1a8c6ed09 github.com/filecoin-project/filecoin-ffi v0.26.1-0.20200508175440-05b30afeb00d github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2 @@ -29,7 +29,7 @@ require ( github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6 + github.com/filecoin-project/sector-storage v0.0.0-20200615192001-42c9e08595b7 github.com/filecoin-project/specs-actors v0.6.0 github.com/filecoin-project/specs-storage v0.1.0 github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743 diff --git a/go.sum b/go.sum index 4ae1fb3af..83af4904c 100644 --- a/go.sum +++ b/go.sum @@ -182,8 +182,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= -github.com/filecoin-project/chain-validation v0.0.6-0.20200615142407-39c0fe23a3d5 h1:8Ytl1uk8/Sff7Z2vfb7MOEM3ZRlQKJPqn19FzDcr+bc= -github.com/filecoin-project/chain-validation v0.0.6-0.20200615142407-39c0fe23a3d5/go.mod h1:HEJn6kOXMNhCNBYNTO/lrEI7wSgqCOR6hN5ecfYUnC8= +github.com/filecoin-project/chain-validation v0.0.6-0.20200615191232-6be1a8c6ed09 h1:GuiNSEZ9nc05LUpKhABw/SO6t9wqCfsJX1D0ByWQjkc= +github.com/filecoin-project/chain-validation v0.0.6-0.20200615191232-6be1a8c6ed09/go.mod h1:HEJn6kOXMNhCNBYNTO/lrEI7wSgqCOR6hN5ecfYUnC8= github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef h1:Wi5E+P1QfHP8IF27eUiTx5vYfqQZwfPxzq3oFEq8w8U= @@ -219,8 +219,9 @@ github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIi github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= -github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6 h1:NIcubpeasVs++K5EFelMXeURRb8sWCuXQNOSWnvTc14= github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= +github.com/filecoin-project/sector-storage v0.0.0-20200615192001-42c9e08595b7 h1:cjsOpQKvZosPx9/qqq2bucHVdRyXzvBR1f37atiR3/0= +github.com/filecoin-project/sector-storage v0.0.0-20200615192001-42c9e08595b7/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= github.com/filecoin-project/specs-actors v0.6.0 h1:IepUsmDGY60QliENVTkBTAkwqGWw9kNbbHOcU/9oiC0= From 9262661135706070325d8b9bfea9c5e72841c719 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Mon, 15 Jun 2020 15:43:47 -0700 Subject: [PATCH 72/81] chore(deps): update to fil-markets 0.3.0 Updates to latest fil markets with resumability and other fixes --- go.mod | 2 +- go.sum | 6 ++++-- node/builder.go | 2 -- node/impl/storminer.go | 2 +- node/modules/client.go | 3 +-- node/modules/services.go | 18 ------------------ node/modules/storageminer.go | 4 ++-- 7 files changed, 9 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index 89e2e2842..ca503ce49 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.3.0 github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 - github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615162707-5dd6b2eb253b + github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615223906-0fb36d5347bd github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca diff --git a/go.sum b/go.sum index 83af4904c..c6c69454b 100644 --- a/go.sum +++ b/go.sum @@ -203,8 +203,8 @@ github.com/filecoin-project/go-data-transfer v0.3.0 h1:BwBrrXu9Unh9JjjX4GAc5FfzU github.com/filecoin-project/go-data-transfer v0.3.0/go.mod h1:cONglGP4s/d+IUQw5mWZrQK+FQATQxr3AXzi4dRh0l4= github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1:yvQJCW9mmi9zy+51xA01Ea2X7/dL7r8eKDPuGUjRmbo= github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA= -github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615162707-5dd6b2eb253b h1:xhaFY+rdZMDErIkUHvkJWdN6H3h/dn/5U3GOX4HP/Cs= -github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615162707-5dd6b2eb253b/go.mod h1:nCjNrwgRsjX1TSiBUVgewa5PbP19x5Mq07uzlJD9YUw= +github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615223906-0fb36d5347bd h1:yQqt+9CDKdZhg1VZEgV8O5tA+D1f0mairODT7dcwHtU= +github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615223906-0fb36d5347bd/go.mod h1:UXsXi43AyUQ5ieb4yIaLgk4PVt7TAbl1UCccuNw+7ds= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM= github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs= @@ -215,6 +215,8 @@ github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca h github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 h1:k9qVR9ItcziSB2rxtlkN/MDWNlbsI6yzec+zjUatLW0= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= +github.com/filecoin-project/go-statemachine v0.0.0-20200612181802-4eb3d0c68eba h1:GEWb/6KQyNZt4jm8fgVcIFPH0ElAGXfHM59ZSiqPTvY= +github.com/filecoin-project/go-statemachine v0.0.0-20200612181802-4eb3d0c68eba/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= diff --git a/node/builder.go b/node/builder.go index 25b027145..536d81901 100644 --- a/node/builder.go +++ b/node/builder.go @@ -106,7 +106,6 @@ const ( HandleIncomingBlocksKey HandleIncomingMessagesKey - RunDealClientKey RegisterClientValidatorKey // storage miner @@ -266,7 +265,6 @@ func Online() Option { Override(new(storagemarket.StorageClient), modules.StorageClient), Override(new(storagemarket.StorageClientNode), storageadapter.NewClientNodeAdapter), Override(RegisterClientValidatorKey, modules.RegisterClientValidator), - Override(RunDealClientKey, modules.RunDealClient), Override(new(beacon.RandomBeacon), modules.RandomBeacon), Override(new(*paychmgr.Store), paychmgr.NewStore), diff --git a/node/impl/storminer.go b/node/impl/storminer.go index de80eb4cd..a01cad9c2 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -202,7 +202,7 @@ func (sm *StorageMinerAPI) MarketListIncompleteDeals(ctx context.Context) ([]sto } func (sm *StorageMinerAPI) MarketSetPrice(ctx context.Context, p types.BigInt) error { - return sm.StorageProvider.AddAsk(abi.TokenAmount(p), 60*60*24*100) // lasts for 100 days? + return sm.StorageProvider.SetAsk(abi.TokenAmount(p), 60*60*24*100) // lasts for 100 days? } func (sm *StorageMinerAPI) DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) { diff --git a/node/modules/client.go b/node/modules/client.go index 12ec209b5..b4618015a 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -117,8 +117,7 @@ func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, r } lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { - c.Run(ctx) - return nil + return c.Start(ctx) }, OnStop: func(context.Context) error { c.Stop() diff --git a/node/modules/services.go b/node/modules/services.go index f4a007c2b..27244ad3a 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -1,8 +1,6 @@ package modules import ( - "context" - "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" eventbus "github.com/libp2p/go-eventbus" @@ -15,7 +13,6 @@ import ( "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" - "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/beacon" @@ -100,21 +97,6 @@ func HandleIncomingMessages(mctx helpers.MetricsCtx, lc fx.Lifecycle, ps *pubsub go sub.HandleIncomingMessages(ctx, mpool, msgsub) } -func RunDealClient(mctx helpers.MetricsCtx, lc fx.Lifecycle, c storagemarket.StorageClient) { - ctx := helpers.LifecycleCtx(mctx, lc) - - lc.Append(fx.Hook{ - OnStart: func(context.Context) error { - c.Run(ctx) - return nil - }, - OnStop: func(context.Context) error { - c.Stop() - return nil - }, - }) -} - func NewLocalDiscovery(ds dtypes.MetadataDS) *discovery.Local { return discovery.NewLocal(namespace.Wrap(ds, datastore.NewKey("/deals/local"))) } diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 440aa8593..fce9f9e1f 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -299,8 +299,8 @@ func NewStorageAsk(ctx helpers.MetricsCtx, fapi lapi.FullNode, ds dtypes.Metadat return nil, err } // Hacky way to set max piece size to the sector size - a := storedAsk.GetAsk(address.Address(minerAddress)).Ask - err = storedAsk.AddAsk(a.Price, a.Expiry-a.Timestamp, storagemarket.MaxPieceSize(abi.PaddedPieceSize(mi.SectorSize))) + a := storedAsk.GetAsk().Ask + err = storedAsk.SetAsk(a.Price, a.Expiry-a.Timestamp, storagemarket.MaxPieceSize(abi.PaddedPieceSize(mi.SectorSize))) if err != nil { return storedAsk, err } From a4226478198041b969b2f273bfa2d85266dfef3d Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Mon, 15 Jun 2020 15:53:39 -0700 Subject: [PATCH 73/81] fix(lint): fix lint error fix lint error and update to tagged package --- go.mod | 2 +- go.sum | 4 ++-- node/impl/storminer.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index ca503ce49..8457847a6 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.3.0 github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 - github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615223906-0fb36d5347bd + github.com/filecoin-project/go-fil-markets v0.3.0 github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca diff --git a/go.sum b/go.sum index c6c69454b..47a405efa 100644 --- a/go.sum +++ b/go.sum @@ -203,8 +203,8 @@ github.com/filecoin-project/go-data-transfer v0.3.0 h1:BwBrrXu9Unh9JjjX4GAc5FfzU github.com/filecoin-project/go-data-transfer v0.3.0/go.mod h1:cONglGP4s/d+IUQw5mWZrQK+FQATQxr3AXzi4dRh0l4= github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1:yvQJCW9mmi9zy+51xA01Ea2X7/dL7r8eKDPuGUjRmbo= github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA= -github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615223906-0fb36d5347bd h1:yQqt+9CDKdZhg1VZEgV8O5tA+D1f0mairODT7dcwHtU= -github.com/filecoin-project/go-fil-markets v0.2.8-0.20200615223906-0fb36d5347bd/go.mod h1:UXsXi43AyUQ5ieb4yIaLgk4PVt7TAbl1UCccuNw+7ds= +github.com/filecoin-project/go-fil-markets v0.3.0 h1:7iCGiuTSia4f4DmOn3s96NWUwMNSOI0ZHel/XgeApAQ= +github.com/filecoin-project/go-fil-markets v0.3.0/go.mod h1:UXsXi43AyUQ5ieb4yIaLgk4PVt7TAbl1UCccuNw+7ds= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM= github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs= diff --git a/node/impl/storminer.go b/node/impl/storminer.go index a01cad9c2..0cc13177e 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -202,7 +202,7 @@ func (sm *StorageMinerAPI) MarketListIncompleteDeals(ctx context.Context) ([]sto } func (sm *StorageMinerAPI) MarketSetPrice(ctx context.Context, p types.BigInt) error { - return sm.StorageProvider.SetAsk(abi.TokenAmount(p), 60*60*24*100) // lasts for 100 days? + return sm.StorageProvider.SetAsk(p, 60*60*24*100) // lasts for 100 days? } func (sm *StorageMinerAPI) DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) { From 69948cd09c7e2345c4dab220ff7cf4af586781d9 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 15 Jun 2020 21:00:22 -0400 Subject: [PATCH 74/81] WalletSign should resolve ID addresses --- node/impl/full/wallet.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index 7d7cbae67..8da8d3062 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -40,7 +40,11 @@ func (a *WalletAPI) WalletBalance(ctx context.Context, addr address.Address) (ty } func (a *WalletAPI) WalletSign(ctx context.Context, k address.Address, msg []byte) (*crypto.Signature, error) { - return a.Wallet.Sign(ctx, k, msg) + keyAddr, err := a.StateManager.ResolveToKeyAddress(ctx, k, nil) + if err != nil { + return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr) + } + return a.Wallet.Sign(ctx, keyAddr, msg) } func (a *WalletAPI) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { From ed3b2716af6b4dfbd906e5b980c20ae15361db5f Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 15 Jun 2020 21:02:25 -0400 Subject: [PATCH 75/81] Don't verify ElectionPostVRF when insecure validation enabled --- chain/sync.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index 07d470d28..05c0bc7cf 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -650,7 +650,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err return xerrors.Errorf("could not draw randomness: %w", err) } - if err := gen.VerifyVRF(ctx, waddr, vrfBase, h.ElectionProof.VRFProof); err != nil { + if err := VerifyElectionPoStVRF(ctx, waddr, vrfBase, h.ElectionProof.VRFProof); err != nil { return xerrors.Errorf("validating block election proof failed: %w", err) } @@ -711,7 +711,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err return xerrors.Errorf("failed to compute vrf base for ticket: %w", err) } - err = gen.VerifyVRF(ctx, waddr, vrfBase, h.Ticket.VRFProof) + err = VerifyElectionPoStVRF(ctx, waddr, vrfBase, h.Ticket.VRFProof) if err != nil { return xerrors.Errorf("validating block tickets failed: %w", err) } @@ -1336,12 +1336,12 @@ func (syncer *Syncer) collectChain(ctx context.Context, ts *types.TipSet) error return nil } -func VerifyElectionPoStVRF(ctx context.Context, evrf []byte, rand []byte, worker address.Address) error { - if err := gen.VerifyVRF(ctx, worker, rand, evrf); err != nil { - return xerrors.Errorf("failed to verify post_randomness vrf: %w", err) +func VerifyElectionPoStVRF(ctx context.Context, worker address.Address, rand []byte, evrf []byte) error { + if build.InsecurePoStValidation { + return nil + } else { + return gen.VerifyVRF(ctx, worker, rand, evrf) } - - return nil } func (syncer *Syncer) State() []SyncerState { From a9abf3ceb2ac791f3ff2908046d9ac743a3d84ab Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 15 Jun 2020 21:02:53 -0400 Subject: [PATCH 76/81] extract the fake Wpost used for testing --- chain/gen/gen.go | 8 +++++--- cmd/lotus/debug_advance.go | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/chain/gen/gen.go b/chain/gen/gen.go index c9ebebcef..172c73845 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -45,6 +45,10 @@ var log = logging.Logger("gen") const msgsPerBlock = 20 +var ValidWpostForTesting = []abi.PoStProof{{ + ProofBytes: []byte("valid proof"), +}} + type ChainGen struct { msgsPerBlock int @@ -529,9 +533,7 @@ func (wpp *wppProvider) GenerateCandidates(ctx context.Context, _ abi.PoStRandom } func (wpp *wppProvider) ComputeProof(context.Context, []abi.SectorInfo, abi.PoStRandomness) ([]abi.PoStProof, error) { - return []abi.PoStProof{{ - ProofBytes: []byte("valid proof"), - }}, nil + return ValidWpostForTesting, nil } type ProofInput struct { diff --git a/cmd/lotus/debug_advance.go b/cmd/lotus/debug_advance.go index 990bbe95d..b18878d9d 100644 --- a/cmd/lotus/debug_advance.go +++ b/cmd/lotus/debug_advance.go @@ -72,7 +72,7 @@ func init() { uts := head.MinTimestamp() + uint64(build.BlockDelay) nheight := head.Height() + 1 blk, err := api.MinerCreateBlock(ctx, &lapi.BlockTemplate{ - addr, head.Key(), ticket, nil, nil, msgs, nheight, uts, nil, + addr, head.Key(), ticket, &types.ElectionProof{}, nil, msgs, nheight, uts, gen.ValidWpostForTesting, }) if err != nil { return xerrors.Errorf("creating block: %w", err) From f31fcdae08b1ed26b74d8c52de35ee896933c3d6 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Mon, 15 Jun 2020 20:34:48 +0000 Subject: [PATCH 77/81] stats: increase miner power lookup perf --- tools/stats/metrics.go | 84 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 77 insertions(+), 7 deletions(-) diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index eb653e1d7..53fa6ed63 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "encoding/json" "fmt" @@ -13,9 +14,14 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/specs-actors/actors/builtin/power" + "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" + cbg "github.com/whyrusleeping/cbor-gen" + _ "github.com/influxdata/influxdb1-client" models "github.com/influxdata/influxdb1-client/models" client "github.com/influxdata/influxdb1-client/v2" @@ -135,6 +141,36 @@ func RecordTipsetPoints(ctx context.Context, api api.FullNode, pl *PointList, ti return nil } +type apiIpldStore struct { + ctx context.Context + api api.FullNode +} + +func (ht *apiIpldStore) Context() context.Context { + return ht.ctx +} + +func (ht *apiIpldStore) Get(ctx context.Context, c cid.Cid, out interface{}) error { + raw, err := ht.api.ChainReadObj(ctx, c) + if err != nil { + return err + } + + cu, ok := out.(cbg.CBORUnmarshaler) + if ok { + if err := cu.UnmarshalCBOR(bytes.NewReader(raw)); err != nil { + return err + } + return nil + } + + return fmt.Errorf("Object does not implement CBORUnmarshaler") +} + +func (ht *apiIpldStore) Put(ctx context.Context, v interface{}) (cid.Cid, error) { + return cid.Undef, fmt.Errorf("Put is not implemented on apiIpldStore") +} + func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointList, tipset *types.TipSet) error { pc, err := api.StatePledgeCollateral(ctx, tipset.Key()) if err != nil { @@ -158,24 +194,58 @@ func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointLis p = NewPoint("network.balance", netBalFilFloat) pl.AddPoint(p) - power, err := api.StateMinerPower(ctx, address.Address{}, tipset.Key()) + totalPower, err := api.StateMinerPower(ctx, address.Address{}, tipset.Key()) if err != nil { return err } - p = NewPoint("chain.power", power.TotalPower.QualityAdjPower.Int64()) + p = NewPoint("chain.power", totalPower.TotalPower.QualityAdjPower.Int64()) pl.AddPoint(p) - miners, err := api.StateListMiners(ctx, tipset.Key()) - for _, miner := range miners { - power, err := api.StateMinerPower(ctx, miner, tipset.Key()) + powerActor, err := api.StateGetActor(ctx, builtin.StoragePowerActorAddr, tipset.Key()) + if err != nil { + return err + } + + powerRaw, err := api.ChainReadObj(ctx, powerActor.Head) + if err != nil { + return err + } + + var powerActorState power.State + + if err := powerActorState.UnmarshalCBOR(bytes.NewReader(powerRaw)); err != nil { + return fmt.Errorf("failed to unmarshal power actor state: %w", err) + } + + s := &apiIpldStore{ctx, api} + mp, err := adt.AsMap(s, powerActorState.Claims) + if err != nil { + return err + } + + err = mp.ForEach(nil, func(key string) error { + addr, err := address.NewFromBytes([]byte(key)) if err != nil { return err } - p = NewPoint("chain.miner_power", power.MinerPower.QualityAdjPower.Int64()) - p.AddTag("miner", miner.String()) + var claim power.Claim + keyerAddr := adt.AddrKey(addr) + mp.Get(keyerAddr, &claim) + + if claim.QualityAdjPower.Int64() == 0 { + return nil + } + + p = NewPoint("chain.miner_power", claim.QualityAdjPower.Int64()) + p.AddTag("miner", addr.String()) pl.AddPoint(p) + + return nil + }) + if err != nil { + return err } return nil From 18dd1c277be738316616671efd3e0ff8a5d33dd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 16 Jun 2020 12:28:28 +0200 Subject: [PATCH 78/81] Update ffi --- extern/filecoin-ffi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 5bb4a309b..ca281af0b 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 5bb4a309bce9d446ac618f34a8b9e2883af2002f +Subproject commit ca281af0b6c00314382a75ae869e5cb22c83655b From bac6a47c6753382b5f746c9984d8e5a8c733d2a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 16 Jun 2020 12:30:54 +0200 Subject: [PATCH 79/81] specs-actors v0.6.1 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 8457847a6..14c3a39f3 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/sector-storage v0.0.0-20200615192001-42c9e08595b7 - github.com/filecoin-project/specs-actors v0.6.0 + github.com/filecoin-project/specs-actors v0.6.1 github.com/filecoin-project/specs-storage v0.1.0 github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index 47a405efa..6c45b1daa 100644 --- a/go.sum +++ b/go.sum @@ -228,6 +228,8 @@ github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.m github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= github.com/filecoin-project/specs-actors v0.6.0 h1:IepUsmDGY60QliENVTkBTAkwqGWw9kNbbHOcU/9oiC0= github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= +github.com/filecoin-project/specs-actors v0.6.1 h1:rhHlEzqcuuQU6oKc4csuq+/kQBDZ4EXtSomoN2XApCA= +github.com/filecoin-project/specs-actors v0.6.1/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sgOVdJbvFjOnD5w94= github.com/filecoin-project/specs-storage v0.1.0/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= github.com/filecoin-project/storage-fsm v0.0.0-20200615162749-494c3bc48743 h1:a8f1p6UdeD+ZINBKJN4FhEos8uaKeASOAabq5RCpQdg= From fd6c8769dd266baf24f256577fbad016b769b805 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 10 Jun 2020 20:17:03 +0200 Subject: [PATCH 80/81] Update to new drand WIP Signed-off-by: Jakub Sztandera --- chain/beacon/drand/drand.go | 26 ++-- chain/beacon/drand/drand_test.go | 4 +- go.mod | 8 +- go.sum | 203 +++++++++++++++++++++++++++++-- 4 files changed, 212 insertions(+), 29 deletions(-) diff --git a/chain/beacon/drand/drand.go b/chain/beacon/drand/drand.go index 8742350ba..21bd2501a 100644 --- a/chain/beacon/drand/drand.go +++ b/chain/beacon/drand/drand.go @@ -8,8 +8,9 @@ import ( dchain "github.com/drand/drand/chain" dclient "github.com/drand/drand/client" - gclient "github.com/drand/drand/cmd/relay-gossip/client" + hclient "github.com/drand/drand/client/http" dlog "github.com/drand/drand/log" + gclient "github.com/drand/drand/lp2p/client" "github.com/drand/kyber" kzap "github.com/go-kit/kit/log/zap" "go.uber.org/zap/zapcore" @@ -79,11 +80,22 @@ func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub) (*DrandBeacon dlogger := dlog.NewKitLoggerFrom(kzap.NewZapSugarLogger( log.SugaredLogger.Desugar(), zapcore.InfoLevel)) + + var clients []dclient.Client + for _, url := range drandServers { + hc, err := hclient.NewWithInfo(url, drandChain, nil) + if err != nil { + return nil, xerrors.Errorf("could not create http drand client: %w", err) + } + clients = append(clients, hc) + + } + opts := []dclient.Option{ dclient.WithChainInfo(drandChain), - dclient.WithHTTPEndpoints(drandServers), dclient.WithCacheSize(1024), dclient.WithLogger(dlogger), + dclient.WithAutoWatch(), } if ps != nil { @@ -92,19 +104,11 @@ func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub) (*DrandBeacon log.Info("drand beacon without pubsub") } - client, err := dclient.New(opts...) + client, err := dclient.Wrap(clients, opts...) if err != nil { return nil, xerrors.Errorf("creating drand client") } - go func() { - // Explicitly Watch until that is fixed in drand - ch := client.Watch(context.Background()) - for range ch { - } - log.Error("dranch Watch bork") - }() - db := &DrandBeacon{ client: client, localCache: make(map[uint64]types.BeaconEntry), diff --git a/chain/beacon/drand/drand_test.go b/chain/beacon/drand/drand_test.go index 85b80cbe6..f35f3e4cc 100644 --- a/chain/beacon/drand/drand_test.go +++ b/chain/beacon/drand/drand_test.go @@ -5,12 +5,12 @@ import ( "testing" dchain "github.com/drand/drand/chain" - dclient "github.com/drand/drand/client" + hclient "github.com/drand/drand/client/http" "github.com/stretchr/testify/assert" ) func TestPrintGroupInfo(t *testing.T) { - c, err := dclient.NewHTTPClient(drandServers[0], nil, nil) + c, err := hclient.New(drandServers[0], nil, nil) assert.NoError(t, err) cg := c.(interface { FetchChainInfo(groupHash []byte) (*dchain.Info, error) diff --git a/go.mod b/go.mod index 89e2e2842..55d33c390 100644 --- a/go.mod +++ b/go.mod @@ -11,8 +11,8 @@ require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/coreos/go-systemd/v22 v22.0.0 github.com/docker/go-units v0.4.0 - github.com/drand/drand v0.9.2-0.20200529123141-6b138aefaef2 - github.com/drand/kyber v1.0.2 + github.com/drand/drand v0.9.2-0.20200616080806-a94e9c1636a4 + github.com/drand/kyber v1.1.0 github.com/fatih/color v1.8.0 github.com/filecoin-project/chain-validation v0.0.6-0.20200615191232-6be1a8c6ed09 github.com/filecoin-project/filecoin-ffi v0.26.1-0.20200508175440-05b30afeb00d @@ -36,7 +36,6 @@ require ( github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect - github.com/golang/protobuf v1.4.2 // indirect github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.4 github.com/gorilla/websocket v1.4.2 @@ -118,9 +117,10 @@ require ( go.uber.org/multierr v1.5.0 go.uber.org/zap v1.15.0 go4.org v0.0.0-20190313082347-94abd6928b1d // indirect - golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 + golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 + google.golang.org/api v0.25.0 // indirect launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect ) diff --git a/go.sum b/go.sum index 83af4904c..1e7e750aa 100644 --- a/go.sum +++ b/go.sum @@ -2,11 +2,32 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= contrib.go.opencensus.io/exporter/jaeger v0.1.0 h1:WNc9HbA38xEQmsI40Tjd/MNU/g8byN2Of7lwIjv0Jdc= contrib.go.opencensus.io/exporter/jaeger v0.1.0/go.mod h1:VYianECmuFPwU37O699Vc1GOcy+y8kOsfaxHRImmjbA= contrib.go.opencensus.io/exporter/prometheus v0.1.0 h1:SByaIoWwNgMdPSgl5sMqM2KDE5H/ukPWBRo314xiDvg= contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= @@ -16,6 +37,7 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkBy github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg= @@ -49,7 +71,7 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= -github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0 h1:5hryIiq9gtn+MiLVn0wP37kb/uTeRZgN08WoCsAhIhI= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -96,6 +118,9 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -119,6 +144,8 @@ github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= @@ -154,11 +181,15 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/drand/bls12-381 v0.3.2 h1:RImU8Wckmx8XQx1tp1q04OV73J9Tj6mmpQLYDP7V1XE= github.com/drand/bls12-381 v0.3.2/go.mod h1:dtcLgPtYT38L3NO6mPDYH0nbpc5tjPassDqiniuAt4Y= -github.com/drand/drand v0.9.2-0.20200529123141-6b138aefaef2 h1:IUxiqYU8BM8CyjHqxOTUbjrq7OvSDm8n1v9igaxtapM= -github.com/drand/drand v0.9.2-0.20200529123141-6b138aefaef2/go.mod h1:UIhZq1vHM1BQbpLa9K2HkxpCtbdsGkCc9MnCdKJxYjk= +github.com/drand/drand v0.9.2-0.20200616080806-a94e9c1636a4 h1:wEpu4hGFF0m0uDq/gxT9Ca/HWek0tvsMqsyPpLBWJ/E= +github.com/drand/drand v0.9.2-0.20200616080806-a94e9c1636a4/go.mod h1:Bu8QYdU0YdB2ZQZezHxabmOIciddiwLRnyV4nuZ2HQE= github.com/drand/kyber v1.0.1-0.20200110225416-8de27ed8c0e2/go.mod h1:UpXoA0Upd1N9l4TvRPHr1qAUBBERj6JQ/mnKI3BPEmw= github.com/drand/kyber v1.0.2 h1:dHjtWJZJdn3zBBZ9pqLsLfcR9ScvDvSqzS1sWA8seao= github.com/drand/kyber v1.0.2/go.mod h1:x6KOpK7avKj0GJ4emhXFP5n7M7W7ChAPmnQh/OL6vRw= +github.com/drand/kyber v1.1.0 h1:uBfD8gwpVufr+7Dvbxi4jGQ+qoMCO5tRfhYPyn+Tpqk= +github.com/drand/kyber v1.1.0/go.mod h1:x6KOpK7avKj0GJ4emhXFP5n7M7W7ChAPmnQh/OL6vRw= +github.com/drand/kyber-bls12381 v0.1.0 h1:/P4C65VnyEwxzR5ZYYVMNzY1If+aYBrdUU5ukwh7LQw= +github.com/drand/kyber-bls12381 v0.1.0/go.mod h1:N1emiHpm+jj7kMlxEbu3MUyOiooTgNySln564cgD9mk= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -243,6 +274,9 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= @@ -260,8 +294,11 @@ github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968 h1:s+PDl6lozQ+dEUtUtQn github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.1.0 h1:kFkMAZBNAn4j7K0GiZr8cRYzejq68VbheufiV3YuyFI= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/googleapis v1.4.0 h1:zgVt4UpGxcqVOw97aRGxT4svlcmdK35fynLNctY32zI= +github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -270,15 +307,23 @@ github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/status v1.0.3 h1:WkVBY59mw7qUNTr/bLwO7J2vesJ0rQ2C3tMXrTd3w5M= github.com/gogo/status v1.0.3/go.mod h1:SavQ51ycCLnc7dGyJxp8YAmudx8xqiVrRf+6IXRsugc= +github.com/gogo/status v1.1.0 h1:+eIkrewn5q6b30y+g/BJINVVdi2xH7je5MPJ3ZPK3JA= +github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -291,6 +336,7 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -310,12 +356,18 @@ github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbB github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f h1:KMlcu9X58lhTA/KrfX8Bi1LQSO4pzoVjTiL3h4Jk+Zk= @@ -336,12 +388,15 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.0 h1:0IKlLyQ3Hs9nDaiK5cSHAGmcQEIC8l2Ts1u6x5Dfrqg= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.0/go.mod h1:mJzapYve32yjrKlk9GbyCZHuPgZsrbyIbyKhSzOpg6s= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.14.3 h1:OCJlWkOUoTnl0neNGlf4fUm3TmbEtguw7vR+nGtnDjY= -github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= +github.com/grpc-ecosystem/grpc-gateway v1.14.6 h1:8ERzHx8aj1Sc47mu9n/AksaKCSWrMchFtkdrS4BIj5o= +github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4GkDEdKCRJduHpTxT3/jcw= +github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= github.com/gxed/go-shellwords v1.0.3/go.mod h1:N7paucT91ByIjmVJHhvoarjoQnmsi3Jd3vH7VqgtMxQ= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= @@ -384,6 +439,7 @@ github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOo github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d h1:/WZQPMZNsjZ7IlCpsLGdQBINg5bxKQ1K1sh6awxLtkA= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= @@ -523,7 +579,6 @@ github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBW github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= github.com/ipfs/go-log/v2 v2.0.8/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= -github.com/ipfs/go-log/v2 v2.1.0/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/ipfs/go-log/v2 v2.1.2-0.20200609205458-f8d20c392cb7 h1:LtL/rvdfbKSthZGmAAD9o4KKg6HA6Qn8gXCCdgnj7lw= github.com/ipfs/go-log/v2 v2.1.2-0.20200609205458-f8d20c392cb7/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/ipfs/go-merkledag v0.0.3/go.mod h1:Oc5kIXLHokkE1hWGMBHw+oxehkAaTOqtEb7Zbh6BhLA= @@ -599,6 +654,7 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -608,12 +664,16 @@ github.com/kabukky/httpscerts v0.0.0-20150320125433-617593d7dcb3/go.mod h1:BYpt4 github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kilic/bls12-381 v0.0.0-20200607163746-32e1441c8a9f h1:qET3Wx0v8tMtoTOQnsJXVvqvCopSf48qobR6tcJuDHo= +github.com/kilic/bls12-381 v0.0.0-20200607163746-32e1441c8a9f/go.mod h1:XXfR6YFCRSrkEXbNlIyDsgXVNJWVUV30m/ebkVy9n6s= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d h1:68u9r4wEvL3gYg2jvAOgROwZ3H+Y3hIDk4tbbmIjcYQ= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= @@ -777,8 +837,8 @@ github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYc github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= -github.com/libp2p/go-libp2p-pubsub v0.3.0/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato= github.com/libp2p/go-libp2p-pubsub v0.3.1/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato= +github.com/libp2p/go-libp2p-pubsub v0.3.2-0.20200527132641-c0712c6e92cf/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato= github.com/libp2p/go-libp2p-pubsub v0.3.2 h1:k3cJm5JW5mjaWZkobS50sJLJWaB2mBi0HW4eRlE8mSo= github.com/libp2p/go-libp2p-pubsub v0.3.2/go.mod h1:Uss7/Cfz872KggNb+doCVPHeCDmXB7z500m/R8DaAUk= github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU= @@ -1060,9 +1120,13 @@ github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoT github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02 h1:0R5mDLI66Qw13qN80TRz85zthQ2nf2+uDyiV23w6c3Q= github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02/go.mod h1:JNdpVEzCpXBgIiv4ds+TzhN1hrtxq6ClLrTlT9OQRSc= +github.com/opentracing-contrib/go-grpc v0.0.0-20191001143057-db30781987df h1:vdYtBU6zvL7v+Tr+0xFM/qhahw/EvY8DMMunZHKH6eE= +github.com/opentracing-contrib/go-grpc v0.0.0-20191001143057-db30781987df/go.mod h1:DYR5Eij8rJl8h7gblRrOZ8g0kW1umSpKqYIBTgeDtLo= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9 h1:QsgXACQhd9QJhEmRumbsMQQvBtmdS0mafoVEBplWXEg= github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w= +github.com/opentracing-contrib/go-stdlib v1.0.0 h1:TBS7YuVotp8myLon4Pv7BtCBzOTo1DeZCld0Z63mW2w= +github.com/opentracing-contrib/go-stdlib v1.0.0/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= @@ -1100,8 +1164,8 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= -github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A= +github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -1117,6 +1181,8 @@ github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/node_exporter v1.0.0-rc.0.0.20200428091818-01054558c289/go.mod h1:FGbBv5OPKjch+jNUJmEQpMZytIdyW0NdBtWFcfSKusc= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1128,6 +1194,8 @@ github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.1.0 h1:jhMy6QXfi3y2HEzFoyuCj40z4OZIIHHPtFyCMftmvKA= +github.com/prometheus/procfs v0.1.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -1175,6 +1243,8 @@ github.com/siebenmann/go-kstat v0.0.0-20160321171754-d34789b79745/go.mod h1:G81a github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= @@ -1306,6 +1376,7 @@ go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= @@ -1354,6 +1425,7 @@ golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -1368,19 +1440,38 @@ golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200427165652-729f1e841bcc/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM= +golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1402,6 +1493,8 @@ golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1410,17 +1503,27 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200519113804-d87ec0cfa476 h1:E7ct1C6/33eOdrGZKMoyntcEvs2dwZnDe30crG5vpYU= golang.org/x/net v0.0.0-20200519113804-d87ec0cfa476/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1429,6 +1532,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1453,10 +1558,13 @@ golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190524122548-abf6ff778158/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190524152521-dbbf3f1254d4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1466,25 +1574,34 @@ golang.org/x/sys v0.0.0-20190902133755-9109b7679e13/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191025090151-53bf42e6b339/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200427175716-29b57079015a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1492,6 +1609,7 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1503,26 +1621,44 @@ golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191030062658-86caa796c7ab/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200108195415-316d2f248479/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200216192241-b320d3a0f5a2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200318150045-ba25ddc85566 h1:OXjomkWHhzUx4+HldlJ2TsMxJdWgEo5CTtspD1wdhdk= golang.org/x/tools v0.0.0-20200318150045-ba25ddc85566/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4 h1:kDtqNkeBrZb8B+atrj50B5XLHpzXXqcCdZPP/ApQ5NY= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -1533,21 +1669,54 @@ google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx1 google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.3.2 h1:iTp+3yyl/KOtxa/d1/JUE0GGSoR6FuW5udver22iwpw= google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.25.0 h1:LodzhlzZEUfhXzNUMIfVlf9Gr6Ua5MMtoFWh7+f47qA= +google.golang.org/api v0.25.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c h1:hrpEMCZ2O7DR5gC1n2AJGVhrwiEjOi35+jxtIuZpTMo= -google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200608115520-7c474a2e3482 h1:i+Aiej6cta/Frzp13/swvwz5O00kYcSe0A/C5Wd7zX8= +google.golang.org/genproto v0.0.0-20200608115520-7c474a2e3482/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1555,21 +1724,29 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1603,6 +1780,7 @@ grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJd honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= @@ -1611,6 +1789,7 @@ howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqp howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= From 9690bc9c638909b7136887c6f6f5db8bec76a8a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 17 Jun 2020 00:35:51 +0200 Subject: [PATCH 81/81] seed: Cleanup unsealed sectors --- cmd/lotus-seed/seed/seed.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index ffb651c00..be366d4db 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -92,6 +92,10 @@ func PreSeal(maddr address.Address, spt abi.RegisteredSealProof, offset abi.Sect return nil, nil, xerrors.Errorf("trim cache: %w", err) } + if err := cleanupUnsealed(sbfs, sid); err != nil { + return nil, nil, xerrors.Errorf("remove unsealed file: %w", err) + } + log.Warn("PreCommitOutput: ", sid, cids.Sealed, cids.Unsealed) sealedSectors = append(sealedSectors, &genesis.PreSeal{ CommR: cids.Sealed, @@ -161,6 +165,16 @@ func PreSeal(maddr address.Address, spt abi.RegisteredSealProof, offset abi.Sect return miner, &minerAddr.KeyInfo, nil } +func cleanupUnsealed(sbfs *basicfs.Provider, sid abi.SectorID) error { + paths, done, err := sbfs.AcquireSector(context.TODO(), sid, stores.FTUnsealed, stores.FTNone, stores.PathSealing) + if err != nil { + return err + } + defer done() + + return os.Remove(paths.Unsealed) +} + func WriteGenesisMiner(maddr address.Address, sbroot string, gm *genesis.Miner, key *types.KeyInfo) error { output := map[string]genesis.Miner{ maddr.String(): *gm,