diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 7876715e2..244be5078 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -43,13 +43,14 @@ body: id: version attributes: label: Lotus Version + render: text description: Enter the output of `lotus version` and `lotus-miner version` if applicable. placeholder: | e.g. Daemon:1.11.0-rc2+debug+git.0519cd371.dirty+api1.3.0 Local: lotus version 1.11.0-rc2+debug+git.0519cd371.dirty validations: - reuiqred: true + required: true - type: textarea id: Description attributes: @@ -62,19 +63,18 @@ body: * For sealing issues, include the output of `lotus-miner sectors status --log ` for the failed sector(s). * For proving issues, include the output of `lotus-miner proving` info. * For deal making issues, include the output of `lotus client list-deals -v` and/or `lotus-miner storage-deals|retrieval-deals|data-transfers list [-v]` commands for the deal(s) in question. - render: bash validations: required: true - type: textarea id: extraInfo attributes: label: Logging Information + render: text description: | Please provide debug logs of the problem, remember you can get set log level control for: * lotus: use `lotus log list` to get all log systems available and set level by `lotus log set-level`. An example can be found [here](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/#log-level-control). * lotus-miner:`lotus-miner log list` to get all log systems available and set level by `lotus-miner log set-level If you don't provide detailed logs when you raise the issue it will almost certainly be the first request I make before furthur diagnosing the problem. - render: bash validations: required: true - type: textarea @@ -87,6 +87,5 @@ body: 2. Do '...' 3. See error '...' ... - render: bash validations: - required: false \ No newline at end of file + required: false diff --git a/.github/ISSUE_TEMPLATE/m1_bug_report_deal.yml b/.github/ISSUE_TEMPLATE/m1_bug_report_deal.yml index 4402e97da..3a24d9564 100644 --- a/.github/ISSUE_TEMPLATE/m1_bug_report_deal.yml +++ b/.github/ISSUE_TEMPLATE/m1_bug_report_deal.yml @@ -35,10 +35,11 @@ body: - type: textarea id: version attributes: + render: text label: Lotus Tag and Version description: Enter the lotus tag, output of `lotus version` and `lotus-miner version`. validations: - reuiqred: true + required: true - type: textarea id: Description attributes: @@ -48,7 +49,6 @@ body: * What you were doding when you experienced the bug? * Any *error* messages you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas). * What is the expected behaviour? - render: bash validations: required: true - type: textarea @@ -72,6 +72,7 @@ body: - type: textarea id: logging attributes: + render: text label: Logging Information description: Please link to the whole of the miner logs on your side of the transaction. You can upload the logs to a [gist](https://gist.github.com). validations: @@ -86,6 +87,5 @@ body: 2. Do '...' 3. See error '...' ... - render: bash validations: - required: false \ No newline at end of file + required: false diff --git a/.github/ISSUE_TEMPLATE/m1_bug_report_non_deal.yml b/.github/ISSUE_TEMPLATE/m1_bug_report_non_deal.yml index ede3593e5..363c3a1ab 100644 --- a/.github/ISSUE_TEMPLATE/m1_bug_report_non_deal.yml +++ b/.github/ISSUE_TEMPLATE/m1_bug_report_non_deal.yml @@ -36,10 +36,11 @@ body: - type: textarea id: version attributes: + render: text label: Lotus Tag and Version description: Enter the lotus tag, output of `lotus version` and `lotus-miner version`. validations: - reuiqred: true + required: true - type: textarea id: Description attributes: @@ -51,19 +52,18 @@ body: * What is the expected behaviour? * For sealing issues, include the output of `lotus-miner sectors status --log ` for the failed sector(s). * For proving issues, include the output of `lotus-miner proving` info. - render: bash validations: required: true - type: textarea id: extraInfo attributes: label: Logging Information + render: text description: | Please provide debug logs of the problem, remember you can get set log level control for: * lotus: use `lotus log list` to get all log systems available and set level by `lotus log set-level`. An example can be found [here](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/#log-level-control). * lotus-miner:`lotus-miner log list` to get all log systems available and set level by `lotus-miner log set-level If you don't provide detailed logs when you raise the issue it will almost certainly be the first request I make before furthur diagnosing the problem. - render: bash validations: required: true - type: textarea @@ -76,6 +76,5 @@ body: 2. Do '...' 3. See error '...' ... - render: bash validations: - required: false \ No newline at end of file + required: false diff --git a/Makefile b/Makefile index 77fd38b9e..dfef9f262 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ all: build unexport GOFLAGS -GOVERSION:=$(shell go version | cut -d' ' -f 3 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}') +GOVERSION:=$(shell $(GOCC) version | tr ' ' '\n' | grep go1 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}') ifeq ($(shell expr $(GOVERSION) \< 1016000), 1) $(warning Your Golang version is go$(shell expr $(GOVERSION) / 1000000).$(shell expr $(GOVERSION) % 1000000 / 1000).$(shell expr $(GOVERSION) % 1000)) $(error Update Golang to version to at least 1.16.0) @@ -17,6 +17,8 @@ MODULES:= CLEAN:= BINS:= +GOCC?=go + ldflags=-X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.$(subst -,.,$(shell git describe --always --match=NeVeRmAtCh --dirty 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)) ifneq ($(strip $(LDFLAGS)),) ldflags+=-extldflags=$(LDFLAGS) @@ -85,32 +87,32 @@ interopnet: build-devnets lotus: $(BUILD_DEPS) rm -f lotus - go build $(GOFLAGS) -o lotus ./cmd/lotus + $(GOCC) build $(GOFLAGS) -o lotus ./cmd/lotus .PHONY: lotus BINS+=lotus lotus-miner: $(BUILD_DEPS) rm -f lotus-miner - go build $(GOFLAGS) -o lotus-miner ./cmd/lotus-miner + $(GOCC) build $(GOFLAGS) -o lotus-miner ./cmd/lotus-miner .PHONY: lotus-miner BINS+=lotus-miner lotus-worker: $(BUILD_DEPS) rm -f lotus-worker - go build $(GOFLAGS) -o lotus-worker ./cmd/lotus-seal-worker + $(GOCC) build $(GOFLAGS) -o lotus-worker ./cmd/lotus-seal-worker .PHONY: lotus-worker BINS+=lotus-worker lotus-shed: $(BUILD_DEPS) rm -f lotus-shed - go build $(GOFLAGS) -o lotus-shed ./cmd/lotus-shed + $(GOCC) build $(GOFLAGS) -o lotus-shed ./cmd/lotus-shed .PHONY: lotus-shed BINS+=lotus-shed lotus-gateway: $(BUILD_DEPS) rm -f lotus-gateway - go build $(GOFLAGS) -o lotus-gateway ./cmd/lotus-gateway + $(GOCC) build $(GOFLAGS) -o lotus-gateway ./cmd/lotus-gateway .PHONY: lotus-gateway BINS+=lotus-gateway @@ -138,19 +140,19 @@ install-app: lotus-seed: $(BUILD_DEPS) rm -f lotus-seed - go build $(GOFLAGS) -o lotus-seed ./cmd/lotus-seed + $(GOCC) build $(GOFLAGS) -o lotus-seed ./cmd/lotus-seed .PHONY: lotus-seed BINS+=lotus-seed benchmarks: - go run github.com/whyrusleeping/bencher ./... > bench.json + $(GOCC) run github.com/whyrusleeping/bencher ./... > bench.json @echo Submitting results @curl -X POST 'http://benchmark.kittyhawk.wtf/benchmark' -d '@bench.json' -u "${benchmark_http_cred}" .PHONY: benchmarks lotus-pond: 2k - go build -o lotus-pond ./lotuspond + $(GOCC) build -o lotus-pond ./lotuspond .PHONY: lotus-pond BINS+=lotus-pond @@ -161,85 +163,63 @@ lotus-pond-front: lotus-pond-app: lotus-pond-front lotus-pond .PHONY: lotus-pond-app -lotus-townhall: - rm -f lotus-townhall - go build -o lotus-townhall ./cmd/lotus-townhall -.PHONY: lotus-townhall -BINS+=lotus-townhall - -lotus-townhall-front: - (cd ./cmd/lotus-townhall/townhall && npm i && npm run build) -.PHONY: lotus-townhall-front - -lotus-townhall-app: lotus-touch lotus-townhall-front -.PHONY: lotus-townhall-app - lotus-fountain: rm -f lotus-fountain - go build -o lotus-fountain ./cmd/lotus-fountain + $(GOCC) build -o lotus-fountain ./cmd/lotus-fountain .PHONY: lotus-fountain BINS+=lotus-fountain -lotus-chainwatch: - rm -f lotus-chainwatch - go build $(GOFLAGS) -o lotus-chainwatch ./cmd/lotus-chainwatch -.PHONY: lotus-chainwatch -BINS+=lotus-chainwatch - lotus-bench: rm -f lotus-bench - go build -o lotus-bench ./cmd/lotus-bench + $(GOCC) build -o lotus-bench ./cmd/lotus-bench .PHONY: lotus-bench BINS+=lotus-bench lotus-stats: rm -f lotus-stats - go build $(GOFLAGS) -o lotus-stats ./cmd/lotus-stats + $(GOCC) build $(GOFLAGS) -o lotus-stats ./cmd/lotus-stats .PHONY: lotus-stats BINS+=lotus-stats lotus-pcr: rm -f lotus-pcr - go build $(GOFLAGS) -o lotus-pcr ./cmd/lotus-pcr + $(GOCC) build $(GOFLAGS) -o lotus-pcr ./cmd/lotus-pcr .PHONY: lotus-pcr BINS+=lotus-pcr lotus-health: rm -f lotus-health - go build -o lotus-health ./cmd/lotus-health + $(GOCC) build -o lotus-health ./cmd/lotus-health .PHONY: lotus-health BINS+=lotus-health lotus-wallet: rm -f lotus-wallet - go build -o lotus-wallet ./cmd/lotus-wallet + $(GOCC) build -o lotus-wallet ./cmd/lotus-wallet .PHONY: lotus-wallet BINS+=lotus-wallet lotus-keygen: rm -f lotus-keygen - go build -o lotus-keygen ./cmd/lotus-keygen + $(GOCC) build -o lotus-keygen ./cmd/lotus-keygen .PHONY: lotus-keygen BINS+=lotus-keygen testground: - go build -tags testground -o /dev/null ./cmd/lotus + $(GOCC) build -tags testground -o /dev/null ./cmd/lotus .PHONY: testground BINS+=testground tvx: rm -f tvx - go build -o tvx ./cmd/tvx + $(GOCC) build -o tvx ./cmd/tvx .PHONY: tvx BINS+=tvx -install-chainwatch: lotus-chainwatch - install -C ./lotus-chainwatch /usr/local/bin/lotus-chainwatch - lotus-sim: $(BUILD_DEPS) rm -f lotus-sim - go build $(GOFLAGS) -o lotus-sim ./cmd/lotus-sim + $(GOCC) build $(GOFLAGS) -o lotus-sim ./cmd/lotus-sim .PHONY: lotus-sim BINS+=lotus-sim @@ -261,21 +241,13 @@ install-miner-service: install-miner install-daemon-service @echo @echo "lotus-miner service installed. Don't forget to run 'sudo systemctl start lotus-miner' to start it and 'sudo systemctl enable lotus-miner' for it to be enabled on startup." -install-chainwatch-service: install-chainwatch install-daemon-service - mkdir -p /etc/systemd/system - mkdir -p /var/log/lotus - install -C -m 0644 ./scripts/lotus-chainwatch.service /etc/systemd/system/lotus-chainwatch.service - systemctl daemon-reload - @echo - @echo "chainwatch service installed. Don't forget to run 'sudo systemctl start lotus-chainwatch' to start it and 'sudo systemctl enable lotus-chainwatch' for it to be enabled on startup." - install-main-services: install-miner-service -install-all-services: install-main-services install-chainwatch-service +install-all-services: install-main-services install-services: install-main-services -clean-daemon-service: clean-miner-service clean-chainwatch-service +clean-daemon-service: clean-miner-service -systemctl stop lotus-daemon -systemctl disable lotus-daemon rm -f /etc/systemd/system/lotus-daemon.service @@ -287,12 +259,6 @@ clean-miner-service: rm -f /etc/systemd/system/lotus-miner.service systemctl daemon-reload -clean-chainwatch-service: - -systemctl stop lotus-chainwatch - -systemctl disable lotus-chainwatch - rm -f /etc/systemd/system/lotus-chainwatch.service - systemctl daemon-reload - clean-main-services: clean-daemon-service clean-all-services: clean-main-services @@ -319,25 +285,25 @@ dist-clean: .PHONY: dist-clean type-gen: api-gen - go run ./gen/main.go - go generate -x ./... + $(GOCC) run ./gen/main.go + $(GOCC) generate -x ./... goimports -w api/ method-gen: api-gen - (cd ./lotuspond/front/src/chain && go run ./methodgen.go) + (cd ./lotuspond/front/src/chain && $(GOCC) run ./methodgen.go) actors-gen: - go run ./chain/actors/agen - go fmt ./... + $(GOCC) run ./chain/actors/agen + $(GOCC) fmt ./... api-gen: - go run ./gen/api + $(GOCC) run ./gen/api goimports -w api goimports -w api .PHONY: api-gen cfgdoc-gen: - go run ./node/config/cfgdocgen > ./node/config/doc_gen.go + $(GOCC) run ./node/config/cfgdocgen > ./node/config/doc_gen.go appimage: lotus rm -rf appimage-builder-cache || true @@ -351,9 +317,9 @@ appimage: lotus docsgen: docsgen-md docsgen-openrpc docsgen-md-bin: api-gen actors-gen - go build $(GOFLAGS) -o docgen-md ./api/docgen/cmd + $(GOCC) build $(GOFLAGS) -o docgen-md ./api/docgen/cmd docsgen-openrpc-bin: api-gen actors-gen - go build $(GOFLAGS) -o docgen-openrpc ./api/docgen-openrpc/cmd + $(GOCC) build $(GOFLAGS) -o docgen-openrpc ./api/docgen-openrpc/cmd docsgen-md: docsgen-md-full docsgen-md-storage docsgen-md-worker @@ -393,4 +359,4 @@ print-%: @echo $*=$($*) circleci: - go generate -x ./.circleci \ No newline at end of file + $(GOCC) generate -x ./.circleci \ No newline at end of file diff --git a/api/api_storage.go b/api/api_storage.go index 154abcea7..c39114929 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -166,6 +166,10 @@ type StorageMiner interface { MarketPendingDeals(ctx context.Context) (PendingDealInfo, error) //perm:write MarketPublishPendingDeals(ctx context.Context) error //perm:admin + // RuntimeSubsystems returns the subsystems that are enabled + // in this instance. + RuntimeSubsystems(ctx context.Context) (MinerSubsystems, error) //perm:read + DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error //perm:admin DealsList(ctx context.Context) ([]MarketDeal, error) //perm:admin DealsConsiderOnlineStorageDeals(context.Context) (bool, error) //perm:admin diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index 39980023f..f9addc940 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -16,10 +16,10 @@ import ( "github.com/google/uuid" "github.com/ipfs/go-cid" "github.com/ipfs/go-filestore" - metrics "github.com/libp2p/go-libp2p-core/metrics" + "github.com/libp2p/go-libp2p-core/metrics" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" - protocol "github.com/libp2p/go-libp2p-core/protocol" + "github.com/libp2p/go-libp2p-core/protocol" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/multiformats/go-multiaddr" @@ -46,11 +46,12 @@ import ( ) var ExampleValues = map[reflect.Type]interface{}{ - reflect.TypeOf(auth.Permission("")): auth.Permission("write"), - reflect.TypeOf(""): "string value", - reflect.TypeOf(uint64(42)): uint64(42), - reflect.TypeOf(byte(7)): byte(7), - reflect.TypeOf([]byte{}): []byte("byte array"), + reflect.TypeOf(api.MinerSubsystem(0)): api.MinerSubsystem(1), + reflect.TypeOf(auth.Permission("")): auth.Permission("write"), + reflect.TypeOf(""): "string value", + reflect.TypeOf(uint64(42)): uint64(42), + reflect.TypeOf(byte(7)): byte(7), + reflect.TypeOf([]byte{}): []byte("byte array"), } func addExample(v interface{}) { @@ -264,6 +265,12 @@ func init() { addExample(api.CheckStatusCode(0)) addExample(map[string]interface{}{"abc": 123}) + addExample(api.MinerSubsystems{ + api.SubsystemMining, + api.SubsystemSealing, + api.SubsystemSectorStorage, + api.SubsystemMarkets, + }) } func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []reflect.Type) { diff --git a/api/miner_subsystems.go b/api/miner_subsystems.go new file mode 100644 index 000000000..a77de7e3c --- /dev/null +++ b/api/miner_subsystems.go @@ -0,0 +1,79 @@ +package api + +import ( + "encoding/json" +) + +// MinerSubsystem represents a miner subsystem. Int and string values are not +// guaranteed to be stable over time is not +// guaranteed to be stable over time. +type MinerSubsystem int + +const ( + // SubsystemUnknown is a placeholder for the zero value. It should never + // be used. + SubsystemUnknown MinerSubsystem = iota + // SubsystemMarkets signifies the storage and retrieval + // deal-making subsystem. + SubsystemMarkets + // SubsystemMining signifies the mining subsystem. + SubsystemMining + // SubsystemSealing signifies the sealing subsystem. + SubsystemSealing + // SubsystemSectorStorage signifies the sector storage subsystem. + SubsystemSectorStorage +) + +var MinerSubsystemToString = map[MinerSubsystem]string{ + SubsystemUnknown: "Unknown", + SubsystemMarkets: "Markets", + SubsystemMining: "Mining", + SubsystemSealing: "Sealing", + SubsystemSectorStorage: "SectorStorage", +} + +var MinerSubsystemToID = map[string]MinerSubsystem{ + "Unknown": SubsystemUnknown, + "Markets": SubsystemMarkets, + "Mining": SubsystemMining, + "Sealing": SubsystemSealing, + "SectorStorage": SubsystemSectorStorage, +} + +func (ms MinerSubsystem) MarshalJSON() ([]byte, error) { + return json.Marshal(MinerSubsystemToString[ms]) +} + +func (ms *MinerSubsystem) UnmarshalJSON(b []byte) error { + var j string + err := json.Unmarshal(b, &j) + if err != nil { + return err + } + s, ok := MinerSubsystemToID[j] + if !ok { + *ms = SubsystemUnknown + } else { + *ms = s + } + return nil +} + +type MinerSubsystems []MinerSubsystem + +func (ms MinerSubsystems) Has(entry MinerSubsystem) bool { + for _, v := range ms { + if v == entry { + return true + } + } + return false +} + +func (ms MinerSubsystem) String() string { + s, ok := MinerSubsystemToString[ms] + if !ok { + return MinerSubsystemToString[SubsystemUnknown] + } + return s +} diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 7d96425ff..a4feb7be1 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -699,6 +699,8 @@ type StorageMinerStruct struct { ReturnUnsealPiece func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"` + RuntimeSubsystems func(p0 context.Context) (MinerSubsystems, error) `perm:"read"` + SealingAbort func(p0 context.Context, p1 storiface.CallID) error `perm:"admin"` SealingSchedDiag func(p0 context.Context, p1 bool) (interface{}, error) `perm:"admin"` @@ -4095,6 +4097,17 @@ func (s *StorageMinerStub) ReturnUnsealPiece(p0 context.Context, p1 storiface.Ca return ErrNotSupported } +func (s *StorageMinerStruct) RuntimeSubsystems(p0 context.Context) (MinerSubsystems, error) { + if s.Internal.RuntimeSubsystems == nil { + return *new(MinerSubsystems), ErrNotSupported + } + return s.Internal.RuntimeSubsystems(p0) +} + +func (s *StorageMinerStub) RuntimeSubsystems(p0 context.Context) (MinerSubsystems, error) { + return *new(MinerSubsystems), ErrNotSupported +} + func (s *StorageMinerStruct) SealingAbort(p0 context.Context, p1 storiface.CallID) error { if s.Internal.SealingAbort == nil { return ErrNotSupported diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index c940d921b..db41fb2d3 100644 Binary files a/build/openrpc/full.json.gz and b/build/openrpc/full.json.gz differ diff --git a/build/openrpc/miner.json.gz b/build/openrpc/miner.json.gz index 41e51a13e..57a56d94f 100644 Binary files a/build/openrpc/miner.json.gz and b/build/openrpc/miner.json.gz differ diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index 075b54c32..07740e5e2 100644 Binary files a/build/openrpc/worker.json.gz and b/build/openrpc/worker.json.gz differ diff --git a/build/version.go b/build/version.go index c6a1be3e2..4e155c98a 100644 --- a/build/version.go +++ b/build/version.go @@ -34,7 +34,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "1.11.1-dev" +const BuildVersion = "1.11.2-dev" func UserVersion() string { if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" { diff --git a/chain/metrics/consensus.go b/chain/metrics/consensus.go deleted file mode 100644 index c3c4a10d1..000000000 --- a/chain/metrics/consensus.go +++ /dev/null @@ -1,129 +0,0 @@ -package metrics - -import ( - "context" - "encoding/json" - - "github.com/filecoin-project/go-state-types/abi" - "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log/v2" - pubsub "github.com/libp2p/go-libp2p-pubsub" - "go.uber.org/fx" - - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/node/impl/full" - "github.com/filecoin-project/lotus/node/modules/helpers" -) - -var log = logging.Logger("metrics") - -const baseTopic = "/fil/headnotifs/" - -type Update struct { - Type string -} - -func SendHeadNotifs(nickname string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, ps *pubsub.PubSub, chain full.ChainAPI) error { - return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, ps *pubsub.PubSub, chain full.ChainAPI) error { - ctx := helpers.LifecycleCtx(mctx, lc) - - lc.Append(fx.Hook{ - OnStart: func(_ context.Context) error { - gen, err := chain.Chain.GetGenesis() - if err != nil { - return err - } - - topic := baseTopic + gen.Cid().String() - - go func() { - if err := sendHeadNotifs(ctx, ps, topic, chain, nickname); err != nil { - log.Error("consensus metrics error", err) - return - } - }() - go func() { - sub, err := ps.Subscribe(topic) //nolint - if err != nil { - return - } - defer sub.Cancel() - - for { - if _, err := sub.Next(ctx); err != nil { - return - } - } - - }() - return nil - }, - }) - - return nil - } -} - -type message struct { - // TipSet - Cids []cid.Cid - Blocks []*types.BlockHeader - Height abi.ChainEpoch - Weight types.BigInt - Time uint64 - Nonce uint64 - - // Meta - - NodeName string -} - -func sendHeadNotifs(ctx context.Context, ps *pubsub.PubSub, topic string, chain full.ChainAPI, nickname string) error { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - notifs, err := chain.ChainNotify(ctx) - if err != nil { - return err - } - - // using unix nano time makes very sure we pick a nonce higher than previous restart - nonce := uint64(build.Clock.Now().UnixNano()) - - for { - select { - case notif := <-notifs: - n := notif[len(notif)-1] - - w, err := chain.ChainTipSetWeight(ctx, n.Val.Key()) - if err != nil { - return err - } - - m := message{ - Cids: n.Val.Cids(), - Blocks: n.Val.Blocks(), - Height: n.Val.Height(), - Weight: w, - NodeName: nickname, - Time: uint64(build.Clock.Now().UnixNano() / 1000_000), - Nonce: nonce, - } - - b, err := json.Marshal(m) - if err != nil { - return err - } - - //nolint - if err := ps.Publish(topic, b); err != nil { - return err - } - case <-ctx.Done(): - return nil - } - - nonce++ - } -} diff --git a/chain/state/statetree.go b/chain/state/statetree.go index 8705aeff8..8140cd4db 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -273,7 +273,7 @@ func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) { } if err != nil { log.Errorf("failed to load state tree: %s", err) - return nil, xerrors.Errorf("failed to load state tree: %w", err) + return nil, xerrors.Errorf("failed to load state tree %s: %w", c, err) } s := &StateTree{ diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index b76dd0faf..1748c341e 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -221,7 +221,7 @@ func (sm *StateManager) ResolveToKeyAddress(ctx context.Context, addr address.Ad // First try to resolve the actor in the parent state, so we don't have to compute anything. tree, err := state.LoadStateTree(cst, ts.ParentState()) if err != nil { - return address.Undef, xerrors.Errorf("failed to load parent state tree: %w", err) + return address.Undef, xerrors.Errorf("failed to load parent state tree at tipset %s: %w", ts.Parents(), err) } resolved, err := vm.ResolveToKeyAddr(tree, cst, addr) @@ -232,12 +232,12 @@ func (sm *StateManager) ResolveToKeyAddress(ctx context.Context, addr address.Ad // If that fails, compute the tip-set and try again. st, _, err := sm.TipSetState(ctx, ts) if err != nil { - return address.Undef, xerrors.Errorf("resolve address failed to get tipset state: %w", err) + return address.Undef, xerrors.Errorf("resolve address failed to get tipset %s state: %w", ts, err) } tree, err = state.LoadStateTree(cst, st) if err != nil { - return address.Undef, xerrors.Errorf("failed to load state tree") + return address.Undef, xerrors.Errorf("failed to load state tree at tipset %s: %w", ts, err) } return vm.ResolveToKeyAddr(tree, cst, addr) diff --git a/chain/store/messages.go b/chain/store/messages.go index 50cf0e6a2..9f5160559 100644 --- a/chain/store/messages.go +++ b/chain/store/messages.go @@ -110,7 +110,7 @@ func (cs *ChainStore) BlockMsgsForTipset(ts *types.TipSet) ([]BlockMessages, err cst := cbor.NewCborStore(cs.stateBlockstore) st, err := state.LoadStateTree(cst, ts.Blocks()[0].ParentStateRoot) if err != nil { - return nil, xerrors.Errorf("failed to load state tree") + return nil, xerrors.Errorf("failed to load state tree at tipset %s: %w", ts, err) } selectMsg := func(m *types.Message) (bool, error) { diff --git a/chain/vm/invoker_test.go b/chain/vm/invoker_test.go index 6822e2371..8499f001a 100644 --- a/chain/vm/invoker_test.go +++ b/chain/vm/invoker_test.go @@ -6,6 +6,8 @@ import ( "io" "testing" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/network" cbor "github.com/ipfs/go-ipld-cbor" @@ -80,6 +82,30 @@ func (basicContract) InvokeSomething10(rt runtime2.Runtime, params *basicParams) return nil } +type basicRtMessage struct{} + +var _ runtime2.Message = (*basicRtMessage)(nil) + +func (*basicRtMessage) Caller() address.Address { + a, err := address.NewIDAddress(0) + if err != nil { + panic(err) + } + return a +} + +func (*basicRtMessage) Receiver() address.Address { + a, err := address.NewIDAddress(1) + if err != nil { + panic(err) + } + return a +} + +func (*basicRtMessage) ValueReceived() abi.TokenAmount { + return big.NewInt(0) +} + func TestInvokerBasic(t *testing.T) { inv := ActorRegistry{} code, err := inv.transform(basicContract{}) @@ -89,7 +115,7 @@ func TestInvokerBasic(t *testing.T) { bParam, err := actors.SerializeParams(&basicParams{B: 1}) assert.NoError(t, err) - _, aerr := code[0](&Runtime{}, bParam) + _, aerr := code[0](&Runtime{Message: &basicRtMessage{}}, bParam) assert.Equal(t, exitcode.ExitCode(1), aerrors.RetCode(aerr), "return code should be 1") if aerrors.IsFatal(aerr) { @@ -101,7 +127,7 @@ func TestInvokerBasic(t *testing.T) { bParam, err := actors.SerializeParams(&basicParams{B: 2}) assert.NoError(t, err) - _, aerr := code[10](&Runtime{}, bParam) + _, aerr := code[10](&Runtime{Message: &basicRtMessage{}}, bParam) assert.Equal(t, exitcode.ExitCode(12), aerrors.RetCode(aerr), "return code should be 12") if aerrors.IsFatal(aerr) { t.Fatal("err should not be fatal") @@ -113,6 +139,7 @@ func TestInvokerBasic(t *testing.T) { vm: &VM{ntwkVersion: func(ctx context.Context, epoch abi.ChainEpoch) network.Version { return network.Version0 }}, + Message: &basicRtMessage{}, }, []byte{99}) if aerrors.IsFatal(aerr) { t.Fatal("err should not be fatal") @@ -125,6 +152,7 @@ func TestInvokerBasic(t *testing.T) { vm: &VM{ntwkVersion: func(ctx context.Context, epoch abi.ChainEpoch) network.Version { return network.Version7 }}, + Message: &basicRtMessage{}, }, []byte{99}) if aerrors.IsFatal(aerr) { t.Fatal("err should not be fatal") diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 2845c7696..7117f3443 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -146,7 +146,7 @@ func (rt *Runtime) shimCall(f func() interface{}) (rval []byte, aerr aerrors.Act defer func() { if r := recover(); r != nil { if ar, ok := r.(aerrors.ActorError); ok { - log.Warnf("VM.Call failure: %+v", ar) + log.Warnf("VM.Call failure in call from: %s to %s: %+v", rt.Caller(), rt.Receiver(), ar) aerr = ar return } @@ -391,7 +391,7 @@ func (rt *Runtime) Send(to address.Address, method abi.MethodNum, m cbor.Marshal if err.IsFatal() { panic(err) } - log.Warnf("vmctx send failed: to: %s, method: %d: ret: %d, err: %s", to, method, ret, err) + log.Warnf("vmctx send failed: from: %s to: %s, method: %d: err: %s", rt.Receiver(), to, method, err) return err.RetCode() } _ = rt.chargeGasSafe(gasOnActorExec) diff --git a/cli/mpool.go b/cli/mpool.go index b128ccc15..adefd25a8 100644 --- a/cli/mpool.go +++ b/cli/mpool.go @@ -381,8 +381,8 @@ var MpoolReplaceCmd = &cli.Command{ Usage: "automatically reprice the specified message", }, &cli.StringFlag{ - Name: "max-fee", - Usage: "Spend up to X attoFIL for this message (applicable for auto mode)", + Name: "fee-limit", + Usage: "Spend up to X FIL for this message in units of FIL. Previously when flag was `max-fee` units were in attoFIL. Applicable for auto mode", }, }, ArgsUsage: " | ", @@ -457,13 +457,13 @@ var MpoolReplaceCmd = &cli.Command{ minRBF := messagepool.ComputeMinRBF(msg.GasPremium) var mss *lapi.MessageSendSpec - if cctx.IsSet("max-fee") { - maxFee, err := types.BigFromString(cctx.String("max-fee")) + if cctx.IsSet("fee-limit") { + maxFee, err := types.ParseFIL(cctx.String("fee-limit")) if err != nil { return fmt.Errorf("parsing max-spend: %w", err) } mss = &lapi.MessageSendSpec{ - MaxFee: maxFee, + MaxFee: abi.TokenAmount(maxFee), } } diff --git a/cmd/lotus-chainwatch/dot.go b/cmd/lotus-chainwatch/dot.go deleted file mode 100644 index 3149d65f5..000000000 --- a/cmd/lotus-chainwatch/dot.go +++ /dev/null @@ -1,131 +0,0 @@ -package main - -import ( - "database/sql" - "fmt" - "hash/crc32" - "strconv" - - "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log/v2" - "github.com/urfave/cli/v2" - "golang.org/x/xerrors" -) - -var dotCmd = &cli.Command{ - Name: "dot", - Usage: "generate dot graphs", - ArgsUsage: " ", - Action: func(cctx *cli.Context) error { - ll := cctx.String("log-level") - if err := logging.SetLogLevel("*", ll); err != nil { - return err - } - - db, err := sql.Open("postgres", cctx.String("db")) - if err != nil { - return err - } - defer func() { - if err := db.Close(); err != nil { - log.Errorw("Failed to close database", "error", err) - } - }() - - if err := db.Ping(); err != nil { - return xerrors.Errorf("Database failed to respond to ping (is it online?): %w", err) - } - - minH, err := strconv.ParseInt(cctx.Args().Get(0), 10, 32) - if err != nil { - return err - } - tosee, err := strconv.ParseInt(cctx.Args().Get(1), 10, 32) - if err != nil { - return err - } - maxH := minH + tosee - - res, err := db.Query(`select block, parent, b.miner, b.height, p.height from block_parents - inner join blocks b on block_parents.block = b.cid - inner join blocks p on block_parents.parent = p.cid -where b.height > $1 and b.height < $2`, minH, maxH) - - if err != nil { - return err - } - - fmt.Println("digraph D {") - - hl, err := syncedBlocks(db) - if err != nil { - log.Fatal(err) - } - - for res.Next() { - var block, parent, miner string - var height, ph uint64 - if err := res.Scan(&block, &parent, &miner, &height, &ph); err != nil { - return err - } - - bc, err := cid.Parse(block) - if err != nil { - return err - } - - _, has := hl[bc] - - col := crc32.Checksum([]byte(miner), crc32.MakeTable(crc32.Castagnoli))&0xc0c0c0c0 + 0x30303030 - - hasstr := "" - if !has { - //col = 0xffffffff - hasstr = " UNSYNCED" - } - - nulls := height - ph - 1 - for i := uint64(0); i < nulls; i++ { - name := block + "NP" + fmt.Sprint(i) - - fmt.Printf("%s [label = \"NULL:%d\", fillcolor = \"#ffddff\", style=filled, forcelabels=true]\n%s -> %s\n", - name, height-nulls+i, name, parent) - - parent = name - } - - fmt.Printf("%s [label = \"%s:%d%s\", fillcolor = \"#%06x\", style=filled, forcelabels=true]\n%s -> %s\n", block, miner, height, hasstr, col, block, parent) - } - if res.Err() != nil { - return res.Err() - } - - fmt.Println("}") - - return nil - }, -} - -func syncedBlocks(db *sql.DB) (map[cid.Cid]struct{}, error) { - // timestamp is used to return a configurable amount of rows based on when they were last added. - rws, err := db.Query(`select cid FROM blocks_synced`) - if err != nil { - return nil, xerrors.Errorf("Failed to query blocks_synced: %w", err) - } - out := map[cid.Cid]struct{}{} - - for rws.Next() { - var c string - if err := rws.Scan(&c); err != nil { - return nil, xerrors.Errorf("Failed to scan blocks_synced: %w", err) - } - - ci, err := cid.Parse(c) - if err != nil { - return nil, xerrors.Errorf("Failed to parse blocks_synced: %w", err) - } - - out[ci] = struct{}{} - } - return out, nil -} diff --git a/cmd/lotus-chainwatch/main.go b/cmd/lotus-chainwatch/main.go deleted file mode 100644 index 5cb0f3507..000000000 --- a/cmd/lotus-chainwatch/main.go +++ /dev/null @@ -1,54 +0,0 @@ -package main - -import ( - "os" - - "github.com/filecoin-project/lotus/build" - logging "github.com/ipfs/go-log/v2" - "github.com/urfave/cli/v2" -) - -var log = logging.Logger("chainwatch") - -func main() { - if err := logging.SetLogLevel("*", "info"); err != nil { - log.Fatal(err) - } - log.Info("Starting chainwatch", " v", build.UserVersion()) - - app := &cli.App{ - Name: "lotus-chainwatch", - Usage: "Devnet token distribution utility", - Version: build.UserVersion(), - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "repo", - EnvVars: []string{"LOTUS_PATH"}, - Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME - }, - &cli.StringFlag{ - Name: "api", - EnvVars: []string{"FULLNODE_API_INFO"}, - Value: "", - }, - &cli.StringFlag{ - Name: "db", - EnvVars: []string{"LOTUS_DB"}, - Value: "", - }, - &cli.StringFlag{ - Name: "log-level", - EnvVars: []string{"GOLOG_LOG_LEVEL"}, - Value: "info", - }, - }, - Commands: []*cli.Command{ - dotCmd, - runCmd, - }, - } - - if err := app.Run(os.Args); err != nil { - log.Fatal(err) - } -} diff --git a/cmd/lotus-chainwatch/processor/common_actors.go b/cmd/lotus-chainwatch/processor/common_actors.go deleted file mode 100644 index 0f2c0d2ea..000000000 --- a/cmd/lotus-chainwatch/processor/common_actors.go +++ /dev/null @@ -1,299 +0,0 @@ -package processor - -import ( - "context" - "time" - - "golang.org/x/sync/errgroup" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" - "github.com/ipfs/go-cid" - - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" - - "github.com/filecoin-project/lotus/chain/actors/builtin" - _init "github.com/filecoin-project/lotus/chain/actors/builtin/init" - "github.com/filecoin-project/lotus/chain/events/state" - "github.com/filecoin-project/lotus/chain/types" - cw_util "github.com/filecoin-project/lotus/cmd/lotus-chainwatch/util" -) - -func (p *Processor) setupCommonActors() error { - tx, err := p.db.Begin() - if err != nil { - return err - } - - if _, err := tx.Exec(` -create table if not exists id_address_map -( - id text not null, - address text not null, - constraint id_address_map_pk - primary key (id, address) -); - -create unique index if not exists id_address_map_id_uindex - on id_address_map (id); - -create unique index if not exists id_address_map_address_uindex - on id_address_map (address); - -create table if not exists actors - ( - id text not null - constraint id_address_map_actors_id_fk - references id_address_map (id), - code text not null, - head text not null, - nonce int not null, - balance text not null, - stateroot text - ); - -create index if not exists actors_id_index - on actors (id); - -create index if not exists id_address_map_address_index - on id_address_map (address); - -create index if not exists id_address_map_id_index - on id_address_map (id); - -create or replace function actor_tips(epoch bigint) - returns table (id text, - code text, - head text, - nonce int, - balance text, - stateroot text, - height bigint, - parentstateroot text) as -$body$ - select distinct on (id) * from actors - inner join state_heights sh on sh.parentstateroot = stateroot - where height < $1 - order by id, height desc; -$body$ language sql; - -create table if not exists actor_states -( - head text not null, - code text not null, - state json not null -); - -create unique index if not exists actor_states_head_code_uindex - on actor_states (head, code); - -create index if not exists actor_states_head_index - on actor_states (head); - -create index if not exists actor_states_code_head_index - on actor_states (head, code); - -`); err != nil { - return err - } - - return tx.Commit() -} - -func (p *Processor) HandleCommonActorsChanges(ctx context.Context, actors map[cid.Cid]ActorTips) error { - if err := p.storeActorAddresses(ctx, actors); err != nil { - return err - } - - grp, _ := errgroup.WithContext(ctx) - - grp.Go(func() error { - if err := p.storeActorHeads(actors); err != nil { - return err - } - return nil - }) - - grp.Go(func() error { - if err := p.storeActorStates(actors); err != nil { - return err - } - return nil - }) - - return grp.Wait() -} - -type UpdateAddresses struct { - Old state.AddressPair - New state.AddressPair -} - -func (p Processor) storeActorAddresses(ctx context.Context, actors map[cid.Cid]ActorTips) error { - start := time.Now() - defer func() { - log.Debugw("Stored Actor Addresses", "duration", time.Since(start).String()) - }() - - addressToID := map[address.Address]address.Address{} - // HACK until genesis storage is figured out: - addressToID[builtin2.SystemActorAddr] = builtin2.SystemActorAddr - addressToID[builtin2.InitActorAddr] = builtin2.InitActorAddr - addressToID[builtin2.RewardActorAddr] = builtin2.RewardActorAddr - addressToID[builtin2.CronActorAddr] = builtin2.CronActorAddr - addressToID[builtin2.StoragePowerActorAddr] = builtin2.StoragePowerActorAddr - addressToID[builtin2.StorageMarketActorAddr] = builtin2.StorageMarketActorAddr - addressToID[builtin2.VerifiedRegistryActorAddr] = builtin2.VerifiedRegistryActorAddr - addressToID[builtin2.BurntFundsActorAddr] = builtin2.BurntFundsActorAddr - initActor, err := p.node.StateGetActor(ctx, builtin2.InitActorAddr, types.EmptyTSK) - if err != nil { - return err - } - - initActorState, err := _init.Load(cw_util.NewAPIIpldStore(ctx, p.node), initActor) - if err != nil { - return err - } - // gross.. - if err := initActorState.ForEachActor(func(id abi.ActorID, addr address.Address) error { - idAddr, err := address.NewIDAddress(uint64(id)) - if err != nil { - return err - } - addressToID[addr] = idAddr - return nil - }); err != nil { - return err - } - tx, err := p.db.Begin() - if err != nil { - return err - } - - if _, err := tx.Exec(` -create temp table iam (like id_address_map excluding constraints) on commit drop; -`); err != nil { - return xerrors.Errorf("prep temp: %w", err) - } - - stmt, err := tx.Prepare(`copy iam (id, address) from STDIN `) - if err != nil { - return err - } - - for a, i := range addressToID { - if i == address.Undef { - continue - } - if _, err := stmt.Exec( - i.String(), - a.String(), - ); err != nil { - return err - } - } - if err := stmt.Close(); err != nil { - return err - } - - // HACK until chain watch can handle reorgs we need to update this table when ID -> PubKey mappings change - if _, err := tx.Exec(`insert into id_address_map select * from iam on conflict (id) do update set address = EXCLUDED.address`); err != nil { - log.Warnw("Failed to update id_address_map table, this is a known issue") - return nil - } - - return tx.Commit() -} - -func (p *Processor) storeActorHeads(actors map[cid.Cid]ActorTips) error { - start := time.Now() - defer func() { - log.Debugw("Stored Actor Heads", "duration", time.Since(start).String()) - }() - // Basic - tx, err := p.db.Begin() - if err != nil { - return err - } - if _, err := tx.Exec(` - create temp table a_tmp (like actors excluding constraints) on commit drop; - `); err != nil { - return xerrors.Errorf("prep temp: %w", err) - } - - stmt, err := tx.Prepare(`copy a_tmp (id, code, head, nonce, balance, stateroot) from stdin `) - if err != nil { - return err - } - - for code, actTips := range actors { - actorName := code.String() - if builtin.IsBuiltinActor(code) { - actorName = builtin.ActorNameByCode(code) - } - for _, actorInfo := range actTips { - for _, a := range actorInfo { - if _, err := stmt.Exec(a.addr.String(), actorName, a.act.Head.String(), a.act.Nonce, a.act.Balance.String(), a.stateroot.String()); err != nil { - return err - } - } - } - } - - if err := stmt.Close(); err != nil { - return err - } - - if _, err := tx.Exec(`insert into actors select * from a_tmp on conflict do nothing `); err != nil { - return xerrors.Errorf("actor put: %w", err) - } - - return tx.Commit() -} - -func (p *Processor) storeActorStates(actors map[cid.Cid]ActorTips) error { - start := time.Now() - defer func() { - log.Debugw("Stored Actor States", "duration", time.Since(start).String()) - }() - // States - tx, err := p.db.Begin() - if err != nil { - return err - } - if _, err := tx.Exec(` - create temp table as_tmp (like actor_states excluding constraints) on commit drop; - `); err != nil { - return xerrors.Errorf("prep temp: %w", err) - } - - stmt, err := tx.Prepare(`copy as_tmp (head, code, state) from stdin `) - if err != nil { - return err - } - - for code, actTips := range actors { - actorName := code.String() - if builtin.IsBuiltinActor(code) { - actorName = builtin.ActorNameByCode(code) - } - for _, actorInfo := range actTips { - for _, a := range actorInfo { - if _, err := stmt.Exec(a.act.Head.String(), actorName, a.state); err != nil { - return err - } - } - } - } - - if err := stmt.Close(); err != nil { - return err - } - - if _, err := tx.Exec(`insert into actor_states select * from as_tmp on conflict do nothing `); err != nil { - return xerrors.Errorf("actor put: %w", err) - } - - return tx.Commit() -} diff --git a/cmd/lotus-chainwatch/processor/market.go b/cmd/lotus-chainwatch/processor/market.go deleted file mode 100644 index 17aa1c37b..000000000 --- a/cmd/lotus-chainwatch/processor/market.go +++ /dev/null @@ -1,316 +0,0 @@ -package processor - -import ( - "context" - "strconv" - "time" - - "golang.org/x/sync/errgroup" - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/chain/actors/builtin/market" - "github.com/filecoin-project/lotus/chain/events/state" -) - -func (p *Processor) setupMarket() error { - tx, err := p.db.Begin() - if err != nil { - return err - } - - if _, err := tx.Exec(` -create table if not exists market_deal_proposals -( - deal_id bigint not null, - - state_root text not null, - - piece_cid text not null, - padded_piece_size bigint not null, - unpadded_piece_size bigint not null, - is_verified bool not null, - - client_id text not null, - provider_id text not null, - - start_epoch bigint not null, - end_epoch bigint not null, - slashed_epoch bigint, - storage_price_per_epoch text not null, - - provider_collateral text not null, - client_collateral text not null, - - constraint market_deal_proposal_pk - primary key (deal_id) -); - -create table if not exists market_deal_states -( - deal_id bigint not null, - - sector_start_epoch bigint not null, - last_update_epoch bigint not null, - slash_epoch bigint not null, - - state_root text not null, - - unique (deal_id, sector_start_epoch, last_update_epoch, slash_epoch), - - constraint market_deal_states_pk - primary key (deal_id, state_root) - -); - -create table if not exists minerid_dealid_sectorid -( - deal_id bigint not null - constraint sectors_sector_ids_id_fk - references market_deal_proposals(deal_id), - - sector_id bigint not null, - miner_id text not null, - foreign key (sector_id, miner_id) references sector_precommit_info(sector_id, miner_id), - - constraint miner_sector_deal_ids_pk - primary key (miner_id, sector_id, deal_id) -); - -`); err != nil { - return err - } - - return tx.Commit() -} - -type marketActorInfo struct { - common actorInfo -} - -func (p *Processor) HandleMarketChanges(ctx context.Context, marketTips ActorTips) error { - marketChanges, err := p.processMarket(ctx, marketTips) - if err != nil { - log.Fatalw("Failed to process market actors", "error", err) - } - - if err := p.persistMarket(ctx, marketChanges); err != nil { - log.Fatalw("Failed to persist market actors", "error", err) - } - - if err := p.updateMarket(ctx, marketChanges); err != nil { - log.Fatalw("Failed to update market actors", "error", err) - } - return nil -} - -func (p *Processor) processMarket(ctx context.Context, marketTips ActorTips) ([]marketActorInfo, error) { - start := time.Now() - defer func() { - log.Debugw("Processed Market", "duration", time.Since(start).String()) - }() - - var out []marketActorInfo - for _, markets := range marketTips { - for _, mt := range markets { - // NB: here is where we can extract the market state when we need it. - out = append(out, marketActorInfo{common: mt}) - } - } - return out, nil -} - -func (p *Processor) persistMarket(ctx context.Context, info []marketActorInfo) error { - start := time.Now() - defer func() { - log.Debugw("Persisted Market", "duration", time.Since(start).String()) - }() - - grp, ctx := errgroup.WithContext(ctx) - - grp.Go(func() error { - if err := p.storeMarketActorDealProposals(ctx, info); err != nil { - return xerrors.Errorf("Failed to store marker deal proposals: %w", err) - } - return nil - }) - - grp.Go(func() error { - if err := p.storeMarketActorDealStates(info); err != nil { - return xerrors.Errorf("Failed to store marker deal states: %w", err) - } - return nil - }) - - return grp.Wait() - -} - -func (p *Processor) updateMarket(ctx context.Context, info []marketActorInfo) error { - if err := p.updateMarketActorDealProposals(ctx, info); err != nil { - return xerrors.Errorf("Failed to update market info: %w", err) - } - return nil -} - -func (p *Processor) storeMarketActorDealStates(marketTips []marketActorInfo) error { - start := time.Now() - defer func() { - log.Debugw("Stored Market Deal States", "duration", time.Since(start).String()) - }() - tx, err := p.db.Begin() - if err != nil { - return err - } - if _, err := tx.Exec(`create temp table mds (like market_deal_states excluding constraints) on commit drop;`); err != nil { - return err - } - stmt, err := tx.Prepare(`copy mds (deal_id, sector_start_epoch, last_update_epoch, slash_epoch, state_root) from STDIN`) - if err != nil { - return err - } - for _, mt := range marketTips { - dealStates, err := p.node.StateMarketDeals(context.TODO(), mt.common.tsKey) - if err != nil { - return err - } - - for dealID, ds := range dealStates { - id, err := strconv.ParseUint(dealID, 10, 64) - if err != nil { - return err - } - - if _, err := stmt.Exec( - id, - ds.State.SectorStartEpoch, - ds.State.LastUpdatedEpoch, - ds.State.SlashEpoch, - mt.common.stateroot.String(), - ); err != nil { - return err - } - - } - } - if err := stmt.Close(); err != nil { - return err - } - - if _, err := tx.Exec(`insert into market_deal_states select * from mds on conflict do nothing`); err != nil { - return err - } - - return tx.Commit() -} - -func (p *Processor) storeMarketActorDealProposals(ctx context.Context, marketTips []marketActorInfo) error { - start := time.Now() - defer func() { - log.Debugw("Stored Market Deal Proposals", "duration", time.Since(start).String()) - }() - tx, err := p.db.Begin() - if err != nil { - return err - } - - if _, err := tx.Exec(`create temp table mdp (like market_deal_proposals excluding constraints) on commit drop;`); err != nil { - return xerrors.Errorf("prep temp: %w", err) - } - - stmt, err := tx.Prepare(`copy mdp (deal_id, state_root, piece_cid, padded_piece_size, unpadded_piece_size, is_verified, client_id, provider_id, start_epoch, end_epoch, slashed_epoch, storage_price_per_epoch, provider_collateral, client_collateral) from STDIN`) - if err != nil { - return err - } - - // insert in sorted order (lowest height -> highest height) since dealid is pk of table. - for _, mt := range marketTips { - dealStates, err := p.node.StateMarketDeals(ctx, mt.common.tsKey) - if err != nil { - return err - } - - for dealID, ds := range dealStates { - id, err := strconv.ParseUint(dealID, 10, 64) - if err != nil { - return err - } - - if _, err := stmt.Exec( - id, - mt.common.stateroot.String(), - ds.Proposal.PieceCID.String(), - ds.Proposal.PieceSize, - ds.Proposal.PieceSize.Unpadded(), - ds.Proposal.VerifiedDeal, - ds.Proposal.Client.String(), - ds.Proposal.Provider.String(), - ds.Proposal.StartEpoch, - ds.Proposal.EndEpoch, - nil, // slashed_epoch - ds.Proposal.StoragePricePerEpoch.String(), - ds.Proposal.ProviderCollateral.String(), - ds.Proposal.ClientCollateral.String(), - ); err != nil { - return err - } - - } - } - if err := stmt.Close(); err != nil { - return err - } - if _, err := tx.Exec(`insert into market_deal_proposals select * from mdp on conflict do nothing`); err != nil { - return err - } - - return tx.Commit() - -} - -func (p *Processor) updateMarketActorDealProposals(ctx context.Context, marketTip []marketActorInfo) error { - start := time.Now() - defer func() { - log.Debugw("Updated Market Deal Proposals", "duration", time.Since(start).String()) - }() - pred := state.NewStatePredicates(p.node) - - tx, err := p.db.Begin() - if err != nil { - return err - } - - stmt, err := tx.Prepare(`update market_deal_proposals set slashed_epoch=$1 where deal_id=$2`) - if err != nil { - return err - } - - for _, mt := range marketTip { - stateDiff := pred.OnStorageMarketActorChanged(pred.OnDealStateChanged(pred.OnDealStateAmtChanged())) - - changed, val, err := stateDiff(ctx, mt.common.parentTsKey, mt.common.tsKey) - if err != nil { - log.Warnw("error getting market deal state diff", "error", err) - } - if !changed { - continue - } - changes, ok := val.(*market.DealStateChanges) - if !ok { - return xerrors.Errorf("Unknown type returned by Deal State AMT predicate: %T", val) - } - - for _, modified := range changes.Modified { - if modified.From.SlashEpoch != modified.To.SlashEpoch { - if _, err := stmt.Exec(modified.To.SlashEpoch, modified.ID); err != nil { - return err - } - } - } - } - - if err := stmt.Close(); err != nil { - return err - } - - return tx.Commit() -} diff --git a/cmd/lotus-chainwatch/processor/messages.go b/cmd/lotus-chainwatch/processor/messages.go deleted file mode 100644 index 333477c6a..000000000 --- a/cmd/lotus-chainwatch/processor/messages.go +++ /dev/null @@ -1,318 +0,0 @@ -package processor - -import ( - "context" - "sync" - - "golang.org/x/sync/errgroup" - "golang.org/x/xerrors" - - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/parmap" -) - -func (p *Processor) setupMessages() error { - tx, err := p.db.Begin() - if err != nil { - return err - } - - if _, err := tx.Exec(` -create table if not exists messages -( - cid text not null - constraint messages_pk - primary key, - "from" text not null, - "to" text not null, - size_bytes bigint not null, - nonce bigint not null, - value text not null, - gas_fee_cap text not null, - gas_premium text not null, - gas_limit bigint not null, - method bigint, - params bytea -); - -create unique index if not exists messages_cid_uindex - on messages (cid); - -create index if not exists messages_from_index - on messages ("from"); - -create index if not exists messages_to_index - on messages ("to"); - -create table if not exists block_messages -( - block text not null - constraint blocks_block_cids_cid_fk - references block_cids (cid), - message text not null, - constraint block_messages_pk - primary key (block, message) -); - -create table if not exists mpool_messages -( - msg text not null - constraint mpool_messages_pk - primary key - constraint mpool_messages_messages_cid_fk - references messages, - add_ts int not null -); - -create unique index if not exists mpool_messages_msg_uindex - on mpool_messages (msg); - -create table if not exists receipts -( - msg text not null, - state text not null, - idx int not null, - exit int not null, - gas_used bigint not null, - return bytea, - constraint receipts_pk - primary key (msg, state) -); - -create index if not exists receipts_msg_state_index - on receipts (msg, state); -`); err != nil { - return err - } - - return tx.Commit() -} - -func (p *Processor) HandleMessageChanges(ctx context.Context, blocks map[cid.Cid]*types.BlockHeader) error { - if err := p.persistMessagesAndReceipts(ctx, blocks); err != nil { - return err - } - return nil -} - -func (p *Processor) persistMessagesAndReceipts(ctx context.Context, blocks map[cid.Cid]*types.BlockHeader) error { - messages, inclusions := p.fetchMessages(ctx, blocks) - receipts := p.fetchParentReceipts(ctx, blocks) - - grp, _ := errgroup.WithContext(ctx) - - grp.Go(func() error { - return p.storeMessages(messages) - }) - - grp.Go(func() error { - return p.storeMsgInclusions(inclusions) - }) - - grp.Go(func() error { - return p.storeReceipts(receipts) - }) - - return grp.Wait() -} - -func (p *Processor) storeReceipts(recs map[mrec]*types.MessageReceipt) error { - tx, err := p.db.Begin() - if err != nil { - return err - } - - if _, err := tx.Exec(` -create temp table recs (like receipts excluding constraints) on commit drop; -`); err != nil { - return xerrors.Errorf("prep temp: %w", err) - } - - stmt, err := tx.Prepare(`copy recs (msg, state, idx, exit, gas_used, return) from stdin `) - if err != nil { - return err - } - - for c, m := range recs { - if _, err := stmt.Exec( - c.msg.String(), - c.state.String(), - c.idx, - m.ExitCode, - m.GasUsed, - m.Return, - ); err != nil { - return err - } - } - if err := stmt.Close(); err != nil { - return err - } - - if _, err := tx.Exec(`insert into receipts select * from recs on conflict do nothing `); err != nil { - return xerrors.Errorf("actor put: %w", err) - } - - return tx.Commit() -} - -func (p *Processor) storeMsgInclusions(incls map[cid.Cid][]cid.Cid) error { - tx, err := p.db.Begin() - if err != nil { - return err - } - - if _, err := tx.Exec(` -create temp table mi (like block_messages excluding constraints) on commit drop; -`); err != nil { - return xerrors.Errorf("prep temp: %w", err) - } - - stmt, err := tx.Prepare(`copy mi (block, message) from STDIN `) - if err != nil { - return err - } - - for b, msgs := range incls { - for _, msg := range msgs { - if _, err := stmt.Exec( - b.String(), - msg.String(), - ); err != nil { - return err - } - } - } - if err := stmt.Close(); err != nil { - return err - } - - if _, err := tx.Exec(`insert into block_messages select * from mi on conflict do nothing `); err != nil { - return xerrors.Errorf("actor put: %w", err) - } - - return tx.Commit() -} - -func (p *Processor) storeMessages(msgs map[cid.Cid]*types.Message) error { - tx, err := p.db.Begin() - if err != nil { - return err - } - - if _, err := tx.Exec(` -create temp table msgs (like messages excluding constraints) on commit drop; -`); err != nil { - return xerrors.Errorf("prep temp: %w", err) - } - - stmt, err := tx.Prepare(`copy msgs (cid, "from", "to", size_bytes, nonce, "value", gas_premium, gas_fee_cap, gas_limit, method, params) from stdin `) - if err != nil { - return err - } - - for c, m := range msgs { - var msgBytes int - if b, err := m.Serialize(); err == nil { - msgBytes = len(b) - } - - if _, err := stmt.Exec( - c.String(), - m.From.String(), - m.To.String(), - msgBytes, - m.Nonce, - m.Value.String(), - m.GasPremium.String(), - m.GasFeeCap.String(), - m.GasLimit, - m.Method, - m.Params, - ); err != nil { - return err - } - } - if err := stmt.Close(); err != nil { - return err - } - - if _, err := tx.Exec(`insert into messages select * from msgs on conflict do nothing `); err != nil { - return xerrors.Errorf("actor put: %w", err) - } - - return tx.Commit() -} - -func (p *Processor) fetchMessages(ctx context.Context, blocks map[cid.Cid]*types.BlockHeader) (map[cid.Cid]*types.Message, map[cid.Cid][]cid.Cid) { - var lk sync.Mutex - messages := map[cid.Cid]*types.Message{} - inclusions := map[cid.Cid][]cid.Cid{} // block -> msgs - - parmap.Par(50, parmap.MapArr(blocks), func(header *types.BlockHeader) { - msgs, err := p.node.ChainGetBlockMessages(ctx, header.Cid()) - if err != nil { - log.Error(err) - log.Debugw("ChainGetBlockMessages", "header_cid", header.Cid()) - return - } - - vmm := make([]*types.Message, 0, len(msgs.Cids)) - for _, m := range msgs.BlsMessages { - vmm = append(vmm, m) - } - - for _, m := range msgs.SecpkMessages { - vmm = append(vmm, &m.Message) - } - - lk.Lock() - for _, message := range vmm { - messages[message.Cid()] = message - inclusions[header.Cid()] = append(inclusions[header.Cid()], message.Cid()) - } - lk.Unlock() - }) - - return messages, inclusions -} - -type mrec struct { - msg cid.Cid - state cid.Cid - idx int -} - -func (p *Processor) fetchParentReceipts(ctx context.Context, toSync map[cid.Cid]*types.BlockHeader) map[mrec]*types.MessageReceipt { - var lk sync.Mutex - out := map[mrec]*types.MessageReceipt{} - - parmap.Par(50, parmap.MapArr(toSync), func(header *types.BlockHeader) { - recs, err := p.node.ChainGetParentReceipts(ctx, header.Cid()) - if err != nil { - log.Error(err) - log.Debugw("ChainGetParentReceipts", "header_cid", header.Cid()) - return - } - msgs, err := p.node.ChainGetParentMessages(ctx, header.Cid()) - if err != nil { - log.Error(err) - log.Debugw("ChainGetParentMessages", "header_cid", header.Cid()) - return - } - - lk.Lock() - for i, r := range recs { - out[mrec{ - msg: msgs[i].Cid, - state: header.ParentStateRoot, - idx: i, - }] = r - } - lk.Unlock() - }) - - return out -} diff --git a/cmd/lotus-chainwatch/processor/miner.go b/cmd/lotus-chainwatch/processor/miner.go deleted file mode 100644 index f3514df88..000000000 --- a/cmd/lotus-chainwatch/processor/miner.go +++ /dev/null @@ -1,1035 +0,0 @@ -package processor - -import ( - "context" - "strings" - "time" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-bitfield" - "github.com/ipfs/go-cid" - "golang.org/x/sync/errgroup" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" - - "github.com/filecoin-project/lotus/api/v0api" - "github.com/filecoin-project/lotus/blockstore" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" - "github.com/filecoin-project/lotus/chain/actors/builtin/power" - "github.com/filecoin-project/lotus/chain/events/state" - "github.com/filecoin-project/lotus/chain/store" - "github.com/filecoin-project/lotus/chain/types" - cw_util "github.com/filecoin-project/lotus/cmd/lotus-chainwatch/util" -) - -func (p *Processor) setupMiners() error { - tx, err := p.db.Begin() - if err != nil { - return err - } - - if _, err := tx.Exec(` - -create table if not exists miner_info -( - miner_id text not null, - owner_addr text not null, - worker_addr text not null, - peer_id text, - sector_size text not null, - - constraint miner_info_pk - primary key (miner_id) -); - -create table if not exists sector_precommit_info -( - miner_id text not null, - sector_id bigint not null, - sealed_cid text not null, - state_root text not null, - - seal_rand_epoch bigint not null, - expiration_epoch bigint not null, - - precommit_deposit text not null, - precommit_epoch bigint not null, - deal_weight text not null, - verified_deal_weight text not null, - - - is_replace_capacity bool not null, - replace_sector_deadline bigint, - replace_sector_partition bigint, - replace_sector_number bigint, - - unique (miner_id, sector_id), - - constraint sector_precommit_info_pk - primary key (miner_id, sector_id, sealed_cid) - -); - -create table if not exists sector_info -( - miner_id text not null, - sector_id bigint not null, - sealed_cid text not null, - state_root text not null, - - activation_epoch bigint not null, - expiration_epoch bigint not null, - - deal_weight text not null, - verified_deal_weight text not null, - - initial_pledge text not null, - expected_day_reward text not null, - expected_storage_pledge text not null, - - constraint sector_info_pk - primary key (miner_id, sector_id, sealed_cid) -); - -/* -* captures miner-specific power state for any given stateroot -*/ -create table if not exists miner_power -( - miner_id text not null, - state_root text not null, - raw_bytes_power text not null, - quality_adjusted_power text not null, - constraint miner_power_pk - primary key (miner_id, state_root) -); - -DO $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'miner_sector_event_type') THEN - CREATE TYPE miner_sector_event_type AS ENUM - ( - 'PRECOMMIT_ADDED', 'PRECOMMIT_EXPIRED', 'COMMIT_CAPACITY_ADDED', 'SECTOR_ADDED', - 'SECTOR_EXTENDED', 'SECTOR_EXPIRED', 'SECTOR_FAULTED', 'SECTOR_RECOVERING', 'SECTOR_RECOVERED', 'SECTOR_TERMINATED' - ); - END IF; -END$$; - -create table if not exists miner_sector_events -( - miner_id text not null, - sector_id bigint not null, - state_root text not null, - event miner_sector_event_type not null, - - constraint miner_sector_events_pk - primary key (sector_id, event, miner_id, state_root) -); - -`); err != nil { - return err - } - - return tx.Commit() -} - -type SectorLifecycleEvent string - -const ( - PreCommitAdded = "PRECOMMIT_ADDED" - PreCommitExpired = "PRECOMMIT_EXPIRED" - - CommitCapacityAdded = "COMMIT_CAPACITY_ADDED" - - SectorAdded = "SECTOR_ADDED" - SectorExpired = "SECTOR_EXPIRED" - SectorExtended = "SECTOR_EXTENDED" - SectorFaulted = "SECTOR_FAULTED" - SectorRecovering = "SECTOR_RECOVERING" - SectorRecovered = "SECTOR_RECOVERED" - SectorTerminated = "SECTOR_TERMINATED" -) - -type MinerSectorsEvent struct { - MinerID address.Address - SectorIDs []uint64 - StateRoot cid.Cid - Event SectorLifecycleEvent -} - -type SectorDealEvent struct { - MinerID address.Address - SectorID uint64 - DealIDs []abi.DealID -} - -type PartitionStatus struct { - Terminated bitfield.BitField - Expired bitfield.BitField - Faulted bitfield.BitField - InRecovery bitfield.BitField - Recovered bitfield.BitField -} - -type minerActorInfo struct { - common actorInfo - - state miner.State - - // tracked by power actor - rawPower big.Int - qalPower big.Int -} - -func (p *Processor) HandleMinerChanges(ctx context.Context, minerTips ActorTips) error { - minerChanges, err := p.processMiners(ctx, minerTips) - if err != nil { - log.Fatalw("Failed to process miner actors", "error", err) - } - - if err := p.persistMiners(ctx, minerChanges); err != nil { - log.Fatalw("Failed to persist miner actors", "error", err) - } - - return nil -} - -func (p *Processor) processMiners(ctx context.Context, minerTips map[types.TipSetKey][]actorInfo) ([]minerActorInfo, error) { - start := time.Now() - defer func() { - log.Debugw("Processed Miners", "duration", time.Since(start).String()) - }() - - stor := store.ActorStore(ctx, blockstore.NewAPIBlockstore(p.node)) - - var out []minerActorInfo - // TODO add parallel calls if this becomes slow - for tipset, miners := range minerTips { - // get the power actors claims map - powerState, err := getPowerActorState(ctx, p.node, tipset) - if err != nil { - return nil, err - } - - // Get miner raw and quality power - for _, act := range miners { - var mi minerActorInfo - mi.common = act - - // get miner claim from power actors claim map and store if found, else the miner had no claim at - // this tipset - claim, found, err := powerState.MinerPower(act.addr) - if err != nil { - return nil, err - } - if found { - mi.qalPower = claim.QualityAdjPower - mi.rawPower = claim.RawBytePower - } - - // Get the miner state - mas, err := miner.Load(stor, &act.act) - if err != nil { - log.Warnw("failed to find miner actor state", "address", act.addr, "error", err) - continue - } - mi.state = mas - out = append(out, mi) - } - } - return out, nil -} - -func (p *Processor) persistMiners(ctx context.Context, miners []minerActorInfo) error { - start := time.Now() - defer func() { - log.Debugw("Persisted Miners", "duration", time.Since(start).String()) - }() - - grp, _ := errgroup.WithContext(ctx) - - grp.Go(func() error { - if err := p.storeMinersPower(miners); err != nil { - return err - } - return nil - }) - - grp.Go(func() error { - if err := p.storeMinersActorInfoState(ctx, miners); err != nil { - return err - } - return nil - }) - - // 8 is arbitrary, idk what a good value here is. - preCommitEvents := make(chan *MinerSectorsEvent, 8) - sectorEvents := make(chan *MinerSectorsEvent, 8) - partitionEvents := make(chan *MinerSectorsEvent, 8) - dealEvents := make(chan *SectorDealEvent, 8) - - grp.Go(func() error { - return p.storePreCommitDealInfo(dealEvents) - }) - - grp.Go(func() error { - return p.storeMinerSectorEvents(ctx, sectorEvents, preCommitEvents, partitionEvents) - }) - - grp.Go(func() error { - defer func() { - close(preCommitEvents) - close(dealEvents) - }() - return p.storeMinerPreCommitInfo(ctx, miners, preCommitEvents, dealEvents) - }) - - grp.Go(func() error { - defer close(sectorEvents) - return p.storeMinerSectorInfo(ctx, miners, sectorEvents) - }) - - grp.Go(func() error { - defer close(partitionEvents) - return p.getMinerPartitionsDifferences(ctx, miners, partitionEvents) - }) - - return grp.Wait() -} - -func (p *Processor) storeMinerPreCommitInfo(ctx context.Context, miners []minerActorInfo, sectorEvents chan<- *MinerSectorsEvent, sectorDeals chan<- *SectorDealEvent) error { - tx, err := p.db.Begin() - if err != nil { - return err - } - - if _, err := tx.Exec(`create temp table spi (like sector_precommit_info excluding constraints) on commit drop;`); err != nil { - return xerrors.Errorf("Failed to create temp table for sector_precommit_info: %w", err) - } - - stmt, err := tx.Prepare(`copy spi (miner_id, sector_id, sealed_cid, state_root, seal_rand_epoch, expiration_epoch, precommit_deposit, precommit_epoch, deal_weight, verified_deal_weight, is_replace_capacity, replace_sector_deadline, replace_sector_partition, replace_sector_number) from STDIN`) - - if err != nil { - return xerrors.Errorf("Failed to prepare miner precommit info statement: %w", err) - } - - grp, _ := errgroup.WithContext(ctx) - for _, m := range miners { - m := m - grp.Go(func() error { - changes, err := p.getMinerPreCommitChanges(ctx, m) - if err != nil { - if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) { - return nil - } - return err - } - if changes == nil { - return nil - } - - preCommitAdded := make([]uint64, len(changes.Added)) - for i, added := range changes.Added { - if len(added.Info.DealIDs) > 0 { - sectorDeals <- &SectorDealEvent{ - MinerID: m.common.addr, - SectorID: uint64(added.Info.SectorNumber), - DealIDs: added.Info.DealIDs, - } - } - if added.Info.ReplaceCapacity { - if _, err := stmt.Exec( - m.common.addr.String(), - added.Info.SectorNumber, - added.Info.SealedCID.String(), - m.common.stateroot.String(), - added.Info.SealRandEpoch, - added.Info.Expiration, - added.PreCommitDeposit.String(), - added.PreCommitEpoch, - added.DealWeight.String(), - added.VerifiedDealWeight.String(), - added.Info.ReplaceCapacity, - added.Info.ReplaceSectorDeadline, - added.Info.ReplaceSectorPartition, - added.Info.ReplaceSectorNumber, - ); err != nil { - return err - } - } else { - if _, err := stmt.Exec( - m.common.addr.String(), - added.Info.SectorNumber, - added.Info.SealedCID.String(), - m.common.stateroot.String(), - added.Info.SealRandEpoch, - added.Info.Expiration, - added.PreCommitDeposit.String(), - added.PreCommitEpoch, - added.DealWeight.String(), - added.VerifiedDealWeight.String(), - added.Info.ReplaceCapacity, - nil, // replace deadline - nil, // replace partition - nil, // replace sector - ); err != nil { - return err - } - - } - preCommitAdded[i] = uint64(added.Info.SectorNumber) - } - if len(preCommitAdded) > 0 { - sectorEvents <- &MinerSectorsEvent{ - MinerID: m.common.addr, - StateRoot: m.common.stateroot, - SectorIDs: preCommitAdded, - Event: PreCommitAdded, - } - } - var preCommitExpired []uint64 - for _, removed := range changes.Removed { - // TODO: we can optimize this to not load the AMT every time, if necessary. - si, err := m.state.GetSector(removed.Info.SectorNumber) - if err != nil { - return err - } - if si == nil { - preCommitExpired = append(preCommitExpired, uint64(removed.Info.SectorNumber)) - } - } - if len(preCommitExpired) > 0 { - sectorEvents <- &MinerSectorsEvent{ - MinerID: m.common.addr, - StateRoot: m.common.stateroot, - SectorIDs: preCommitExpired, - Event: PreCommitExpired, - } - } - return nil - }) - } - if err := grp.Wait(); err != nil { - return err - } - - if err := stmt.Close(); err != nil { - return xerrors.Errorf("Failed to close sector precommit info statement: %w", err) - } - - if _, err := tx.Exec(`insert into sector_precommit_info select * from spi on conflict do nothing`); err != nil { - return xerrors.Errorf("Failed to insert into sector precommit info table: %w", err) - } - - if err := tx.Commit(); err != nil { - return xerrors.Errorf("Failed to commit sector precommit info: %w", err) - } - return nil -} - -func (p *Processor) storeMinerSectorInfo(ctx context.Context, miners []minerActorInfo, events chan<- *MinerSectorsEvent) error { - tx, err := p.db.Begin() - if err != nil { - return err - } - - if _, err := tx.Exec(`create temp table si (like sector_info excluding constraints) on commit drop;`); err != nil { - return xerrors.Errorf("Failed to create temp table for sector_: %w", err) - } - - stmt, err := tx.Prepare(`copy si (miner_id, sector_id, sealed_cid, state_root, activation_epoch, expiration_epoch, deal_weight, verified_deal_weight, initial_pledge, expected_day_reward, expected_storage_pledge) from STDIN`) - if err != nil { - return xerrors.Errorf("Failed to prepare miner sector info statement: %w", err) - } - - grp, _ := errgroup.WithContext(ctx) - for _, m := range miners { - m := m - grp.Go(func() error { - changes, err := p.getMinerSectorChanges(ctx, m) - if err != nil { - if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) { - return nil - } - return err - } - if changes == nil { - return nil - } - var sectorsAdded []uint64 - var ccAdded []uint64 - var extended []uint64 - for _, added := range changes.Added { - // add the sector to the table - if _, err := stmt.Exec( - m.common.addr.String(), - added.SectorNumber, - added.SealedCID.String(), - m.common.stateroot.String(), - added.Activation.String(), - added.Expiration.String(), - added.DealWeight.String(), - added.VerifiedDealWeight.String(), - added.InitialPledge.String(), - added.ExpectedDayReward.String(), - added.ExpectedStoragePledge.String(), - ); err != nil { - log.Errorw("writing miner sector changes statement", "error", err.Error()) - } - if len(added.DealIDs) == 0 { - ccAdded = append(ccAdded, uint64(added.SectorNumber)) - } else { - sectorsAdded = append(sectorsAdded, uint64(added.SectorNumber)) - } - } - - for _, mod := range changes.Extended { - extended = append(extended, uint64(mod.To.SectorNumber)) - } - - events <- &MinerSectorsEvent{ - MinerID: m.common.addr, - StateRoot: m.common.stateroot, - SectorIDs: ccAdded, - Event: CommitCapacityAdded, - } - events <- &MinerSectorsEvent{ - MinerID: m.common.addr, - StateRoot: m.common.stateroot, - SectorIDs: sectorsAdded, - Event: SectorAdded, - } - events <- &MinerSectorsEvent{ - MinerID: m.common.addr, - StateRoot: m.common.stateroot, - SectorIDs: extended, - Event: SectorExtended, - } - return nil - }) - } - - if err := grp.Wait(); err != nil { - return err - } - - if err := stmt.Close(); err != nil { - return xerrors.Errorf("Failed to close sector info statement: %w", err) - } - - if _, err := tx.Exec(`insert into sector_info select * from si on conflict do nothing`); err != nil { - return xerrors.Errorf("Failed to insert into sector info table: %w", err) - } - - if err := tx.Commit(); err != nil { - return xerrors.Errorf("Failed to commit sector info: %w", err) - } - return nil - -} - -func (p *Processor) getMinerPartitionsDifferences(ctx context.Context, miners []minerActorInfo, events chan<- *MinerSectorsEvent) error { - grp, ctx := errgroup.WithContext(ctx) - for _, m := range miners { - m := m - grp.Go(func() error { - if err := p.diffMinerPartitions(ctx, m, events); err != nil { - if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) { - return nil - } - return err - } - return nil - }) - } - return grp.Wait() -} - -func (p *Processor) storeMinerSectorEvents(ctx context.Context, sectorEvents, preCommitEvents, partitionEvents <-chan *MinerSectorsEvent) error { - tx, err := p.db.Begin() - if err != nil { - return err - } - - if _, err := tx.Exec(`create temp table mse (like miner_sector_events excluding constraints) on commit drop;`); err != nil { - return xerrors.Errorf("Failed to create temp table for sector_: %w", err) - } - - stmt, err := tx.Prepare(`copy mse (miner_id, sector_id, event, state_root) from STDIN`) - if err != nil { - return xerrors.Errorf("Failed to prepare miner sector info statement: %w", err) - } - - grp, ctx := errgroup.WithContext(ctx) - grp.Go(func() error { - innerGrp, _ := errgroup.WithContext(ctx) - for mse := range sectorEvents { - mse := mse - innerGrp.Go(func() error { - for _, sid := range mse.SectorIDs { - if _, err := stmt.Exec( - mse.MinerID.String(), - sid, - mse.Event, - mse.StateRoot.String(), - ); err != nil { - return err - } - } - return nil - }) - } - return innerGrp.Wait() - }) - - grp.Go(func() error { - innerGrp, _ := errgroup.WithContext(ctx) - for mse := range preCommitEvents { - mse := mse - innerGrp.Go(func() error { - for _, sid := range mse.SectorIDs { - if _, err := stmt.Exec( - mse.MinerID.String(), - sid, - mse.Event, - mse.StateRoot.String(), - ); err != nil { - return err - } - } - return nil - }) - } - return innerGrp.Wait() - }) - - grp.Go(func() error { - innerGrp, _ := errgroup.WithContext(ctx) - for mse := range partitionEvents { - mse := mse - grp.Go(func() error { - for _, sid := range mse.SectorIDs { - if _, err := stmt.Exec( - mse.MinerID.String(), - sid, - mse.Event, - mse.StateRoot.String(), - ); err != nil { - return err - } - } - return nil - }) - } - return innerGrp.Wait() - }) - - if err := grp.Wait(); err != nil { - return err - } - - if err := stmt.Close(); err != nil { - return xerrors.Errorf("Failed to close sector event statement: %w", err) - } - - if _, err := tx.Exec(`insert into miner_sector_events select * from mse on conflict do nothing`); err != nil { - return xerrors.Errorf("Failed to insert into sector event table: %w", err) - } - - if err := tx.Commit(); err != nil { - return xerrors.Errorf("Failed to commit sector events: %w", err) - } - return nil -} - -func (p *Processor) getMinerStateAt(ctx context.Context, maddr address.Address, tskey types.TipSetKey) (miner.State, error) { - prevActor, err := p.node.StateGetActor(ctx, maddr, tskey) - if err != nil { - return nil, err - } - return miner.Load(store.ActorStore(ctx, blockstore.NewAPIBlockstore(p.node)), prevActor) -} - -func (p *Processor) getMinerPreCommitChanges(ctx context.Context, m minerActorInfo) (*miner.PreCommitChanges, error) { - pred := state.NewStatePredicates(p.node) - changed, val, err := pred.OnMinerActorChange(m.common.addr, pred.OnMinerPreCommitChange())(ctx, m.common.parentTsKey, m.common.tsKey) - if err != nil { - return nil, xerrors.Errorf("Failed to diff miner precommit amt: %w", err) - } - if !changed { - return nil, nil - } - out := val.(*miner.PreCommitChanges) - return out, nil -} - -func (p *Processor) getMinerSectorChanges(ctx context.Context, m minerActorInfo) (*miner.SectorChanges, error) { - pred := state.NewStatePredicates(p.node) - changed, val, err := pred.OnMinerActorChange(m.common.addr, pred.OnMinerSectorChange())(ctx, m.common.parentTsKey, m.common.tsKey) - if err != nil { - return nil, xerrors.Errorf("Failed to diff miner sectors amt: %w", err) - } - if !changed { - return nil, nil - } - out := val.(*miner.SectorChanges) - return out, nil -} - -func (p *Processor) diffMinerPartitions(ctx context.Context, m minerActorInfo, events chan<- *MinerSectorsEvent) error { - prevMiner, err := p.getMinerStateAt(ctx, m.common.addr, m.common.parentTsKey) - if err != nil { - return err - } - curMiner := m.state - dc, err := prevMiner.DeadlinesChanged(curMiner) - if err != nil { - return err - } - if !dc { - return nil - } - panic("TODO") - - // FIXME: This code doesn't work. - // 1. We need to diff all deadlines, not just the "current" deadline. - // 2. We need to handle the case where we _add_ a partition. (i.e., - // where len(newPartitions) != len(oldPartitions). - /* - - // NOTE: If we change the number of deadlines in an upgrade, this will - // break. - - // load the old deadline - prevDls, err := prevMiner.LoadDeadlines(p.ctxStore) - if err != nil { - return err - } - var prevDl miner.Deadline - if err := p.ctxStore.Get(ctx, prevDls.Due[dlIdx], &prevDl); err != nil { - return err - } - - prevPartitions, err := prevDl.PartitionsArray(p.ctxStore) - if err != nil { - return err - } - - // load the new deadline - curDls, err := curMiner.LoadDeadlines(p.ctxStore) - if err != nil { - return err - } - - var curDl miner.Deadline - if err := p.ctxStore.Get(ctx, curDls.Due[dlIdx], &curDl); err != nil { - return err - } - - curPartitions, err := curDl.PartitionsArray(p.ctxStore) - if err != nil { - return err - } - - // TODO this can be optimized by inspecting the miner state for partitions that have changed and only inspecting those. - var prevPart miner.Partition - if err := prevPartitions.ForEach(&prevPart, func(i int64) error { - var curPart miner.Partition - if found, err := curPartitions.Get(uint64(i), &curPart); err != nil { - return err - } else if !found { - log.Fatal("I don't know what this means, are partitions ever removed?") - } - partitionDiff, err := p.diffPartition(prevPart, curPart) - if err != nil { - return err - } - - recovered, err := partitionDiff.Recovered.All(miner.SectorsMax) - if err != nil { - return err - } - events <- &MinerSectorsEvent{ - MinerID: m.common.addr, - StateRoot: m.common.stateroot, - SectorIDs: recovered, - Event: SectorRecovered, - } - inRecovery, err := partitionDiff.InRecovery.All(miner.SectorsMax) - if err != nil { - return err - } - events <- &MinerSectorsEvent{ - MinerID: m.common.addr, - StateRoot: m.common.stateroot, - SectorIDs: inRecovery, - Event: SectorRecovering, - } - faulted, err := partitionDiff.Faulted.All(miner.SectorsMax) - if err != nil { - return err - } - events <- &MinerSectorsEvent{ - MinerID: m.common.addr, - StateRoot: m.common.stateroot, - SectorIDs: faulted, - Event: SectorFaulted, - } - terminated, err := partitionDiff.Terminated.All(miner.SectorsMax) - if err != nil { - return err - } - events <- &MinerSectorsEvent{ - MinerID: m.common.addr, - StateRoot: m.common.stateroot, - SectorIDs: terminated, - Event: SectorTerminated, - } - expired, err := partitionDiff.Expired.All(miner.SectorsMax) - if err != nil { - return err - } - events <- &MinerSectorsEvent{ - MinerID: m.common.addr, - StateRoot: m.common.stateroot, - SectorIDs: expired, - Event: SectorExpired, - } - - return nil - }); err != nil { - return err - } - - return nil - */ -} - -func (p *Processor) diffPartition(prevPart, curPart miner.Partition) (*PartitionStatus, error) { - prevLiveSectors, err := prevPart.LiveSectors() - if err != nil { - return nil, err - } - curLiveSectors, err := curPart.LiveSectors() - if err != nil { - return nil, err - } - - removedSectors, err := bitfield.SubtractBitField(prevLiveSectors, curLiveSectors) - if err != nil { - return nil, err - } - - prevRecoveries, err := prevPart.RecoveringSectors() - if err != nil { - return nil, err - } - - curRecoveries, err := curPart.RecoveringSectors() - if err != nil { - return nil, err - } - - newRecoveries, err := bitfield.SubtractBitField(curRecoveries, prevRecoveries) - if err != nil { - return nil, err - } - - prevFaults, err := prevPart.FaultySectors() - if err != nil { - return nil, err - } - - curFaults, err := curPart.FaultySectors() - if err != nil { - return nil, err - } - - newFaults, err := bitfield.SubtractBitField(curFaults, prevFaults) - if err != nil { - return nil, err - } - - // all current good sectors - curActiveSectors, err := curPart.ActiveSectors() - if err != nil { - return nil, err - } - - // sectors that were previously fault and are now currently active are considered recovered. - recovered, err := bitfield.IntersectBitField(prevFaults, curActiveSectors) - if err != nil { - return nil, err - } - - // TODO: distinguish between "terminated" and "expired" sectors. The - // previous code here never had a chance of working in the first place, - // so I'm not going to try to replicate it right now. - // - // How? If the sector expires before it should (according to sector - // info) and it wasn't replaced by a pre-commit deleted in this change - // set, it was "early terminated". - - return &PartitionStatus{ - Terminated: bitfield.New(), - Expired: removedSectors, - Faulted: newFaults, - InRecovery: newRecoveries, - Recovered: recovered, - }, nil -} - -func (p *Processor) storeMinersActorInfoState(ctx context.Context, miners []minerActorInfo) error { - start := time.Now() - defer func() { - log.Debugw("Stored Miners Actor State", "duration", time.Since(start).String()) - }() - - tx, err := p.db.Begin() - if err != nil { - return err - } - - if _, err := tx.Exec(`create temp table mi (like miner_info excluding constraints) on commit drop;`); err != nil { - return xerrors.Errorf("prep temp: %w", err) - } - - stmt, err := tx.Prepare(`copy mi (miner_id, owner_addr, worker_addr, peer_id, sector_size) from STDIN`) - if err != nil { - return err - } - for _, m := range miners { - mi, err := p.node.StateMinerInfo(ctx, m.common.addr, m.common.tsKey) - if err != nil { - if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) { - continue - } else { - return err - } - } - var pid string - if mi.PeerId != nil { - pid = mi.PeerId.String() - } - if _, err := stmt.Exec( - m.common.addr.String(), - mi.Owner.String(), - mi.Worker.String(), - pid, - mi.SectorSize.ShortString(), - ); err != nil { - log.Errorw("failed to store miner state", "state", m.state, "info", m.state.Info, "error", err) - return xerrors.Errorf("failed to store miner state: %w", err) - } - - } - if err := stmt.Close(); err != nil { - return err - } - - if _, err := tx.Exec(`insert into miner_info select * from mi on conflict do nothing `); err != nil { - return xerrors.Errorf("actor put: %w", err) - } - - return tx.Commit() -} - -func (p *Processor) storePreCommitDealInfo(dealEvents <-chan *SectorDealEvent) error { - tx, err := p.db.Begin() - if err != nil { - return err - } - - if _, err := tx.Exec(`create temp table mds (like minerid_dealid_sectorid excluding constraints) on commit drop;`); err != nil { - return xerrors.Errorf("Failed to create temp table for minerid_dealid_sectorid: %w", err) - } - - stmt, err := tx.Prepare(`copy mds (deal_id, miner_id, sector_id) from STDIN`) - if err != nil { - return xerrors.Errorf("Failed to prepare minerid_dealid_sectorid statement: %w", err) - } - - for sde := range dealEvents { - for _, did := range sde.DealIDs { - if _, err := stmt.Exec( - uint64(did), - sde.MinerID.String(), - sde.SectorID, - ); err != nil { - return err - } - } - } - - if err := stmt.Close(); err != nil { - return xerrors.Errorf("Failed to close miner sector deals statement: %w", err) - } - - if _, err := tx.Exec(`insert into minerid_dealid_sectorid select * from mds on conflict do nothing`); err != nil { - return xerrors.Errorf("Failed to insert into miner deal sector table: %w", err) - } - - if err := tx.Commit(); err != nil { - return xerrors.Errorf("Failed to commit miner deal sector table: %w", err) - } - return nil - -} - -func (p *Processor) storeMinersPower(miners []minerActorInfo) error { - start := time.Now() - defer func() { - log.Debugw("Stored Miners Power", "duration", time.Since(start).String()) - }() - - tx, err := p.db.Begin() - if err != nil { - return xerrors.Errorf("begin miner_power tx: %w", err) - } - - if _, err := tx.Exec(`create temp table mp (like miner_power excluding constraints) on commit drop`); err != nil { - return xerrors.Errorf("prep miner_power temp: %w", err) - } - - stmt, err := tx.Prepare(`copy mp (miner_id, state_root, raw_bytes_power, quality_adjusted_power) from STDIN`) - if err != nil { - return xerrors.Errorf("prepare tmp miner_power: %w", err) - } - - for _, m := range miners { - if _, err := stmt.Exec( - m.common.addr.String(), - m.common.stateroot.String(), - m.rawPower.String(), - m.qalPower.String(), - ); err != nil { - log.Errorw("failed to store miner power", "miner", m.common.addr, "stateroot", m.common.stateroot, "error", err) - } - } - - if err := stmt.Close(); err != nil { - return xerrors.Errorf("close prepared miner_power: %w", err) - } - - if _, err := tx.Exec(`insert into miner_power select * from mp on conflict do nothing`); err != nil { - return xerrors.Errorf("insert miner_power from tmp: %w", err) - } - - if err := tx.Commit(); err != nil { - return xerrors.Errorf("commit miner_power tx: %w", err) - } - - return nil - -} - -// load the power actor state clam as an adt.Map at the tipset `ts`. -func getPowerActorState(ctx context.Context, api v0api.FullNode, ts types.TipSetKey) (power.State, error) { - powerActor, err := api.StateGetActor(ctx, power.Address, ts) - if err != nil { - return nil, err - } - return power.Load(cw_util.NewAPIIpldStore(ctx, api), powerActor) -} diff --git a/cmd/lotus-chainwatch/processor/mpool.go b/cmd/lotus-chainwatch/processor/mpool.go deleted file mode 100644 index 0a6445d78..000000000 --- a/cmd/lotus-chainwatch/processor/mpool.go +++ /dev/null @@ -1,100 +0,0 @@ -package processor - -import ( - "context" - "time" - - "golang.org/x/xerrors" - - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/types" -) - -func (p *Processor) subMpool(ctx context.Context) { - sub, err := p.node.MpoolSub(ctx) - if err != nil { - return - } - - for { - var updates []api.MpoolUpdate - - select { - case update := <-sub: - updates = append(updates, update) - case <-ctx.Done(): - return - } - - loop: - for { - select { - case update := <-sub: - updates = append(updates, update) - case <-time.After(10 * time.Millisecond): - break loop - } - } - - msgs := map[cid.Cid]*types.Message{} - for _, v := range updates { - if v.Type != api.MpoolAdd { - continue - } - - msgs[v.Message.Message.Cid()] = &v.Message.Message - } - - err := p.storeMessages(msgs) - if err != nil { - log.Error(err) - } - - if err := p.storeMpoolInclusions(updates); err != nil { - log.Error(err) - } - } -} - -func (p *Processor) storeMpoolInclusions(msgs []api.MpoolUpdate) error { - tx, err := p.db.Begin() - if err != nil { - return err - } - - if _, err := tx.Exec(` - create temp table mi (like mpool_messages excluding constraints) on commit drop; - `); err != nil { - return xerrors.Errorf("prep temp: %w", err) - } - - stmt, err := tx.Prepare(`copy mi (msg, add_ts) from stdin `) - if err != nil { - return err - } - - for _, msg := range msgs { - if msg.Type != api.MpoolAdd { - continue - } - - if _, err := stmt.Exec( - msg.Message.Message.Cid().String(), - time.Now().Unix(), - ); err != nil { - return err - } - } - - if err := stmt.Close(); err != nil { - return err - } - - if _, err := tx.Exec(`insert into mpool_messages select * from mi on conflict do nothing `); err != nil { - return xerrors.Errorf("actor put: %w", err) - } - - return tx.Commit() -} diff --git a/cmd/lotus-chainwatch/processor/power.go b/cmd/lotus-chainwatch/processor/power.go deleted file mode 100644 index 726a46706..000000000 --- a/cmd/lotus-chainwatch/processor/power.go +++ /dev/null @@ -1,190 +0,0 @@ -package processor - -import ( - "context" - "time" - - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-state-types/big" - - "github.com/filecoin-project/lotus/chain/actors/builtin" -) - -type powerActorInfo struct { - common actorInfo - - totalRawBytes big.Int - totalRawBytesCommitted big.Int - totalQualityAdjustedBytes big.Int - totalQualityAdjustedBytesCommitted big.Int - totalPledgeCollateral big.Int - - qaPowerSmoothed builtin.FilterEstimate - - minerCount int64 - minerCountAboveMinimumPower int64 -} - -func (p *Processor) setupPower() error { - tx, err := p.db.Begin() - if err != nil { - return err - } - - if _, err := tx.Exec(` -create table if not exists chain_power -( - state_root text not null - constraint power_smoothing_estimates_pk - primary key, - - total_raw_bytes_power text not null, - total_raw_bytes_committed text not null, - total_qa_bytes_power text not null, - total_qa_bytes_committed text not null, - total_pledge_collateral text not null, - - qa_smoothed_position_estimate text not null, - qa_smoothed_velocity_estimate text not null, - - miner_count int not null, - minimum_consensus_miner_count int not null -); -`); err != nil { - return err - } - - return tx.Commit() -} - -func (p *Processor) HandlePowerChanges(ctx context.Context, powerTips ActorTips) error { - powerChanges, err := p.processPowerActors(ctx, powerTips) - if err != nil { - return xerrors.Errorf("Failed to process power actors: %w", err) - } - - if err := p.persistPowerActors(ctx, powerChanges); err != nil { - return err - } - - return nil -} - -func (p *Processor) processPowerActors(ctx context.Context, powerTips ActorTips) ([]powerActorInfo, error) { - start := time.Now() - defer func() { - log.Debugw("Processed Power Actors", "duration", time.Since(start).String()) - }() - - var out []powerActorInfo - for tipset, powerStates := range powerTips { - for _, act := range powerStates { - var pw powerActorInfo - pw.common = act - - powerActorState, err := getPowerActorState(ctx, p.node, tipset) - if err != nil { - return nil, xerrors.Errorf("get power state (@ %s): %w", pw.common.stateroot.String(), err) - } - - totalPower, err := powerActorState.TotalPower() - if err != nil { - return nil, xerrors.Errorf("failed to compute total power: %w", err) - } - - totalCommitted, err := powerActorState.TotalCommitted() - if err != nil { - return nil, xerrors.Errorf("failed to compute total committed: %w", err) - } - - totalLocked, err := powerActorState.TotalLocked() - if err != nil { - return nil, xerrors.Errorf("failed to compute total locked: %w", err) - } - - powerSmoothed, err := powerActorState.TotalPowerSmoothed() - if err != nil { - return nil, xerrors.Errorf("failed to determine smoothed power: %w", err) - } - - // NOTE: this doesn't set new* fields. Previously, we - // filled these using ThisEpoch* fields from the actor - // state, but these fields are effectively internal - // state and don't represent "new" power, as was - // assumed. - - participatingMiners, totalMiners, err := powerActorState.MinerCounts() - if err != nil { - return nil, xerrors.Errorf("failed to count miners: %w", err) - } - - pw.totalRawBytes = totalPower.RawBytePower - pw.totalQualityAdjustedBytes = totalPower.QualityAdjPower - pw.totalRawBytesCommitted = totalCommitted.RawBytePower - pw.totalQualityAdjustedBytesCommitted = totalCommitted.QualityAdjPower - pw.totalPledgeCollateral = totalLocked - pw.qaPowerSmoothed = powerSmoothed - pw.minerCountAboveMinimumPower = int64(participatingMiners) - pw.minerCount = int64(totalMiners) - } - } - - return out, nil -} - -func (p *Processor) persistPowerActors(ctx context.Context, powerStates []powerActorInfo) error { - // NB: use errgroup when there is more than a single store operation - return p.storePowerSmoothingEstimates(powerStates) -} - -func (p *Processor) storePowerSmoothingEstimates(powerStates []powerActorInfo) error { - tx, err := p.db.Begin() - if err != nil { - return xerrors.Errorf("begin chain_power tx: %w", err) - } - - if _, err := tx.Exec(`create temp table cp (like chain_power) on commit drop`); err != nil { - return xerrors.Errorf("prep chain_power: %w", err) - } - - stmt, err := tx.Prepare(`copy cp (state_root, total_raw_bytes_power, total_raw_bytes_committed, total_qa_bytes_power, total_qa_bytes_committed, total_pledge_collateral, qa_smoothed_position_estimate, qa_smoothed_velocity_estimate, miner_count, minimum_consensus_miner_count) from stdin;`) - if err != nil { - return xerrors.Errorf("prepare tmp chain_power: %w", err) - } - - for _, ps := range powerStates { - if _, err := stmt.Exec( - ps.common.stateroot.String(), - - ps.totalRawBytes.String(), - ps.totalRawBytesCommitted.String(), - ps.totalQualityAdjustedBytes.String(), - ps.totalQualityAdjustedBytesCommitted.String(), - ps.totalPledgeCollateral.String(), - - ps.qaPowerSmoothed.PositionEstimate.String(), - ps.qaPowerSmoothed.VelocityEstimate.String(), - - ps.minerCount, - ps.minerCountAboveMinimumPower, - ); err != nil { - return xerrors.Errorf("failed to store smoothing estimate: %w", err) - } - } - - if err := stmt.Close(); err != nil { - return xerrors.Errorf("close prepared chain_power: %w", err) - } - - if _, err := tx.Exec(`insert into chain_power select * from cp on conflict do nothing`); err != nil { - return xerrors.Errorf("insert chain_power from tmp: %w", err) - } - - if err := tx.Commit(); err != nil { - return xerrors.Errorf("commit chain_power tx: %w", err) - } - - return nil - -} diff --git a/cmd/lotus-chainwatch/processor/processor.go b/cmd/lotus-chainwatch/processor/processor.go deleted file mode 100644 index af5935d47..000000000 --- a/cmd/lotus-chainwatch/processor/processor.go +++ /dev/null @@ -1,420 +0,0 @@ -package processor - -import ( - "context" - "database/sql" - "encoding/json" - "math" - "sync" - "time" - - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log/v2" - - "github.com/filecoin-project/go-state-types/abi" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" - - "github.com/filecoin-project/lotus/api/v0api" - "github.com/filecoin-project/lotus/chain/types" - cw_util "github.com/filecoin-project/lotus/cmd/lotus-chainwatch/util" - "github.com/filecoin-project/lotus/lib/parmap" -) - -var log = logging.Logger("processor") - -type Processor struct { - db *sql.DB - - node v0api.FullNode - ctxStore *cw_util.APIIpldStore - - genesisTs *types.TipSet - - // number of blocks processed at a time - batch int -} - -type ActorTips map[types.TipSetKey][]actorInfo - -type actorInfo struct { - act types.Actor - - stateroot cid.Cid - height abi.ChainEpoch // so that we can walk the actor changes in chronological order. - - tsKey types.TipSetKey - parentTsKey types.TipSetKey - - addr address.Address - state string -} - -func NewProcessor(ctx context.Context, db *sql.DB, node v0api.FullNode, batch int) *Processor { - ctxStore := cw_util.NewAPIIpldStore(ctx, node) - return &Processor{ - db: db, - ctxStore: ctxStore, - node: node, - batch: batch, - } -} - -func (p *Processor) setupSchemas() error { - // maintain order, subsequent calls create tables with foreign keys. - if err := p.setupMiners(); err != nil { - return err - } - - if err := p.setupMarket(); err != nil { - return err - } - - if err := p.setupRewards(); err != nil { - return err - } - - if err := p.setupMessages(); err != nil { - return err - } - - if err := p.setupCommonActors(); err != nil { - return err - } - - if err := p.setupPower(); err != nil { - return err - } - - return nil -} - -func (p *Processor) Start(ctx context.Context) { - log.Debug("Starting Processor") - - if err := p.setupSchemas(); err != nil { - log.Fatalw("Failed to setup processor", "error", err) - } - - var err error - p.genesisTs, err = p.node.ChainGetGenesis(ctx) - if err != nil { - log.Fatalw("Failed to get genesis state from lotus", "error", err.Error()) - } - - go p.subMpool(ctx) - - // main processor loop - go func() { - for { - select { - case <-ctx.Done(): - log.Info("Stopping Processor...") - return - default: - loopStart := time.Now() - toProcess, err := p.unprocessedBlocks(ctx, p.batch) - if err != nil { - log.Fatalw("Failed to get unprocessed blocks", "error", err) - } - - if len(toProcess) == 0 { - log.Info("No unprocessed blocks. Wait then try again...") - time.Sleep(time.Second * 30) - continue - } - - // TODO special case genesis state handling here to avoid all the special cases that will be needed for it else where - // before doing "normal" processing. - - actorChanges, nullRounds, err := p.collectActorChanges(ctx, toProcess) - if err != nil { - log.Fatalw("Failed to collect actor changes", "error", err) - } - log.Infow("Collected Actor Changes", - "MarketChanges", len(actorChanges[builtin2.StorageMarketActorCodeID]), - "MinerChanges", len(actorChanges[builtin2.StorageMinerActorCodeID]), - "RewardChanges", len(actorChanges[builtin2.RewardActorCodeID]), - "AccountChanges", len(actorChanges[builtin2.AccountActorCodeID]), - "nullRounds", len(nullRounds)) - - grp := sync.WaitGroup{} - - grp.Add(1) - go func() { - defer grp.Done() - if err := p.HandleMarketChanges(ctx, actorChanges[builtin2.StorageMarketActorCodeID]); err != nil { - log.Errorf("Failed to handle market changes: %v", err) - return - } - }() - - grp.Add(1) - go func() { - defer grp.Done() - if err := p.HandleMinerChanges(ctx, actorChanges[builtin2.StorageMinerActorCodeID]); err != nil { - log.Errorf("Failed to handle miner changes: %v", err) - return - } - }() - - grp.Add(1) - go func() { - defer grp.Done() - if err := p.HandleRewardChanges(ctx, actorChanges[builtin2.RewardActorCodeID], nullRounds); err != nil { - log.Errorf("Failed to handle reward changes: %v", err) - return - } - }() - - grp.Add(1) - go func() { - defer grp.Done() - if err := p.HandlePowerChanges(ctx, actorChanges[builtin2.StoragePowerActorCodeID]); err != nil { - log.Errorf("Failed to handle power actor changes: %v", err) - return - } - }() - - grp.Add(1) - go func() { - defer grp.Done() - if err := p.HandleMessageChanges(ctx, toProcess); err != nil { - log.Errorf("Failed to handle message changes: %v", err) - return - } - }() - - grp.Add(1) - go func() { - defer grp.Done() - if err := p.HandleCommonActorsChanges(ctx, actorChanges); err != nil { - log.Errorf("Failed to handle common actor changes: %v", err) - return - } - }() - - grp.Wait() - - if err := p.markBlocksProcessed(ctx, toProcess); err != nil { - log.Fatalw("Failed to mark blocks as processed", "error", err) - } - - if err := p.refreshViews(); err != nil { - log.Errorw("Failed to refresh views", "error", err) - } - log.Infow("Processed Batch Complete", "duration", time.Since(loopStart).String()) - } - } - }() - -} - -func (p *Processor) refreshViews() error { - if _, err := p.db.Exec(`refresh materialized view state_heights`); err != nil { - return err - } - - return nil -} - -func (p *Processor) collectActorChanges(ctx context.Context, toProcess map[cid.Cid]*types.BlockHeader) (map[cid.Cid]ActorTips, []types.TipSetKey, error) { - start := time.Now() - defer func() { - log.Debugw("Collected Actor Changes", "duration", time.Since(start).String()) - }() - // ActorCode - > tipset->[]actorInfo - out := map[cid.Cid]ActorTips{} - var outMu sync.Mutex - - // map of addresses to changed actors - var changes map[string]types.Actor - actorsSeen := map[cid.Cid]struct{}{} - - var nullRounds []types.TipSetKey - var nullBlkMu sync.Mutex - - // collect all actor state that has changes between block headers - paDone := 0 - parmap.Par(50, parmap.MapArr(toProcess), func(bh *types.BlockHeader) { - paDone++ - if paDone%100 == 0 { - log.Debugw("Collecting actor changes", "done", paDone, "percent", (paDone*100)/len(toProcess)) - } - - pts, err := p.node.ChainGetTipSet(ctx, types.NewTipSetKey(bh.Parents...)) - if err != nil { - log.Error(err) - return - } - - if pts.ParentState().Equals(bh.ParentStateRoot) { - nullBlkMu.Lock() - nullRounds = append(nullRounds, pts.Key()) - nullBlkMu.Unlock() - } - - // collect all actors that had state changes between the blockheader parent-state and its grandparent-state. - // TODO: changes will contain deleted actors, this causes needless processing further down the pipeline, consider - // a separate strategy for deleted actors - changes, err = p.node.StateChangedActors(ctx, pts.ParentState(), bh.ParentStateRoot) - if err != nil { - log.Error(err) - log.Debugw("StateChangedActors", "grandparent_state", pts.ParentState(), "parent_state", bh.ParentStateRoot) - return - } - - // record the state of all actors that have changed - for a, act := range changes { - act := act - a := a - - // ignore actors that were deleted. - has, err := p.node.ChainHasObj(ctx, act.Head) - if err != nil { - log.Error(err) - log.Debugw("ChanHasObj", "actor_head", act.Head) - return - } - if !has { - continue - } - - addr, err := address.NewFromString(a) - if err != nil { - log.Error(err) - log.Debugw("NewFromString", "address_string", a) - return - } - - ast, err := p.node.StateReadState(ctx, addr, pts.Key()) - if err != nil { - log.Error(err) - log.Debugw("StateReadState", "address_string", a, "parent_tipset_key", pts.Key()) - return - } - - // TODO look here for an empty state, maybe thats a sign the actor was deleted? - - state, err := json.Marshal(ast.State) - if err != nil { - log.Error(err) - return - } - - outMu.Lock() - if _, ok := actorsSeen[act.Head]; !ok { - _, ok := out[act.Code] - if !ok { - out[act.Code] = map[types.TipSetKey][]actorInfo{} - } - out[act.Code][pts.Key()] = append(out[act.Code][pts.Key()], actorInfo{ - act: act, - stateroot: bh.ParentStateRoot, - height: bh.Height, - tsKey: pts.Key(), - parentTsKey: pts.Parents(), - addr: addr, - state: string(state), - }) - } - actorsSeen[act.Head] = struct{}{} - outMu.Unlock() - } - }) - return out, nullRounds, nil -} - -func (p *Processor) unprocessedBlocks(ctx context.Context, batch int) (map[cid.Cid]*types.BlockHeader, error) { - start := time.Now() - defer func() { - log.Debugw("Gathered Blocks to process", "duration", time.Since(start).String()) - }() - rows, err := p.db.Query(` -with toProcess as ( - select b.cid, b.height, rank() over (order by height) as rnk - from blocks_synced bs - left join blocks b on bs.cid = b.cid - where bs.processed_at is null and b.height > 0 -) -select cid -from toProcess -where rnk <= $1 -`, batch) - if err != nil { - return nil, xerrors.Errorf("Failed to query for unprocessed blocks: %w", err) - } - out := map[cid.Cid]*types.BlockHeader{} - - minBlock := abi.ChainEpoch(math.MaxInt64) - maxBlock := abi.ChainEpoch(0) - // TODO consider parallel execution here for getting the blocks from the api as is done in fetchMessages() - for rows.Next() { - if rows.Err() != nil { - return nil, err - } - var c string - if err := rows.Scan(&c); err != nil { - log.Errorf("Failed to scan unprocessed blocks: %s", err.Error()) - continue - } - ci, err := cid.Parse(c) - if err != nil { - log.Errorf("Failed to parse unprocessed blocks: %s", err.Error()) - continue - } - bh, err := p.node.ChainGetBlock(ctx, ci) - if err != nil { - // this is a pretty serious issue. - log.Errorf("Failed to get block header %s: %s", ci.String(), err.Error()) - continue - } - out[ci] = bh - if bh.Height < minBlock { - minBlock = bh.Height - } - if bh.Height > maxBlock { - maxBlock = bh.Height - } - } - if minBlock <= maxBlock { - log.Infow("Gathered Blocks to Process", "start", minBlock, "end", maxBlock) - } - return out, rows.Close() -} - -func (p *Processor) markBlocksProcessed(ctx context.Context, processed map[cid.Cid]*types.BlockHeader) error { - start := time.Now() - processedHeight := abi.ChainEpoch(0) - defer func() { - log.Debugw("Marked blocks as Processed", "duration", time.Since(start).String()) - log.Infow("Processed Blocks", "height", processedHeight) - }() - tx, err := p.db.Begin() - if err != nil { - return err - } - - processedAt := time.Now().Unix() - stmt, err := tx.Prepare(`update blocks_synced set processed_at=$1 where cid=$2`) - if err != nil { - return err - } - - for c, bh := range processed { - if bh.Height > processedHeight { - processedHeight = bh.Height - } - if _, err := stmt.Exec(processedAt, c.String()); err != nil { - return err - } - } - - if err := stmt.Close(); err != nil { - return err - } - - return tx.Commit() -} diff --git a/cmd/lotus-chainwatch/processor/reward.go b/cmd/lotus-chainwatch/processor/reward.go deleted file mode 100644 index 72a329c87..000000000 --- a/cmd/lotus-chainwatch/processor/reward.go +++ /dev/null @@ -1,234 +0,0 @@ -package processor - -import ( - "context" - "time" - - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" - - "github.com/filecoin-project/lotus/chain/actors/builtin" - "github.com/filecoin-project/lotus/chain/actors/builtin/reward" - "github.com/filecoin-project/lotus/chain/types" - - cw_util "github.com/filecoin-project/lotus/cmd/lotus-chainwatch/util" -) - -type rewardActorInfo struct { - common actorInfo - - cumSumBaselinePower big.Int - cumSumRealizedPower big.Int - - effectiveNetworkTime abi.ChainEpoch - effectiveBaselinePower big.Int - - // NOTE: These variables are wrong. Talk to @ZX about fixing. These _do - // not_ represent "new" anything. - newBaselinePower big.Int - newBaseReward big.Int - newSmoothingEstimate builtin.FilterEstimate - - totalMinedReward big.Int -} - -func (rw *rewardActorInfo) set(s reward.State) (err error) { - rw.cumSumBaselinePower, err = s.CumsumBaseline() - if err != nil { - return xerrors.Errorf("getting cumsum baseline power (@ %s): %w", rw.common.stateroot.String(), err) - } - - rw.cumSumRealizedPower, err = s.CumsumRealized() - if err != nil { - return xerrors.Errorf("getting cumsum realized power (@ %s): %w", rw.common.stateroot.String(), err) - } - - rw.effectiveNetworkTime, err = s.EffectiveNetworkTime() - if err != nil { - return xerrors.Errorf("getting effective network time (@ %s): %w", rw.common.stateroot.String(), err) - } - - rw.effectiveBaselinePower, err = s.EffectiveBaselinePower() - if err != nil { - return xerrors.Errorf("getting effective baseline power (@ %s): %w", rw.common.stateroot.String(), err) - } - - rw.totalMinedReward, err = s.TotalStoragePowerReward() - if err != nil { - return xerrors.Errorf("getting total mined (@ %s): %w", rw.common.stateroot.String(), err) - } - - rw.newBaselinePower, err = s.ThisEpochBaselinePower() - if err != nil { - return xerrors.Errorf("getting this epoch baseline power (@ %s): %w", rw.common.stateroot.String(), err) - } - - rw.newBaseReward, err = s.ThisEpochReward() - if err != nil { - return xerrors.Errorf("getting this epoch baseline power (@ %s): %w", rw.common.stateroot.String(), err) - } - - rw.newSmoothingEstimate, err = s.ThisEpochRewardSmoothed() - if err != nil { - return xerrors.Errorf("getting this epoch baseline power (@ %s): %w", rw.common.stateroot.String(), err) - } - return nil -} - -func (p *Processor) setupRewards() error { - tx, err := p.db.Begin() - if err != nil { - return err - } - - if _, err := tx.Exec(` -/* captures chain-specific power state for any given stateroot */ -create table if not exists chain_reward -( - state_root text not null - constraint chain_reward_pk - primary key, - cum_sum_baseline text not null, - cum_sum_realized text not null, - effective_network_time int not null, - effective_baseline_power text not null, - - new_baseline_power text not null, - new_reward numeric not null, - new_reward_smoothed_position_estimate text not null, - new_reward_smoothed_velocity_estimate text not null, - - total_mined_reward text not null -); -`); err != nil { - return err - } - - return tx.Commit() -} - -func (p *Processor) HandleRewardChanges(ctx context.Context, rewardTips ActorTips, nullRounds []types.TipSetKey) error { - rewardChanges, err := p.processRewardActors(ctx, rewardTips, nullRounds) - if err != nil { - return xerrors.Errorf("Failed to process reward actors: %w", err) - } - - if err := p.persistRewardActors(ctx, rewardChanges); err != nil { - return err - } - - return nil -} - -func (p *Processor) processRewardActors(ctx context.Context, rewardTips ActorTips, nullRounds []types.TipSetKey) ([]rewardActorInfo, error) { - start := time.Now() - defer func() { - log.Debugw("Processed Reward Actors", "duration", time.Since(start).String()) - }() - - var out []rewardActorInfo - for tipset, rewards := range rewardTips { - for _, act := range rewards { - var rw rewardActorInfo - rw.common = act - - // get reward actor states at each tipset once for all updates - rewardActor, err := p.node.StateGetActor(ctx, reward.Address, tipset) - if err != nil { - return nil, xerrors.Errorf("get reward state (@ %s): %w", rw.common.stateroot.String(), err) - } - - rewardActorState, err := reward.Load(cw_util.NewAPIIpldStore(ctx, p.node), rewardActor) - if err != nil { - return nil, xerrors.Errorf("read state obj (@ %s): %w", rw.common.stateroot.String(), err) - } - if err := rw.set(rewardActorState); err != nil { - return nil, err - } - - out = append(out, rw) - } - } - for _, tsKey := range nullRounds { - var rw rewardActorInfo - tipset, err := p.node.ChainGetTipSet(ctx, tsKey) - if err != nil { - return nil, err - } - rw.common.tsKey = tipset.Key() - rw.common.height = tipset.Height() - rw.common.stateroot = tipset.ParentState() - rw.common.parentTsKey = tipset.Parents() - // get reward actor states at each tipset once for all updates - rewardActor, err := p.node.StateGetActor(ctx, reward.Address, tsKey) - if err != nil { - return nil, err - } - - rewardActorState, err := reward.Load(cw_util.NewAPIIpldStore(ctx, p.node), rewardActor) - if err != nil { - return nil, xerrors.Errorf("read state obj (@ %s): %w", rw.common.stateroot.String(), err) - } - - if err := rw.set(rewardActorState); err != nil { - return nil, err - } - out = append(out, rw) - } - - return out, nil -} - -func (p *Processor) persistRewardActors(ctx context.Context, rewards []rewardActorInfo) error { - start := time.Now() - defer func() { - log.Debugw("Persisted Reward Actors", "duration", time.Since(start).String()) - }() - - tx, err := p.db.Begin() - if err != nil { - return xerrors.Errorf("begin chain_reward tx: %w", err) - } - - if _, err := tx.Exec(`create temp table cr (like chain_reward excluding constraints) on commit drop`); err != nil { - return xerrors.Errorf("prep chain_reward temp: %w", err) - } - - stmt, err := tx.Prepare(`copy cr ( state_root, cum_sum_baseline, cum_sum_realized, effective_network_time, effective_baseline_power, new_baseline_power, new_reward, new_reward_smoothed_position_estimate, new_reward_smoothed_velocity_estimate, total_mined_reward) from STDIN`) - if err != nil { - return xerrors.Errorf("prepare tmp chain_reward: %w", err) - } - - for _, rewardState := range rewards { - if _, err := stmt.Exec( - rewardState.common.stateroot.String(), - rewardState.cumSumBaselinePower.String(), - rewardState.cumSumRealizedPower.String(), - uint64(rewardState.effectiveNetworkTime), - rewardState.effectiveBaselinePower.String(), - rewardState.newBaselinePower.String(), - rewardState.newBaseReward.String(), - rewardState.newSmoothingEstimate.PositionEstimate.String(), - rewardState.newSmoothingEstimate.VelocityEstimate.String(), - rewardState.totalMinedReward.String(), - ); err != nil { - log.Errorw("failed to store chain power", "state_root", rewardState.common.stateroot, "error", err) - } - } - - if err := stmt.Close(); err != nil { - return xerrors.Errorf("close prepared chain_reward: %w", err) - } - - if _, err := tx.Exec(`insert into chain_reward select * from cr on conflict do nothing`); err != nil { - return xerrors.Errorf("insert chain_reward from tmp: %w", err) - } - - if err := tx.Commit(); err != nil { - return xerrors.Errorf("commit chain_reward tx: %w", err) - } - - return nil -} diff --git a/cmd/lotus-chainwatch/run.go b/cmd/lotus-chainwatch/run.go deleted file mode 100644 index 6e47a100d..000000000 --- a/cmd/lotus-chainwatch/run.go +++ /dev/null @@ -1,107 +0,0 @@ -package main - -import ( - "database/sql" - "fmt" - "net/http" - _ "net/http/pprof" - "os" - "strings" - - "github.com/filecoin-project/lotus/api/v0api" - - _ "github.com/lib/pq" - - "github.com/filecoin-project/go-jsonrpc" - logging "github.com/ipfs/go-log/v2" - "github.com/urfave/cli/v2" - "golang.org/x/xerrors" - - lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/cmd/lotus-chainwatch/processor" - "github.com/filecoin-project/lotus/cmd/lotus-chainwatch/scheduler" - "github.com/filecoin-project/lotus/cmd/lotus-chainwatch/syncer" - "github.com/filecoin-project/lotus/cmd/lotus-chainwatch/util" -) - -var runCmd = &cli.Command{ - Name: "run", - Usage: "Start lotus chainwatch", - Flags: []cli.Flag{ - &cli.IntFlag{ - Name: "max-batch", - Value: 50, - }, - }, - Action: func(cctx *cli.Context) error { - go func() { - http.ListenAndServe(":6060", nil) //nolint:errcheck - }() - ll := cctx.String("log-level") - if err := logging.SetLogLevel("*", ll); err != nil { - return err - } - if err := logging.SetLogLevel("rpc", "error"); err != nil { - return err - } - - var api v0api.FullNode - var closer jsonrpc.ClientCloser - var err error - if tokenMaddr := cctx.String("api"); tokenMaddr != "" { - toks := strings.Split(tokenMaddr, ":") - if len(toks) != 2 { - return fmt.Errorf("invalid api tokens, expected :, got: %s", tokenMaddr) - } - - api, closer, err = util.GetFullNodeAPIUsingCredentials(cctx.Context, toks[1], toks[0]) - if err != nil { - return err - } - } else { - api, closer, err = lcli.GetFullNodeAPI(cctx) - if err != nil { - return err - } - } - defer closer() - ctx := lcli.ReqContext(cctx) - - v, err := api.Version(ctx) - if err != nil { - return err - } - - log.Infof("Remote version: %s", v.Version) - - maxBatch := cctx.Int("max-batch") - - db, err := sql.Open("postgres", cctx.String("db")) - if err != nil { - return err - } - defer func() { - if err := db.Close(); err != nil { - log.Errorw("Failed to close database", "error", err) - } - }() - - if err := db.Ping(); err != nil { - return xerrors.Errorf("Database failed to respond to ping (is it online?): %w", err) - } - db.SetMaxOpenConns(1350) - - sync := syncer.NewSyncer(db, api, 1400) - sync.Start(ctx) - - proc := processor.NewProcessor(ctx, db, api, maxBatch) - proc.Start(ctx) - - sched := scheduler.PrepareScheduler(db) - sched.Start(ctx) - - <-ctx.Done() - os.Exit(0) - return nil - }, -} diff --git a/cmd/lotus-chainwatch/scheduler/refresh_top_miners_by_base_reward.go b/cmd/lotus-chainwatch/scheduler/refresh_top_miners_by_base_reward.go deleted file mode 100644 index 145e84229..000000000 --- a/cmd/lotus-chainwatch/scheduler/refresh_top_miners_by_base_reward.go +++ /dev/null @@ -1,78 +0,0 @@ -package scheduler - -import ( - "context" - "database/sql" - - "golang.org/x/xerrors" -) - -func setupTopMinerByBaseRewardSchema(ctx context.Context, db *sql.DB) error { - select { - case <-ctx.Done(): - return nil - default: - } - - tx, err := db.Begin() - if err != nil { - return err - } - if _, err := tx.Exec(` - create materialized view if not exists top_miners_by_base_reward as - with total_rewards_by_miner as ( - select - b.miner, - sum(cr.new_reward * b.win_count) as total_reward - from blocks b - inner join chain_reward cr on b.parentstateroot = cr.state_root - group by 1 - ) select - rank() over (order by total_reward desc), - miner, - total_reward - from total_rewards_by_miner - group by 2, 3; - - create index if not exists top_miners_by_base_reward_miner_index - on top_miners_by_base_reward (miner); - - create materialized view if not exists top_miners_by_base_reward_max_height as - select - b."timestamp"as current_timestamp, - max(b.height) as current_height - from blocks b - join chain_reward cr on b.parentstateroot = cr.state_root - where cr.new_reward is not null - group by 1 - order by 1 desc - limit 1; - `); err != nil { - return xerrors.Errorf("create top_miners_by_base_reward views: %w", err) - } - - if err := tx.Commit(); err != nil { - return xerrors.Errorf("committing top_miners_by_base_reward views; %w", err) - } - return nil -} - -func refreshTopMinerByBaseReward(ctx context.Context, db *sql.DB) error { - select { - case <-ctx.Done(): - return nil - default: - } - - _, err := db.Exec("refresh materialized view top_miners_by_base_reward;") - if err != nil { - return xerrors.Errorf("refresh top_miners_by_base_reward: %w", err) - } - - _, err = db.Exec("refresh materialized view top_miners_by_base_reward_max_height;") - if err != nil { - return xerrors.Errorf("refresh top_miners_by_base_reward_max_height: %w", err) - } - - return nil -} diff --git a/cmd/lotus-chainwatch/scheduler/scheduler.go b/cmd/lotus-chainwatch/scheduler/scheduler.go deleted file mode 100644 index 6782bc16d..000000000 --- a/cmd/lotus-chainwatch/scheduler/scheduler.go +++ /dev/null @@ -1,60 +0,0 @@ -package scheduler - -import ( - "context" - "database/sql" - "time" - - logging "github.com/ipfs/go-log/v2" - - "golang.org/x/xerrors" -) - -var log = logging.Logger("scheduler") - -// Scheduler manages the execution of jobs triggered -// by tickers. Not externally configurable at runtime. -type Scheduler struct { - db *sql.DB -} - -// PrepareScheduler returns a ready-to-run Scheduler -func PrepareScheduler(db *sql.DB) *Scheduler { - return &Scheduler{db} -} - -func (s *Scheduler) setupSchema(ctx context.Context) error { - if err := setupTopMinerByBaseRewardSchema(ctx, s.db); err != nil { - return xerrors.Errorf("setup top miners by reward schema: %w", err) - } - return nil -} - -// Start the scheduler jobs at the defined intervals -func (s *Scheduler) Start(ctx context.Context) { - log.Debug("Starting Scheduler") - - if err := s.setupSchema(ctx); err != nil { - log.Fatalw("applying scheduling schema", "error", err) - } - - go func() { - // run once on start after schema has initialized - time.Sleep(1 * time.Minute) - if err := refreshTopMinerByBaseReward(ctx, s.db); err != nil { - log.Errorw("failed to refresh top miner", "error", err) - } - refreshTopMinerCh := time.NewTicker(30 * time.Second) - defer refreshTopMinerCh.Stop() - for { - select { - case <-refreshTopMinerCh.C: - if err := refreshTopMinerByBaseReward(ctx, s.db); err != nil { - log.Errorw("failed to refresh top miner", "error", err) - } - case <-ctx.Done(): - return - } - } - }() -} diff --git a/cmd/lotus-chainwatch/syncer/blockssub.go b/cmd/lotus-chainwatch/syncer/blockssub.go deleted file mode 100644 index ea9c079e8..000000000 --- a/cmd/lotus-chainwatch/syncer/blockssub.go +++ /dev/null @@ -1,27 +0,0 @@ -package syncer - -import ( - "context" - "time" - - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" -) - -func (s *Syncer) subBlocks(ctx context.Context) { - sub, err := s.node.SyncIncomingBlocks(ctx) - if err != nil { - log.Errorf("opening incoming block channel: %+v", err) - return - } - - log.Infow("Capturing incoming blocks") - for bh := range sub { - err := s.storeHeaders(map[cid.Cid]*types.BlockHeader{ - bh.Cid(): bh, - }, false, time.Now()) - if err != nil { - log.Errorf("storing incoming block header: %+v", err) - } - } -} diff --git a/cmd/lotus-chainwatch/syncer/sync.go b/cmd/lotus-chainwatch/syncer/sync.go deleted file mode 100644 index b5e9c73d6..000000000 --- a/cmd/lotus-chainwatch/syncer/sync.go +++ /dev/null @@ -1,527 +0,0 @@ -package syncer - -import ( - "container/list" - "context" - "database/sql" - "fmt" - "sync" - "time" - - "golang.org/x/xerrors" - - "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log/v2" - - "github.com/filecoin-project/lotus/api/v0api" - "github.com/filecoin-project/lotus/chain/store" - "github.com/filecoin-project/lotus/chain/types" -) - -var log = logging.Logger("syncer") - -type Syncer struct { - db *sql.DB - - lookbackLimit uint64 - - headerLk sync.Mutex - node v0api.FullNode -} - -func NewSyncer(db *sql.DB, node v0api.FullNode, lookbackLimit uint64) *Syncer { - return &Syncer{ - db: db, - node: node, - lookbackLimit: lookbackLimit, - } -} - -func (s *Syncer) setupSchemas() error { - tx, err := s.db.Begin() - if err != nil { - return err - } - - if _, err := tx.Exec(` -/* tracks circulating fil available on the network at each tipset */ -create table if not exists chain_economics -( - parent_state_root text not null - constraint chain_economics_pk primary key, - circulating_fil text not null, - vested_fil text not null, - mined_fil text not null, - burnt_fil text not null, - locked_fil text not null -); - -create table if not exists block_cids -( - cid text not null - constraint block_cids_pk - primary key -); - -create unique index if not exists block_cids_cid_uindex - on block_cids (cid); - -create table if not exists blocks_synced -( - cid text not null - constraint blocks_synced_pk - primary key - constraint blocks_block_cids_cid_fk - references block_cids (cid), - synced_at int not null, - processed_at int -); - -create unique index if not exists blocks_synced_cid_uindex - on blocks_synced (cid,processed_at); - -create table if not exists block_parents -( - block text not null - constraint blocks_block_cids_cid_fk - references block_cids (cid), - parent text not null -); - -create unique index if not exists block_parents_block_parent_uindex - on block_parents (block, parent); - -create table if not exists drand_entries -( - round bigint not null - constraint drand_entries_pk - primary key, - data bytea not null -); -create unique index if not exists drand_entries_round_uindex - on drand_entries (round); - -create table if not exists block_drand_entries -( - round bigint not null - constraint block_drand_entries_drand_entries_round_fk - references drand_entries (round), - block text not null - constraint blocks_block_cids_cid_fk - references block_cids (cid) -); -create unique index if not exists block_drand_entries_round_uindex - on block_drand_entries (round, block); - -create table if not exists blocks -( - cid text not null - constraint blocks_pk - primary key - constraint blocks_block_cids_cid_fk - references block_cids (cid), - parentWeight numeric not null, - parentStateRoot text not null, - height bigint not null, - miner text not null, - timestamp bigint not null, - ticket bytea not null, - election_proof bytea, - win_count bigint, - parent_base_fee text not null, - forksig bigint not null -); - -create unique index if not exists block_cid_uindex - on blocks (cid,height); - -create materialized view if not exists state_heights - as select min(b.height) height, b.parentstateroot - from blocks b group by b.parentstateroot; - -create index if not exists state_heights_height_index - on state_heights (height); - -create index if not exists state_heights_parentstateroot_index - on state_heights (parentstateroot); -`); err != nil { - return err - } - - return tx.Commit() -} - -func (s *Syncer) Start(ctx context.Context) { - if err := logging.SetLogLevel("syncer", "info"); err != nil { - log.Fatal(err) - } - log.Debug("Starting Syncer") - - if err := s.setupSchemas(); err != nil { - log.Fatal(err) - } - - // capture all reported blocks - go s.subBlocks(ctx) - - // we need to ensure that on a restart we don't reprocess the whole flarping chain - var sinceEpoch uint64 - blkCID, height, err := s.mostRecentlySyncedBlockHeight() - if err != nil { - log.Fatalw("failed to find most recently synced block", "error", err) - } else { - if height > 0 { - log.Infow("Found starting point for syncing", "blockCID", blkCID.String(), "height", height) - sinceEpoch = uint64(height) - } - } - - // continue to keep the block headers table up to date. - notifs, err := s.node.ChainNotify(ctx) - if err != nil { - log.Fatal(err) - } - - go func() { - for notif := range notifs { - for _, change := range notif { - switch change.Type { - case store.HCCurrent: - // This case is important for capturing the initial state of a node - // which might be on a dead network with no new blocks being produced. - // It also allows a fresh Chainwatch instance to start walking the - // chain without waiting for a new block to come along. - fallthrough - case store.HCApply: - unsynced, err := s.unsyncedBlocks(ctx, change.Val, sinceEpoch) - if err != nil { - log.Errorw("failed to gather unsynced blocks", "error", err) - } - - if err := s.storeCirculatingSupply(ctx, change.Val); err != nil { - log.Errorw("failed to store circulating supply", "error", err) - } - - if len(unsynced) == 0 { - continue - } - - if err := s.storeHeaders(unsynced, true, time.Now()); err != nil { - // so this is pretty bad, need some kind of retry.. - // for now just log an error and the blocks will be attempted again on next notifi - log.Errorw("failed to store unsynced blocks", "error", err) - } - - sinceEpoch = uint64(change.Val.Height()) - case store.HCRevert: - log.Debug("revert todo") - } - } - } - }() -} - -func (s *Syncer) unsyncedBlocks(ctx context.Context, head *types.TipSet, since uint64) (map[cid.Cid]*types.BlockHeader, error) { - hasList, err := s.syncedBlocks(since, s.lookbackLimit) - if err != nil { - return nil, err - } - - // build a list of blocks that we have not synced. - toVisit := list.New() - for _, header := range head.Blocks() { - toVisit.PushBack(header) - } - - toSync := map[cid.Cid]*types.BlockHeader{} - - for toVisit.Len() > 0 { - bh := toVisit.Remove(toVisit.Back()).(*types.BlockHeader) - _, has := hasList[bh.Cid()] - if _, seen := toSync[bh.Cid()]; seen || has { - continue - } - - toSync[bh.Cid()] = bh - if len(toSync)%500 == 10 { - log.Debugw("To visit", "toVisit", toVisit.Len(), "toSync", len(toSync), "current_height", bh.Height) - } - - if bh.Height == 0 { - continue - } - - pts, err := s.node.ChainGetTipSet(ctx, types.NewTipSetKey(bh.Parents...)) - if err != nil { - log.Error(err) - continue - } - - for _, header := range pts.Blocks() { - toVisit.PushBack(header) - } - } - log.Debugw("Gathered unsynced blocks", "count", len(toSync)) - return toSync, nil -} - -func (s *Syncer) syncedBlocks(since, limit uint64) (map[cid.Cid]struct{}, error) { - rws, err := s.db.Query(`select bs.cid FROM blocks_synced bs left join blocks b on b.cid = bs.cid where b.height <= $1 and bs.processed_at is not null limit $2`, since, limit) - if err != nil { - return nil, xerrors.Errorf("Failed to query blocks_synced: %w", err) - } - out := map[cid.Cid]struct{}{} - - for rws.Next() { - var c string - if err := rws.Scan(&c); err != nil { - return nil, xerrors.Errorf("Failed to scan blocks_synced: %w", err) - } - - ci, err := cid.Parse(c) - if err != nil { - return nil, xerrors.Errorf("Failed to parse blocks_synced: %w", err) - } - - out[ci] = struct{}{} - } - return out, nil -} - -func (s *Syncer) mostRecentlySyncedBlockHeight() (cid.Cid, int64, error) { - rw := s.db.QueryRow(` -select blocks_synced.cid, b.height -from blocks_synced -left join blocks b on blocks_synced.cid = b.cid -where processed_at is not null -order by height desc -limit 1 -`) - - var c string - var h int64 - if err := rw.Scan(&c, &h); err != nil { - if err == sql.ErrNoRows { - return cid.Undef, 0, nil - } - return cid.Undef, -1, err - } - - ci, err := cid.Parse(c) - if err != nil { - return cid.Undef, -1, err - } - - return ci, h, nil -} - -func (s *Syncer) storeCirculatingSupply(ctx context.Context, tipset *types.TipSet) error { - supply, err := s.node.StateVMCirculatingSupplyInternal(ctx, tipset.Key()) - if err != nil { - return err - } - - ceInsert := `insert into chain_economics (parent_state_root, circulating_fil, vested_fil, mined_fil, burnt_fil, locked_fil) ` + - `values ('%s', '%s', '%s', '%s', '%s', '%s') on conflict on constraint chain_economics_pk do ` + - `update set (circulating_fil, vested_fil, mined_fil, burnt_fil, locked_fil) = ('%[2]s', '%[3]s', '%[4]s', '%[5]s', '%[6]s') ` + - `where chain_economics.parent_state_root = '%[1]s';` - - if _, err := s.db.Exec(fmt.Sprintf(ceInsert, - tipset.ParentState().String(), - supply.FilCirculating.String(), - supply.FilVested.String(), - supply.FilMined.String(), - supply.FilBurnt.String(), - supply.FilLocked.String(), - )); err != nil { - return xerrors.Errorf("insert circulating supply for tipset (%s): %w", tipset.Key().String(), err) - } - - return nil -} - -func (s *Syncer) storeHeaders(bhs map[cid.Cid]*types.BlockHeader, sync bool, timestamp time.Time) error { - s.headerLk.Lock() - defer s.headerLk.Unlock() - if len(bhs) == 0 { - return nil - } - log.Debugw("Storing Headers", "count", len(bhs)) - - tx, err := s.db.Begin() - if err != nil { - return xerrors.Errorf("begin: %w", err) - } - - if _, err := tx.Exec(` - -create temp table bc (like block_cids excluding constraints) on commit drop; -create temp table de (like drand_entries excluding constraints) on commit drop; -create temp table bde (like block_drand_entries excluding constraints) on commit drop; -create temp table tbp (like block_parents excluding constraints) on commit drop; -create temp table bs (like blocks_synced excluding constraints) on commit drop; -create temp table b (like blocks excluding constraints) on commit drop; - - -`); err != nil { - return xerrors.Errorf("prep temp: %w", err) - } - - { - stmt, err := tx.Prepare(`copy bc (cid) from STDIN`) - if err != nil { - return err - } - - for _, bh := range bhs { - if _, err := stmt.Exec(bh.Cid().String()); err != nil { - log.Error(err) - } - } - - if err := stmt.Close(); err != nil { - return err - } - - if _, err := tx.Exec(`insert into block_cids select * from bc on conflict do nothing `); err != nil { - return xerrors.Errorf("drand entries put: %w", err) - } - } - - { - stmt, err := tx.Prepare(`copy de (round, data) from STDIN`) - if err != nil { - return err - } - - for _, bh := range bhs { - for _, ent := range bh.BeaconEntries { - if _, err := stmt.Exec(ent.Round, ent.Data); err != nil { - log.Error(err) - } - } - } - - if err := stmt.Close(); err != nil { - return err - } - - if _, err := tx.Exec(`insert into drand_entries select * from de on conflict do nothing `); err != nil { - return xerrors.Errorf("drand entries put: %w", err) - } - } - - { - stmt, err := tx.Prepare(`copy bde (round, block) from STDIN`) - if err != nil { - return err - } - - for _, bh := range bhs { - for _, ent := range bh.BeaconEntries { - if _, err := stmt.Exec(ent.Round, bh.Cid().String()); err != nil { - log.Error(err) - } - } - } - - if err := stmt.Close(); err != nil { - return err - } - - if _, err := tx.Exec(`insert into block_drand_entries select * from bde on conflict do nothing `); err != nil { - return xerrors.Errorf("block drand entries put: %w", err) - } - } - - { - stmt, err := tx.Prepare(`copy tbp (block, parent) from STDIN`) - if err != nil { - return err - } - - for _, bh := range bhs { - for _, parent := range bh.Parents { - if _, err := stmt.Exec(bh.Cid().String(), parent.String()); err != nil { - log.Error(err) - } - } - } - - if err := stmt.Close(); err != nil { - return err - } - - if _, err := tx.Exec(`insert into block_parents select * from tbp on conflict do nothing `); err != nil { - return xerrors.Errorf("parent put: %w", err) - } - } - - if sync { - - stmt, err := tx.Prepare(`copy bs (cid, synced_at) from stdin `) - if err != nil { - return err - } - - for _, bh := range bhs { - if _, err := stmt.Exec(bh.Cid().String(), timestamp.Unix()); err != nil { - log.Error(err) - } - } - - if err := stmt.Close(); err != nil { - return err - } - - if _, err := tx.Exec(`insert into blocks_synced select * from bs on conflict do nothing `); err != nil { - return xerrors.Errorf("syncd put: %w", err) - } - } - - stmt2, err := tx.Prepare(`copy b (cid, parentWeight, parentStateRoot, height, miner, "timestamp", ticket, election_proof, win_count, parent_base_fee, forksig) from stdin`) - if err != nil { - return err - } - - for _, bh := range bhs { - var eproof, winCount interface{} - if bh.ElectionProof != nil { - eproof = bh.ElectionProof.VRFProof - winCount = bh.ElectionProof.WinCount - } - - if bh.Ticket == nil { - log.Warnf("got a block with nil ticket") - - bh.Ticket = &types.Ticket{ - VRFProof: []byte{}, - } - } - - if _, err := stmt2.Exec( - bh.Cid().String(), - bh.ParentWeight.String(), - bh.ParentStateRoot.String(), - bh.Height, - bh.Miner.String(), - bh.Timestamp, - bh.Ticket.VRFProof, - eproof, - winCount, - bh.ParentBaseFee.String(), - bh.ForkSignaling); err != nil { - log.Error(err) - } - } - - if err := stmt2.Close(); err != nil { - return xerrors.Errorf("s2 close: %w", err) - } - - if _, err := tx.Exec(`insert into blocks select * from b on conflict do nothing `); err != nil { - return xerrors.Errorf("blk put: %w", err) - } - - return tx.Commit() -} diff --git a/cmd/lotus-chainwatch/util/api.go b/cmd/lotus-chainwatch/util/api.go deleted file mode 100644 index f8f22cbbf..000000000 --- a/cmd/lotus-chainwatch/util/api.go +++ /dev/null @@ -1,34 +0,0 @@ -package util - -import ( - "context" - "net/http" - - "github.com/filecoin-project/go-jsonrpc" - "github.com/filecoin-project/lotus/api/client" - "github.com/filecoin-project/lotus/api/v0api" - ma "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr/net" -) - -func GetFullNodeAPIUsingCredentials(ctx context.Context, listenAddr, token string) (v0api.FullNode, jsonrpc.ClientCloser, error) { - parsedAddr, err := ma.NewMultiaddr(listenAddr) - if err != nil { - return nil, nil, err - } - - _, addr, err := manet.DialArgs(parsedAddr) - if err != nil { - return nil, nil, err - } - - return client.NewFullNodeRPCV0(ctx, apiURI(addr), apiHeaders(token)) -} -func apiURI(addr string) string { - return "ws://" + addr + "/rpc/v0" -} -func apiHeaders(token string) http.Header { - headers := http.Header{} - headers.Add("Authorization", "Bearer "+token) - return headers -} diff --git a/cmd/lotus-chainwatch/util/contextStore.go b/cmd/lotus-chainwatch/util/contextStore.go deleted file mode 100644 index c93f87f9b..000000000 --- a/cmd/lotus-chainwatch/util/contextStore.go +++ /dev/null @@ -1,51 +0,0 @@ -package util - -import ( - "bytes" - "context" - "fmt" - - "github.com/ipfs/go-cid" - cbg "github.com/whyrusleeping/cbor-gen" - - "github.com/filecoin-project/lotus/api/v0api" -) - -// TODO extract this to a common location in lotus and reuse the code - -// APIIpldStore is required for AMT and HAMT access. -type APIIpldStore struct { - ctx context.Context - api v0api.FullNode -} - -func NewAPIIpldStore(ctx context.Context, api v0api.FullNode) *APIIpldStore { - return &APIIpldStore{ - ctx: ctx, - api: api, - } -} - -func (ht *APIIpldStore) Context() context.Context { - return ht.ctx -} - -func (ht *APIIpldStore) Get(ctx context.Context, c cid.Cid, out interface{}) error { - raw, err := ht.api.ChainReadObj(ctx, c) - if err != nil { - return err - } - - cu, ok := out.(cbg.CBORUnmarshaler) - if ok { - if err := cu.UnmarshalCBOR(bytes.NewReader(raw)); err != nil { - return err - } - return nil - } - return fmt.Errorf("Object does not implement CBORUnmarshaler: %T", out) -} - -func (ht *APIIpldStore) Put(ctx context.Context, v interface{}) (cid.Cid, error) { - return cid.Undef, fmt.Errorf("Put is not implemented on APIIpldStore") -} diff --git a/cmd/lotus-miner/info.go b/cmd/lotus-miner/info.go index 3941ce563..878361dac 100644 --- a/cmd/lotus-miner/info.go +++ b/cmd/lotus-miner/info.go @@ -21,6 +21,7 @@ import ( "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/api/v0api" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/api" @@ -55,7 +56,7 @@ func infoCmdAct(cctx *cli.Context) error { } defer closer() - api, acloser, err := lcli.GetFullNodeAPI(cctx) + fullapi, acloser, err := lcli.GetFullNodeAPI(cctx) if err != nil { return err } @@ -63,9 +64,16 @@ func infoCmdAct(cctx *cli.Context) error { ctx := lcli.ReqContext(cctx) + subsystems, err := nodeApi.RuntimeSubsystems(ctx) + if err != nil { + return err + } + + fmt.Println("Enabled subsystems:", subsystems) + fmt.Print("Chain: ") - head, err := api.ChainHead(ctx) + head, err := fullapi.ChainHead(ctx) if err != nil { return err } @@ -95,24 +103,42 @@ func infoCmdAct(cctx *cli.Context) error { fmt.Println() + if subsystems.Has(api.SubsystemSectorStorage) { + err := handleMiningInfo(ctx, cctx, fullapi, nodeApi) + if err != nil { + return err + } + } + + if subsystems.Has(api.SubsystemMarkets) { + err := handleMarketsInfo(ctx, nodeApi) + if err != nil { + return err + } + } + + return nil +} + +func handleMiningInfo(ctx context.Context, cctx *cli.Context, fullapi v0api.FullNode, nodeApi api.StorageMiner) error { maddr, err := getActorAddress(ctx, cctx) if err != nil { return err } - mact, err := api.StateGetActor(ctx, maddr, types.EmptyTSK) + mact, err := fullapi.StateGetActor(ctx, maddr, types.EmptyTSK) if err != nil { return err } - tbs := blockstore.NewTieredBstore(blockstore.NewAPIBlockstore(api), blockstore.NewMemory()) + tbs := blockstore.NewTieredBstore(blockstore.NewAPIBlockstore(fullapi), blockstore.NewMemory()) mas, err := miner.Load(adt.WrapStore(ctx, cbor.NewCborStore(tbs)), mact) if err != nil { return err } // Sector size - mi, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK) + mi, err := fullapi.StateMinerInfo(ctx, maddr, types.EmptyTSK) if err != nil { return err } @@ -120,7 +146,7 @@ func infoCmdAct(cctx *cli.Context) error { ssize := types.SizeStr(types.NewInt(uint64(mi.SectorSize))) fmt.Printf("Miner: %s (%s sectors)\n", color.BlueString("%s", maddr), ssize) - pow, err := api.StateMinerPower(ctx, maddr, types.EmptyTSK) + pow, err := fullapi.StateMinerPower(ctx, maddr, types.EmptyTSK) if err != nil { return err } @@ -142,7 +168,7 @@ func infoCmdAct(cctx *cli.Context) error { pow.TotalPower.RawBytePower, ), ) - secCounts, err := api.StateMinerSectorCount(ctx, maddr, types.EmptyTSK) + secCounts, err := fullapi.StateMinerSectorCount(ctx, maddr, types.EmptyTSK) if err != nil { return err } @@ -219,6 +245,75 @@ func infoCmdAct(cctx *cli.Context) error { fmt.Println() + spendable := big.Zero() + + // NOTE: there's no need to unlock anything here. Funds only + // vest on deadline boundaries, and they're unlocked by cron. + lockedFunds, err := mas.LockedFunds() + if err != nil { + return xerrors.Errorf("getting locked funds: %w", err) + } + availBalance, err := mas.AvailableBalance(mact.Balance) + if err != nil { + return xerrors.Errorf("getting available balance: %w", err) + } + spendable = big.Add(spendable, availBalance) + + fmt.Printf("Miner Balance: %s\n", color.YellowString("%s", types.FIL(mact.Balance).Short())) + fmt.Printf(" PreCommit: %s\n", types.FIL(lockedFunds.PreCommitDeposits).Short()) + fmt.Printf(" Pledge: %s\n", types.FIL(lockedFunds.InitialPledgeRequirement).Short()) + fmt.Printf(" Vesting: %s\n", types.FIL(lockedFunds.VestingFunds).Short()) + colorTokenAmount(" Available: %s\n", availBalance) + + mb, err := fullapi.StateMarketBalance(ctx, maddr, types.EmptyTSK) + if err != nil { + return xerrors.Errorf("getting market balance: %w", err) + } + spendable = big.Add(spendable, big.Sub(mb.Escrow, mb.Locked)) + + fmt.Printf("Market Balance: %s\n", types.FIL(mb.Escrow).Short()) + fmt.Printf(" Locked: %s\n", types.FIL(mb.Locked).Short()) + colorTokenAmount(" Available: %s\n", big.Sub(mb.Escrow, mb.Locked)) + + wb, err := fullapi.WalletBalance(ctx, mi.Worker) + if err != nil { + return xerrors.Errorf("getting worker balance: %w", err) + } + spendable = big.Add(spendable, wb) + color.Cyan("Worker Balance: %s", types.FIL(wb).Short()) + if len(mi.ControlAddresses) > 0 { + cbsum := big.Zero() + for _, ca := range mi.ControlAddresses { + b, err := fullapi.WalletBalance(ctx, ca) + if err != nil { + return xerrors.Errorf("getting control address balance: %w", err) + } + cbsum = big.Add(cbsum, b) + } + spendable = big.Add(spendable, cbsum) + + fmt.Printf(" Control: %s\n", types.FIL(cbsum).Short()) + } + colorTokenAmount("Total Spendable: %s\n", spendable) + + fmt.Println() + + if !cctx.Bool("hide-sectors-info") { + fmt.Println("Sectors:") + err = sectorsInfo(ctx, nodeApi) + if err != nil { + return err + } + } + + // TODO: grab actr state / info + // * Sealed sectors (count / bytes) + // * Power + + return nil +} + +func handleMarketsInfo(ctx context.Context, nodeApi api.StorageMiner) error { deals, err := nodeApi.MarketListIncompleteDeals(ctx) if err != nil { return err @@ -309,70 +404,6 @@ func infoCmdAct(cctx *cli.Context) error { fmt.Println() - spendable := big.Zero() - - // NOTE: there's no need to unlock anything here. Funds only - // vest on deadline boundaries, and they're unlocked by cron. - lockedFunds, err := mas.LockedFunds() - if err != nil { - return xerrors.Errorf("getting locked funds: %w", err) - } - availBalance, err := mas.AvailableBalance(mact.Balance) - if err != nil { - return xerrors.Errorf("getting available balance: %w", err) - } - spendable = big.Add(spendable, availBalance) - - fmt.Printf("Miner Balance: %s\n", color.YellowString("%s", types.FIL(mact.Balance).Short())) - fmt.Printf(" PreCommit: %s\n", types.FIL(lockedFunds.PreCommitDeposits).Short()) - fmt.Printf(" Pledge: %s\n", types.FIL(lockedFunds.InitialPledgeRequirement).Short()) - fmt.Printf(" Vesting: %s\n", types.FIL(lockedFunds.VestingFunds).Short()) - colorTokenAmount(" Available: %s\n", availBalance) - - mb, err := api.StateMarketBalance(ctx, maddr, types.EmptyTSK) - if err != nil { - return xerrors.Errorf("getting market balance: %w", err) - } - spendable = big.Add(spendable, big.Sub(mb.Escrow, mb.Locked)) - - fmt.Printf("Market Balance: %s\n", types.FIL(mb.Escrow).Short()) - fmt.Printf(" Locked: %s\n", types.FIL(mb.Locked).Short()) - colorTokenAmount(" Available: %s\n", big.Sub(mb.Escrow, mb.Locked)) - - wb, err := api.WalletBalance(ctx, mi.Worker) - if err != nil { - return xerrors.Errorf("getting worker balance: %w", err) - } - spendable = big.Add(spendable, wb) - color.Cyan("Worker Balance: %s", types.FIL(wb).Short()) - if len(mi.ControlAddresses) > 0 { - cbsum := big.Zero() - for _, ca := range mi.ControlAddresses { - b, err := api.WalletBalance(ctx, ca) - if err != nil { - return xerrors.Errorf("getting control address balance: %w", err) - } - cbsum = big.Add(cbsum, b) - } - spendable = big.Add(spendable, cbsum) - - fmt.Printf(" Control: %s\n", types.FIL(cbsum).Short()) - } - colorTokenAmount("Total Spendable: %s\n", spendable) - - fmt.Println() - - if !cctx.Bool("hide-sectors-info") { - fmt.Println("Sectors:") - err = sectorsInfo(ctx, nodeApi) - if err != nil { - return err - } - } - - // TODO: grab actr state / info - // * Sealed sectors (count / bytes) - // * Power return nil } diff --git a/cmd/lotus-townhall/main.go b/cmd/lotus-townhall/main.go deleted file mode 100644 index 1e0460dee..000000000 --- a/cmd/lotus-townhall/main.go +++ /dev/null @@ -1,134 +0,0 @@ -package main - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "net/http" - "time" - - rice "github.com/GeertJohan/go.rice" - "github.com/gorilla/websocket" - "github.com/ipld/go-car" - "github.com/libp2p/go-libp2p" - "github.com/libp2p/go-libp2p-core/peer" - pubsub "github.com/libp2p/go-libp2p-pubsub" - - "github.com/filecoin-project/lotus/blockstore" - "github.com/filecoin-project/lotus/build" -) - -var topic = "/fil/headnotifs/" - -func init() { - genBytes := build.MaybeGenesis() - if len(genBytes) == 0 { - topic = "" - return - } - - bs := blockstore.NewMemory() - - c, err := car.LoadCar(bs, bytes.NewReader(genBytes)) - if err != nil { - panic(err) - } - if len(c.Roots) != 1 { - panic("expected genesis file to have one root") - } - - fmt.Printf("Genesis CID: %s\n", c.Roots[0]) - topic = topic + c.Roots[0].String() -} - -var upgrader = websocket.Upgrader{ - WriteBufferSize: 1024, - CheckOrigin: func(r *http.Request) bool { - return true - }, -} - -func main() { - if topic == "" { - fmt.Println("FATAL: No genesis found") - return - } - - ctx := context.Background() - - host, err := libp2p.New( - ctx, - libp2p.Defaults, - ) - if err != nil { - panic(err) - } - ps, err := pubsub.NewGossipSub(ctx, host) - if err != nil { - panic(err) - } - - pi, err := build.BuiltinBootstrap() - if err != nil { - panic(err) - } - - if err := host.Connect(ctx, pi[0]); err != nil { - panic(err) - } - - http.HandleFunc("/sub", handler(ps)) - http.Handle("/", http.FileServer(rice.MustFindBox("townhall/build").HTTPBox())) - - fmt.Println("listening on http://localhost:2975") - - if err := http.ListenAndServe("0.0.0.0:2975", nil); err != nil { - panic(err) - } -} - -type update struct { - From peer.ID - Update json.RawMessage - Time uint64 -} - -func handler(ps *pubsub.PubSub) func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Access-Control-Allow-Origin", "*") - if r.Header.Get("Sec-WebSocket-Protocol") != "" { - w.Header().Set("Sec-WebSocket-Protocol", r.Header.Get("Sec-WebSocket-Protocol")) - } - - conn, err := upgrader.Upgrade(w, r, nil) - if err != nil { - return - } - - sub, err := ps.Subscribe(topic) //nolint - if err != nil { - return - } - defer sub.Cancel() //nolint:errcheck - - fmt.Println("new conn") - - for { - msg, err := sub.Next(r.Context()) - if err != nil { - return - } - - //fmt.Println(msg) - - if err := conn.WriteJSON(update{ - From: peer.ID(msg.From), - Update: msg.Data, - Time: uint64(time.Now().UnixNano() / 1000_000), - }); err != nil { - return - } - } - } -} diff --git a/cmd/lotus-townhall/townhall/.gitignore b/cmd/lotus-townhall/townhall/.gitignore deleted file mode 100644 index 4d29575de..000000000 --- a/cmd/lotus-townhall/townhall/.gitignore +++ /dev/null @@ -1,23 +0,0 @@ -# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. - -# dependencies -/node_modules -/.pnp -.pnp.js - -# testing -/coverage - -# production -/build - -# misc -.DS_Store -.env.local -.env.development.local -.env.test.local -.env.production.local - -npm-debug.log* -yarn-debug.log* -yarn-error.log* diff --git a/cmd/lotus-townhall/townhall/package.json b/cmd/lotus-townhall/townhall/package.json deleted file mode 100644 index 5a8167622..000000000 --- a/cmd/lotus-townhall/townhall/package.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "name": "townhall", - "version": "0.1.0", - "private": true, - "dependencies": { - "react": "^16.10.2", - "react-dom": "^16.10.2", - "react-scripts": "3.2.0" - }, - "scripts": { - "start": "react-scripts start", - "build": "react-scripts build", - "test": "react-scripts test", - "eject": "react-scripts eject" - }, - "eslintConfig": { - "extends": "react-app" - }, - "browserslist": { - "production": [ - ">0.2%", - "not dead", - "not op_mini all" - ], - "development": [ - "last 1 chrome version", - "last 1 firefox version", - "last 1 safari version" - ] - } -} diff --git a/cmd/lotus-townhall/townhall/public/index.html b/cmd/lotus-townhall/townhall/public/index.html deleted file mode 100644 index 38af10597..000000000 --- a/cmd/lotus-townhall/townhall/public/index.html +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - Lotus TownHall - - - -
- - diff --git a/cmd/lotus-townhall/townhall/public/robots.txt b/cmd/lotus-townhall/townhall/public/robots.txt deleted file mode 100644 index 01b0f9a10..000000000 --- a/cmd/lotus-townhall/townhall/public/robots.txt +++ /dev/null @@ -1,2 +0,0 @@ -# https://www.robotstxt.org/robotstxt.html -User-agent: * diff --git a/cmd/lotus-townhall/townhall/src/App.css b/cmd/lotus-townhall/townhall/src/App.css deleted file mode 100644 index 8b1378917..000000000 --- a/cmd/lotus-townhall/townhall/src/App.css +++ /dev/null @@ -1 +0,0 @@ - diff --git a/cmd/lotus-townhall/townhall/src/App.js b/cmd/lotus-townhall/townhall/src/App.js deleted file mode 100644 index 2f216f5da..000000000 --- a/cmd/lotus-townhall/townhall/src/App.js +++ /dev/null @@ -1,87 +0,0 @@ -import React from 'react'; -import './App.css'; - -function colForH(besth, height) { - const diff = besth - height - if(diff === 0) return '#6f6' - if(diff === 1) return '#df4' - if(diff < 4) return '#ff0' - if(diff < 10) return '#f60' - return '#f00' -} - -function colLag(lag) { - if(lag < 100) return '#6f6' - if(lag < 400) return '#df4' - if(lag < 1000) return '#ff0' - if(lag < 4000) return '#f60' - return '#f00' -} - -function lagCol(lag, good) { - return - {lag} - ms - -} - -class App extends React.Component { - constructor(props) { - super(props); - - let ws = new WebSocket("ws://" + window.location.host + "/sub") - //let ws = new WebSocket("ws://127.0.0.1:2975/sub") - - ws.onmessage = (ev) => { - console.log(ev) - let update = JSON.parse(ev.data) - - update.Update.Weight = Number(update.Update.Weight) - - let wdiff = update.Update.Weight - (this.state[update.From] || {Weight: update.Update.Weight}).Weight - wdiff = {wdiff} - - let utDiff = update.Time - (this.state[update.From] || {utime: update.Time}).utime - utDiff = {utDiff}ms - - this.setState( prev => ({ - ...prev, [update.From]: {...update.Update, utime: update.Time, wdiff: wdiff, utDiff: utDiff}, - })) - } - - ws.onclose = () => { - this.setState({disconnected: true}) - } - - this.state = {} - } - - render() { - if(this.state.disconnected) { - return Error: disconnected - } - - let besth = Object.keys(this.state).map(k => this.state[k]).reduce((p, n) => p > n.Height ? p : n.Height, -1) - let bestw = Object.keys(this.state).map(k => this.state[k]).reduce((p, n) => p > n.Weight ? p : n.Weight, -1) - - return - - {Object.keys(this.state).map(k => [k, this.state[k]]).map(([k, v]) => { - let mnrs = v.Blocks.map(b => ) - let l = [ - , - , - , - , - , - ...mnrs, - ] - - l = {l} - return l - }) - } -
PeerIDNicknameLagWeight(best, prev)HeightBlocks
 m:{b.Miner}({lagCol(v.Time ? v.Time - (b.Timestamp*1000) : v.utime - (b.Timestamp*1000), v.Time)}){k}{v.NodeName}{v.Time ? lagCol(v.utime - v.Time, true) : ""}(Δ{v.utDiff}){v.Weight}({bestw - v.Weight}, {v.wdiff}){v.Height}({besth - v.Height})
- } -} -export default App; diff --git a/cmd/lotus-townhall/townhall/src/App.test.js b/cmd/lotus-townhall/townhall/src/App.test.js deleted file mode 100644 index a754b201b..000000000 --- a/cmd/lotus-townhall/townhall/src/App.test.js +++ /dev/null @@ -1,9 +0,0 @@ -import React from 'react'; -import ReactDOM from 'react-dom'; -import App from './App'; - -it('renders without crashing', () => { - const div = document.createElement('div'); - ReactDOM.render(, div); - ReactDOM.unmountComponentAtNode(div); -}); diff --git a/cmd/lotus-townhall/townhall/src/index.css b/cmd/lotus-townhall/townhall/src/index.css deleted file mode 100644 index fb0d9d10e..000000000 --- a/cmd/lotus-townhall/townhall/src/index.css +++ /dev/null @@ -1,6 +0,0 @@ -body { - margin: 0; - font-family: monospace; - background: #1f1f1f; - color: #f0f0f0; -} diff --git a/cmd/lotus-townhall/townhall/src/index.js b/cmd/lotus-townhall/townhall/src/index.js deleted file mode 100644 index 395b74997..000000000 --- a/cmd/lotus-townhall/townhall/src/index.js +++ /dev/null @@ -1,6 +0,0 @@ -import React from 'react'; -import ReactDOM from 'react-dom'; -import './index.css'; -import App from './App'; - -ReactDOM.render(, document.getElementById('root')); diff --git a/documentation/en/api-v0-methods-miner.md b/documentation/en/api-v0-methods-miner.md index e8598ff0c..3b6d5ac51 100644 --- a/documentation/en/api-v0-methods-miner.md +++ b/documentation/en/api-v0-methods-miner.md @@ -94,6 +94,8 @@ * [ReturnSealPreCommit1](#ReturnSealPreCommit1) * [ReturnSealPreCommit2](#ReturnSealPreCommit2) * [ReturnUnsealPiece](#ReturnUnsealPiece) +* [Runtime](#Runtime) + * [RuntimeSubsystems](#RuntimeSubsystems) * [Sealing](#Sealing) * [SealingAbort](#SealingAbort) * [SealingSchedDiag](#SealingSchedDiag) @@ -1522,6 +1524,28 @@ Inputs: Response: `{}` +## Runtime + + +### RuntimeSubsystems +RuntimeSubsystems returns the subsystems that are enabled +in this instance. + + +Perms: read + +Inputs: `null` + +Response: +```json +[ + "Mining", + "Sealing", + "SectorStorage", + "Markets" +] +``` + ## Sealing diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index 29593fbd4..4aab4bfcf 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -7,7 +7,7 @@ USAGE: lotus-miner [global options] command [command options] [arguments...] VERSION: - 1.11.1-dev + 1.11.2-dev COMMANDS: init Initialize a lotus miner repo diff --git a/documentation/en/cli-lotus-worker.md b/documentation/en/cli-lotus-worker.md index dbfc8da29..a06ad0fff 100644 --- a/documentation/en/cli-lotus-worker.md +++ b/documentation/en/cli-lotus-worker.md @@ -7,7 +7,7 @@ USAGE: lotus-worker [global options] command [command options] [arguments...] VERSION: - 1.11.1-dev + 1.11.2-dev COMMANDS: run Start lotus worker diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index 5faf5c58b..75f329a0f 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -7,7 +7,7 @@ USAGE: lotus [global options] command [command options] [arguments...] VERSION: - 1.11.1-dev + 1.11.2-dev COMMANDS: daemon Start a lotus daemon process @@ -1531,7 +1531,7 @@ OPTIONS: --gas-premium value gas price for new message (pay to miner, attoFIL/GasUnit) --gas-limit value gas limit for new message (GasUnit) (default: 0) --auto automatically reprice the specified message (default: false) - --max-fee value Spend up to X attoFIL for this message (applicable for auto mode) + --fee-limit max-fee Spend up to X FIL for this message in units of FIL. Previously when flag was max-fee units were in attoFIL. Applicable for auto mode --help, -h show help (default: false) ``` diff --git a/extern/storage-sealing/checks.go b/extern/storage-sealing/checks.go index 5ba23026d..115eedea5 100644 --- a/extern/storage-sealing/checks.go +++ b/extern/storage-sealing/checks.go @@ -93,27 +93,29 @@ func checkPrecommit(ctx context.Context, maddr address.Address, si SectorInfo, t return &ErrBadCommD{xerrors.Errorf("on chain CommD differs from sector: %s != %s", commD, si.CommD)} } - ticketEarliest := height - policy.MaxPreCommitRandomnessLookback - - if si.TicketEpoch < ticketEarliest { - return &ErrExpiredTicket{xerrors.Errorf("ticket expired: seal height: %d, head: %d", si.TicketEpoch+policy.SealRandomnessLookback, height)} - } - pci, err := api.StateSectorPreCommitInfo(ctx, maddr, si.SectorNumber, tok) if err != nil { if err == ErrSectorAllocated { + //committed P2 message but commit C2 message too late, pci should be null in this case return &ErrSectorNumberAllocated{err} } return &ErrApi{xerrors.Errorf("getting precommit info: %w", err)} } if pci != nil { + // committed P2 message if pci.Info.SealRandEpoch != si.TicketEpoch { return &ErrBadTicket{xerrors.Errorf("bad ticket epoch: %d != %d", pci.Info.SealRandEpoch, si.TicketEpoch)} } return &ErrPrecommitOnChain{xerrors.Errorf("precommit already on chain")} } + //never commit P2 message before, check ticket expiration + ticketEarliest := height - policy.MaxPreCommitRandomnessLookback + + if si.TicketEpoch < ticketEarliest { + return &ErrExpiredTicket{xerrors.Errorf("ticket expired: seal height: %d, head: %d", si.TicketEpoch+policy.SealRandomnessLookback, height)} + } return nil } diff --git a/extern/storage-sealing/precommit_policy.go b/extern/storage-sealing/precommit_policy.go index a6add5693..c911ccc8c 100644 --- a/extern/storage-sealing/precommit_policy.go +++ b/extern/storage-sealing/precommit_policy.go @@ -3,11 +3,13 @@ package sealing import ( "context" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" - - "github.com/filecoin-project/go-state-types/network" + "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/policy" ) type PreCommitPolicy interface { @@ -34,21 +36,23 @@ type Chain interface { // If we're in Mode 2: The pre-commit expiration epoch will be set to the // current epoch + the provided default duration. type BasicPreCommitPolicy struct { - api Chain + api Chain + getSealingConfig GetSealingConfigFunc provingBoundary abi.ChainEpoch - duration abi.ChainEpoch + provingBuffer abi.ChainEpoch } // NewBasicPreCommitPolicy produces a BasicPreCommitPolicy. // // The provided duration is used as the default sector expiry when the sector // contains no deals. The proving boundary is used to adjust/align the sector's expiration. -func NewBasicPreCommitPolicy(api Chain, duration abi.ChainEpoch, provingBoundary abi.ChainEpoch) BasicPreCommitPolicy { +func NewBasicPreCommitPolicy(api Chain, cfgGetter GetSealingConfigFunc, provingBoundary abi.ChainEpoch, provingBuffer abi.ChainEpoch) BasicPreCommitPolicy { return BasicPreCommitPolicy{ - api: api, - provingBoundary: provingBoundary, - duration: duration, + api: api, + getSealingConfig: cfgGetter, + provingBoundary: provingBoundary, + provingBuffer: provingBuffer, } } @@ -79,7 +83,13 @@ func (p *BasicPreCommitPolicy) Expiration(ctx context.Context, ps ...Piece) (abi } if end == nil { - tmp := epoch + p.duration + // no deal pieces, get expiration for committed capacity sector + expirationDuration, err := p.getCCSectorLifetime() + if err != nil { + return 0, err + } + + tmp := epoch + expirationDuration end = &tmp } @@ -87,3 +97,27 @@ func (p *BasicPreCommitPolicy) Expiration(ctx context.Context, ps ...Piece) (abi return *end, nil } + +func (p *BasicPreCommitPolicy) getCCSectorLifetime() (abi.ChainEpoch, error) { + c, err := p.getSealingConfig() + if err != nil { + return 0, xerrors.Errorf("sealing config load error: %w", err) + } + + var ccLifetimeEpochs = abi.ChainEpoch(uint64(c.CommittedCapacitySectorLifetime.Seconds()) / builtin.EpochDurationSeconds) + // if zero value in config, assume maximum sector extension + if ccLifetimeEpochs == 0 { + ccLifetimeEpochs = policy.GetMaxSectorExpirationExtension() + } + + if minExpiration := abi.ChainEpoch(miner.MinSectorExpiration); ccLifetimeEpochs < minExpiration { + log.Warnf("value for CommittedCapacitySectorLiftime is too short, using default minimum (%d epochs)", minExpiration) + return minExpiration, nil + } + if maxExpiration := policy.GetMaxSectorExpirationExtension(); ccLifetimeEpochs > maxExpiration { + log.Warnf("value for CommittedCapacitySectorLiftime is too long, using default maximum (%d epochs)", maxExpiration) + return maxExpiration, nil + } + + return (ccLifetimeEpochs - p.provingBuffer), nil +} diff --git a/extern/storage-sealing/precommit_policy_test.go b/extern/storage-sealing/precommit_policy_test.go index a6c17d3fd..7f5aff0df 100644 --- a/extern/storage-sealing/precommit_policy_test.go +++ b/extern/storage-sealing/precommit_policy_test.go @@ -3,10 +3,16 @@ package sealing_test import ( "context" "testing" + "time" "github.com/filecoin-project/go-state-types/network" api "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/policy" + sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" "github.com/ipfs/go-cid" "github.com/stretchr/testify/assert" @@ -14,14 +20,28 @@ import ( commcid "github.com/filecoin-project/go-fil-commcid" "github.com/filecoin-project/go-state-types/abi" - - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" ) type fakeChain struct { h abi.ChainEpoch } +type fakeConfigStub struct { + CCSectorLifetime time.Duration +} + +func fakeConfigGetter(stub *fakeConfigStub) sealing.GetSealingConfigFunc { + return func() (sealiface.Config, error) { + if stub == nil { + return sealiface.Config{}, nil + } + + return sealiface.Config{ + CommittedCapacitySectorLifetime: stub.CCSectorLifetime, + }, nil + } +} + func (f *fakeChain) StateNetworkVersion(ctx context.Context, tok sealing.TipSetToken) (network.Version, error) { return build.NewestNetworkVersion, nil } @@ -38,21 +58,49 @@ func fakePieceCid(t *testing.T) cid.Cid { } func TestBasicPolicyEmptySector(t *testing.T) { - policy := sealing.NewBasicPreCommitPolicy(&fakeChain{ - h: abi.ChainEpoch(55), - }, 10, 0) + cfg := fakeConfigGetter(nil) + h := abi.ChainEpoch(55) + pBoundary := abi.ChainEpoch(0) + pBuffer := abi.ChainEpoch(2) + pcp := sealing.NewBasicPreCommitPolicy(&fakeChain{h: h}, cfg, pBoundary, pBuffer) + exp, err := pcp.Expiration(context.Background()) - exp, err := policy.Expiration(context.Background()) require.NoError(t, err) - assert.Equal(t, 2879, int(exp)) + // as set when there are no deal pieces + expected := h + policy.GetMaxSectorExpirationExtension() - (pBuffer * 2) + // as set just before returning within Expiration() + expected += miner.WPoStProvingPeriod - (expected % miner.WPoStProvingPeriod) + pBoundary - 1 + assert.Equal(t, int(expected), int(exp)) +} + +func TestCustomCCSectorConfig(t *testing.T) { + customLifetime := 200 * 24 * time.Hour + customLifetimeEpochs := abi.ChainEpoch(int64(customLifetime.Seconds()) / builtin.EpochDurationSeconds) + cfgStub := fakeConfigStub{CCSectorLifetime: customLifetime} + cfg := fakeConfigGetter(&cfgStub) + h := abi.ChainEpoch(55) + pBoundary := abi.ChainEpoch(0) + pBuffer := abi.ChainEpoch(2) + pcp := sealing.NewBasicPreCommitPolicy(&fakeChain{h: h}, cfg, pBoundary, pBuffer) + exp, err := pcp.Expiration(context.Background()) + + require.NoError(t, err) + + // as set when there are no deal pieces + expected := h + customLifetimeEpochs - (pBuffer * 2) + // as set just before returning within Expiration() + expected += miner.WPoStProvingPeriod - (expected % miner.WPoStProvingPeriod) + pBoundary - 1 + assert.Equal(t, int(expected), int(exp)) } func TestBasicPolicyMostConstrictiveSchedule(t *testing.T) { + cfg := fakeConfigGetter(nil) + pPeriod := abi.ChainEpoch(11) policy := sealing.NewBasicPreCommitPolicy(&fakeChain{ h: abi.ChainEpoch(55), - }, 100, 11) - + }, cfg, pPeriod, 2) + longestDealEpochEnd := abi.ChainEpoch(100) pieces := []sealing.Piece{ { Piece: abi.PieceInfo{ @@ -76,7 +124,7 @@ func TestBasicPolicyMostConstrictiveSchedule(t *testing.T) { DealID: abi.DealID(43), DealSchedule: api.DealSchedule{ StartEpoch: abi.ChainEpoch(80), - EndEpoch: abi.ChainEpoch(100), + EndEpoch: longestDealEpochEnd, }, }, }, @@ -85,13 +133,15 @@ func TestBasicPolicyMostConstrictiveSchedule(t *testing.T) { exp, err := policy.Expiration(context.Background(), pieces...) require.NoError(t, err) - assert.Equal(t, 2890, int(exp)) + expected := longestDealEpochEnd + miner.WPoStProvingPeriod - (longestDealEpochEnd % miner.WPoStProvingPeriod) + pPeriod - 1 + assert.Equal(t, int(expected), int(exp)) } func TestBasicPolicyIgnoresExistingScheduleIfExpired(t *testing.T) { + cfg := fakeConfigGetter(nil) policy := sealing.NewBasicPreCommitPolicy(&fakeChain{ h: abi.ChainEpoch(55), - }, 100, 0) + }, cfg, 0, 0) pieces := []sealing.Piece{ { @@ -112,13 +162,14 @@ func TestBasicPolicyIgnoresExistingScheduleIfExpired(t *testing.T) { exp, err := policy.Expiration(context.Background(), pieces...) require.NoError(t, err) - assert.Equal(t, 2879, int(exp)) + assert.Equal(t, 1558079, int(exp)) } func TestMissingDealIsIgnored(t *testing.T) { + cfg := fakeConfigGetter(nil) policy := sealing.NewBasicPreCommitPolicy(&fakeChain{ h: abi.ChainEpoch(55), - }, 100, 11) + }, cfg, 11, 0) pieces := []sealing.Piece{ { @@ -146,5 +197,5 @@ func TestMissingDealIsIgnored(t *testing.T) { exp, err := policy.Expiration(context.Background(), pieces...) require.NoError(t, err) - assert.Equal(t, 2890, int(exp)) + assert.Equal(t, 1558090, int(exp)) } diff --git a/extern/storage-sealing/sealiface/config.go b/extern/storage-sealing/sealiface/config.go index e33b36263..95b851609 100644 --- a/extern/storage-sealing/sealiface/config.go +++ b/extern/storage-sealing/sealiface/config.go @@ -20,6 +20,8 @@ type Config struct { WaitDealsDelay time.Duration + CommittedCapacitySectorLifetime time.Duration + AlwaysKeepUnsealedCopy bool FinalizeEarly bool diff --git a/go.mod b/go.mod index 5f968f6e0..11d24b0f6 100644 --- a/go.mod +++ b/go.mod @@ -33,16 +33,16 @@ require ( github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 - github.com/filecoin-project/go-data-transfer v1.7.0 + github.com/filecoin-project/go-data-transfer v1.7.1 github.com/filecoin-project/go-fil-commcid v0.1.0 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 - github.com/filecoin-project/go-fil-markets v1.6.0 + github.com/filecoin-project/go-fil-markets v1.6.1 github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1 github.com/filecoin-project/go-paramfetch v0.0.2-0.20210614165157-25a6c7769498 github.com/filecoin-project/go-state-types v0.1.1-0.20210722133031-ad9bfe54c124 - github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe + github.com/filecoin-project/go-statemachine v1.0.0 github.com/filecoin-project/go-statestore v0.1.1 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.14 @@ -56,7 +56,7 @@ require ( github.com/gdamore/tcell/v2 v2.2.0 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect - github.com/golang/mock v1.5.0 + github.com/golang/mock v1.6.0 github.com/google/uuid v1.1.2 github.com/gorilla/mux v1.7.4 github.com/gorilla/websocket v1.4.2 @@ -78,7 +78,7 @@ require ( github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459 github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.6 - github.com/ipfs/go-graphsync v0.6.5 + github.com/ipfs/go-graphsync v0.6.6 github.com/ipfs/go-ipfs-blockstore v1.0.3 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-ds-help v1.0.0 @@ -100,22 +100,21 @@ require ( github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d github.com/ipld/go-ipld-prime v0.5.1-0.20201021195245-109253e8a018 github.com/kelseyhightower/envconfig v1.4.0 - github.com/lib/pq v1.7.0 github.com/libp2p/go-buffer-pool v0.0.2 github.com/libp2p/go-eventbus v0.2.1 github.com/libp2p/go-libp2p v0.14.2 github.com/libp2p/go-libp2p-connmgr v0.2.4 - github.com/libp2p/go-libp2p-core v0.8.5 - github.com/libp2p/go-libp2p-discovery v0.5.0 + github.com/libp2p/go-libp2p-core v0.8.6 + github.com/libp2p/go-libp2p-discovery v0.5.1 github.com/libp2p/go-libp2p-kad-dht v0.11.0 github.com/libp2p/go-libp2p-mplex v0.4.1 github.com/libp2p/go-libp2p-noise v0.2.0 - github.com/libp2p/go-libp2p-peerstore v0.2.7 - github.com/libp2p/go-libp2p-pubsub v0.5.0 - github.com/libp2p/go-libp2p-quic-transport v0.10.0 + github.com/libp2p/go-libp2p-peerstore v0.2.8 + github.com/libp2p/go-libp2p-pubsub v0.5.3 + github.com/libp2p/go-libp2p-quic-transport v0.11.2 github.com/libp2p/go-libp2p-record v0.1.3 github.com/libp2p/go-libp2p-routing-helpers v0.2.3 - github.com/libp2p/go-libp2p-swarm v0.5.0 + github.com/libp2p/go-libp2p-swarm v0.5.3 github.com/libp2p/go-libp2p-tls v0.1.3 github.com/libp2p/go-libp2p-yamux v0.5.4 github.com/libp2p/go-maddr-filter v0.1.0 @@ -124,14 +123,14 @@ require ( github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-base32 v0.0.3 - github.com/multiformats/go-multiaddr v0.3.1 + github.com/multiformats/go-multiaddr v0.3.3 github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multibase v0.0.3 github.com/multiformats/go-multihash v0.0.15 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/opentracing/opentracing-go v1.2.0 github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a - github.com/prometheus/client_golang v1.6.0 + github.com/prometheus/client_golang v1.10.0 github.com/raulk/clock v1.1.0 github.com/raulk/go-watchdog v1.0.1 github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 @@ -150,9 +149,9 @@ require ( go.uber.org/fx v1.9.0 go.uber.org/multierr v1.6.0 go.uber.org/zap v1.16.0 - golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6 + golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20210510120138-977fb7262007 + golang.org/x/sys v0.0.0-20210511113859-b0526f3d8744 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 golang.org/x/tools v0.1.5 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 diff --git a/go.sum b/go.sum index b22f3dc15..ca71b27d2 100644 --- a/go.sum +++ b/go.sum @@ -27,8 +27,9 @@ dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1 dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -199,8 +200,9 @@ github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1: github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= -github.com/dgraph-io/badger v1.6.1 h1:w9pSFNSdq/JPM1N12Fz/F/bzo993Is1W+Q7HjPzi7yg= github.com/dgraph-io/badger v1.6.1/go.mod h1:FRmFw3uxvcpa8zG3Rxs0th+hCLIuaQg8HlNV5bjgnuU= +github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= +github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM= github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= @@ -274,8 +276,9 @@ github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7/ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-data-transfer v1.0.1/go.mod h1:UxvfUAY9v3ub0a21BSK9u3pB2aq30Y0KMsG+w9/ysyo= -github.com/filecoin-project/go-data-transfer v1.7.0 h1:mFRn+UuTdPROmhplLSekzd4rAs9ug8ubtSY4nw9wYkU= github.com/filecoin-project/go-data-transfer v1.7.0/go.mod h1:GLRr5BmLEqsLwXfiRDG7uJvph22KGL2M4iOuF8EINaU= +github.com/filecoin-project/go-data-transfer v1.7.1 h1:Co4bTenvCc3WnOhQWyXRt59FLZvxwH8UeF0ZCOc1ik0= +github.com/filecoin-project/go-data-transfer v1.7.1/go.mod h1:GLRr5BmLEqsLwXfiRDG7uJvph22KGL2M4iOuF8EINaU= github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl8btmWxyFMEeeWGUxIQ= github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= @@ -285,8 +288,8 @@ github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+ github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo= github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8= github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c= -github.com/filecoin-project/go-fil-markets v1.6.0 h1:+1usyX7rXz6Ey6hbHd/Fhx616ZvGCI94rW7wneMcptU= -github.com/filecoin-project/go-fil-markets v1.6.0/go.mod h1:ZuFDagROUV6GfvBU//KReTQDw+EZci4rH7jMYTD10vs= +github.com/filecoin-project/go-fil-markets v1.6.1 h1:8xdFyWrELfOzwcGa229bLu/olD+1l4sEWFIsZR7oz5U= +github.com/filecoin-project/go-fil-markets v1.6.1/go.mod h1:ZuFDagROUV6GfvBU//KReTQDw+EZci4rH7jMYTD10vs= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM= @@ -311,8 +314,9 @@ github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psS github.com/filecoin-project/go-state-types v0.1.1-0.20210506134452-99b279731c48/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.1-0.20210722133031-ad9bfe54c124 h1:veGrNABg/9I7prngrowkhwbvW5d5JN55MNKmbsr5FqA= github.com/filecoin-project/go-state-types v0.1.1-0.20210722133031-ad9bfe54c124/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= -github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe h1:dF8u+LEWeIcTcfUcCf3WFVlc81Fr2JKg8zPzIbBDKDw= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= +github.com/filecoin-project/go-statemachine v1.0.0 h1:b8FpFewPSklyAIUqH0oHt4nvKf03bU7asop1bJpjAtQ= +github.com/filecoin-project/go-statemachine v1.0.0/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-statestore v0.1.1 h1:ufMFq00VqnT2CAuDpcGnwLnCX1I/c3OROw/kXVNSTZk= github.com/filecoin-project/go-statestore v0.1.1/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= @@ -397,6 +401,8 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968 h1:s+PDl6lozQ+dEUtUtQnO7+A2iPG3sK1pI4liU+jxn90= github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME= @@ -429,8 +435,8 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -443,8 +449,10 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf h1:gFVkHXmVAhEbxZVDln5V9GKrLaluNoFHDbrZwAWZgws= @@ -457,8 +465,10 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3 h1:x95R7cp+rSeeqAMI2knLtQ0DKlaBhv2NrtrOvafPHRo= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -609,8 +619,9 @@ github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjv github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s= github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE= -github.com/ipfs/go-ds-badger v0.2.3 h1:J27YvAcpuA5IvZUbeBxOcQgqnYHUPxoygc6QxxkodZ4= github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk= +github.com/ipfs/go-ds-badger v0.2.7 h1:ju5REfIm+v+wgVnQ19xGLYPHYHbYLR6qJfmMbCDSK1I= +github.com/ipfs/go-ds-badger v0.2.7/go.mod h1:02rnztVKA4aZwDuaRPTf8mpqcKmXP7mLl6JPxd14JHA= github.com/ipfs/go-ds-badger2 v0.1.0/go.mod h1:pbR1p817OZbdId9EvLOhKBgUVTM3BMCSTan78lDDVaw= github.com/ipfs/go-ds-badger2 v0.1.1-0.20200708190120-187fc06f714e h1:Xi1nil8K2lBOorBS6Ys7+hmUCzH8fr3U9ipdL/IrcEI= github.com/ipfs/go-ds-badger2 v0.1.1-0.20200708190120-187fc06f714e/go.mod h1:lJnws7amT9Ehqzta0gwMrRsURU04caT0iRPr1W8AsOU= @@ -631,8 +642,8 @@ github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CE github.com/ipfs/go-graphsync v0.4.2/go.mod h1:/VmbZTUdUMTbNkgzAiCEucIIAU3BkLE2cZrDCVUhyi0= github.com/ipfs/go-graphsync v0.4.3/go.mod h1:mPOwDYv128gf8gxPFgXnz4fNrSYPsWyqisJ7ych+XDY= github.com/ipfs/go-graphsync v0.6.4/go.mod h1:5WyaeigpNdpiYQuW2vwpuecOoEfB4h747ZGEOKmAGTg= -github.com/ipfs/go-graphsync v0.6.5 h1:YAJl6Yit23PQcaawzb1rPK9PSnbbq2jjMRPpRpJ0Y5U= -github.com/ipfs/go-graphsync v0.6.5/go.mod h1:GdHT8JeuIZ0R4lSjFR16Oe4zPi5dXwKi9zR9ADVlcdk= +github.com/ipfs/go-graphsync v0.6.6 h1:In7jjzvSXlrAUz4OjN41lxYf/dzkf1bVeVxLpwKMRo8= +github.com/ipfs/go-graphsync v0.6.6/go.mod h1:GdHT8JeuIZ0R4lSjFR16Oe4zPi5dXwKi9zR9ADVlcdk= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= @@ -698,8 +709,9 @@ github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSI github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A= -github.com/ipfs/go-log v1.0.4 h1:6nLQdX4W8P9yZZFH7mO+X/PzjN8Laozm/lMJ6esdgzY= github.com/ipfs/go-log v1.0.4/go.mod h1:oDCg2FkjogeFOhqqb+N39l2RpTNPL6F/StPkB3kPgcs= +github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= +github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= @@ -800,12 +812,14 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kabukky/httpscerts v0.0.0-20150320125433-617593d7dcb3/go.mod h1:BYpt4ufZiIGv2nXn4gMxnfKV306n3mWXgNu/d2TqdTU= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= @@ -819,6 +833,8 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg= +github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -835,11 +851,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY= -github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= -github.com/libp2p/go-addr-util v0.0.2 h1:7cWK5cdA5x72jX0g8iLrQWm5TRJZ6CzGdPEhWj7plWU= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= +github.com/libp2p/go-addr-util v0.1.0 h1:acKsntI33w2bTU7tC9a0SaPimJGfSI0bFKC18ChxeVI= +github.com/libp2p/go-addr-util v0.1.0/go.mod h1:6I3ZYuFr2O/9D+SoyM0zEw0EF3YkldtTX406BpdQMqw= github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= @@ -940,9 +955,9 @@ github.com/libp2p/go-libp2p-core v0.7.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJB github.com/libp2p/go-libp2p-core v0.8.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-core v0.8.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-core v0.8.2/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= -github.com/libp2p/go-libp2p-core v0.8.3/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= -github.com/libp2p/go-libp2p-core v0.8.5 h1:aEgbIcPGsKy6zYcC+5AJivYFedhYa4sW7mIpWpUaLKw= github.com/libp2p/go-libp2p-core v0.8.5/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= +github.com/libp2p/go-libp2p-core v0.8.6 h1:3S8g006qG6Tjpj1JdRK2S+TWc2DJQKX/RG9fdLeiLSU= +github.com/libp2p/go-libp2p-core v0.8.6/go.mod h1:dgHr0l0hIKfWpGpqAMbpo19pen9wJfdCGv51mTmdpmM= github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE= github.com/libp2p/go-libp2p-crypto v0.0.2/go.mod h1:eETI5OUfBnvARGOHrJz2eWNyTUxEGZnBxMcbUjfIj4I= github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= @@ -954,8 +969,9 @@ github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFT github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= github.com/libp2p/go-libp2p-discovery v0.4.0/go.mod h1:bZ0aJSrFc/eX2llP0ryhb1kpgkPyTo23SJ5b7UQCMh4= -github.com/libp2p/go-libp2p-discovery v0.5.0 h1:Qfl+e5+lfDgwdrXdu4YNCWyEo3fWuP+WgN9mN0iWviQ= github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= +github.com/libp2p/go-libp2p-discovery v0.5.1 h1:CJylx+h2+4+s68GvrM4pGNyfNhOYviWBPtVv5PA7sfo= +github.com/libp2p/go-libp2p-discovery v0.5.1/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= github.com/libp2p/go-libp2p-host v0.0.1/go.mod h1:qWd+H1yuU0m5CwzAkvbSjqKairayEHdR5MMl7Cwa7Go= github.com/libp2p/go-libp2p-host v0.0.3/go.mod h1:Y/qPyA6C8j2coYyos1dfRm0I8+nvd4TGrDGt4tA7JR8= github.com/libp2p/go-libp2p-interface-connmgr v0.0.1/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k= @@ -1009,20 +1025,22 @@ github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRj github.com/libp2p/go-libp2p-peerstore v0.2.3/go.mod h1:K8ljLdFn590GMttg/luh4caB/3g0vKuY01psze0upRw= github.com/libp2p/go-libp2p-peerstore v0.2.4/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= github.com/libp2p/go-libp2p-peerstore v0.2.6/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= -github.com/libp2p/go-libp2p-peerstore v0.2.7 h1:83JoLxyR9OYTnNfB5vvFqvMUv/xDNa6NoPHnENhBsGw= github.com/libp2p/go-libp2p-peerstore v0.2.7/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= +github.com/libp2p/go-libp2p-peerstore v0.2.8 h1:nJghUlUkFVvyk7ccsM67oFA6kqUkwyCM1G4WPVMCWYA= +github.com/libp2p/go-libp2p-peerstore v0.2.8/go.mod h1:gGiPlXdz7mIHd2vfAsHzBNAMqSDkt2UBFwgcITgw1lA= github.com/libp2p/go-libp2p-pnet v0.2.0 h1:J6htxttBipJujEjz1y0a5+eYoiPcFHhSYHH6na5f0/k= github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= github.com/libp2p/go-libp2p-pubsub v0.3.2-0.20200527132641-c0712c6e92cf/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato= -github.com/libp2p/go-libp2p-pubsub v0.5.0 h1:OzcIuCWyJpOrWH0PTOfvxTzqFur4tiXpY5jXC8OxjyE= -github.com/libp2p/go-libp2p-pubsub v0.5.0/go.mod h1:MKnrsQkFgPcrQs1KVmOXy6Uz2RDQ1xO7dQo/P0Ba+ig= +github.com/libp2p/go-libp2p-pubsub v0.5.3 h1:XCn5xvgA/AKpbbaeqbomfKtQCbT9QsU39tYsVj0IndQ= +github.com/libp2p/go-libp2p-pubsub v0.5.3/go.mod h1:gVOzwebXVdSMDQBTfH8ACO5EJ4SQrvsHqCmYsCZpD0E= github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU= github.com/libp2p/go-libp2p-quic-transport v0.5.0/go.mod h1:IEcuC5MLxvZ5KuHKjRu+dr3LjCT1Be3rcD/4d8JrX8M= -github.com/libp2p/go-libp2p-quic-transport v0.10.0 h1:koDCbWD9CCHwcHZL3/WEvP2A+e/o5/W5L3QS/2SPMA0= github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqUEJqjiiY8xmEuq3HUDS993MkA= +github.com/libp2p/go-libp2p-quic-transport v0.11.2 h1:p1YQDZRHH4Cv2LPtHubqlQ9ggz4CKng/REZuXZbZMhM= +github.com/libp2p/go-libp2p-quic-transport v0.11.2/go.mod h1:wlanzKtIh6pHrq+0U3p3DY9PJfGqxMgPaGKaK5LifwQ= github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFAymLDLm7lxMgU79Q= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.1.1/go.mod h1:VRgKajOyMVgP/F0L5g3kH7SVskp17vFi2xheb5uMJtg= @@ -1050,9 +1068,9 @@ github.com/libp2p/go-libp2p-swarm v0.2.7/go.mod h1:ZSJ0Q+oq/B1JgfPHJAT2HTall+xYR github.com/libp2p/go-libp2p-swarm v0.2.8/go.mod h1:JQKMGSth4SMqonruY0a8yjlPVIkb0mdNSwckW7OYziM= github.com/libp2p/go-libp2p-swarm v0.3.0/go.mod h1:hdv95GWCTmzkgeJpP+GK/9D9puJegb7H57B5hWQR5Kk= github.com/libp2p/go-libp2p-swarm v0.3.1/go.mod h1:hdv95GWCTmzkgeJpP+GK/9D9puJegb7H57B5hWQR5Kk= -github.com/libp2p/go-libp2p-swarm v0.4.3/go.mod h1:mmxP1pGBSc1Arw4F5DIjcpjFAmsRzA1KADuMtMuCT4g= -github.com/libp2p/go-libp2p-swarm v0.5.0 h1:HIK0z3Eqoo8ugmN8YqWAhD2RORgR+3iNXYG4U2PFd1E= github.com/libp2p/go-libp2p-swarm v0.5.0/go.mod h1:sU9i6BoHE0Ve5SKz3y9WfKrh8dUat6JknzUehFx8xW4= +github.com/libp2p/go-libp2p-swarm v0.5.3 h1:hsYaD/y6+kZff1o1Mc56NcuwSg80lIphTS/zDk3mO4M= +github.com/libp2p/go-libp2p-swarm v0.5.3/go.mod h1:NBn7eNW2lu568L7Ns9wdFrOhgRlkRnIDg0FLKbuu3i8= github.com/libp2p/go-libp2p-testing v0.0.1/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= @@ -1061,8 +1079,9 @@ github.com/libp2p/go-libp2p-testing v0.1.0/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eq github.com/libp2p/go-libp2p-testing v0.1.1/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0= github.com/libp2p/go-libp2p-testing v0.1.2-0.20200422005655-8775583591d8/go.mod h1:Qy8sAncLKpwXtS2dSnDOP8ktexIAHKu+J+pnZOFZLTc= github.com/libp2p/go-libp2p-testing v0.3.0/go.mod h1:efZkql4UZ7OVsEfaxNHZPzIehtsBXMrXnCfJIgDti5g= -github.com/libp2p/go-libp2p-testing v0.4.0 h1:PrwHRi0IGqOwVQWR3xzgigSlhlLfxgfXgkHxr77EghQ= github.com/libp2p/go-libp2p-testing v0.4.0/go.mod h1:Q+PFXYoiYFN5CAEG2w3gLPEzotlKsNSbKQ/lImlOWF0= +github.com/libp2p/go-libp2p-testing v0.4.2 h1:IOiA5mMigi+eEjf4J+B7fepDhsjtsoWA9QbsCqbNp5U= +github.com/libp2p/go-libp2p-testing v0.4.2/go.mod h1:Q+PFXYoiYFN5CAEG2w3gLPEzotlKsNSbKQ/lImlOWF0= github.com/libp2p/go-libp2p-tls v0.1.3 h1:twKMhMu44jQO+HgQK9X8NHO5HkeJu2QbhLzLJpa8oNM= github.com/libp2p/go-libp2p-tls v0.1.3/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= github.com/libp2p/go-libp2p-transport v0.0.1/go.mod h1:UzbUs9X+PHOSw7S3ZmeOxfnwaQY5vGDzZmKPod3N3tk= @@ -1073,8 +1092,9 @@ github.com/libp2p/go-libp2p-transport-upgrader v0.0.4/go.mod h1:RGq+tupk+oj7PzL2 github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSoMuyojOdjx73Q47cYD7s5+gZOlns= github.com/libp2p/go-libp2p-transport-upgrader v0.3.0/go.mod h1:i+SKzbRnvXdVbU3D1dwydnTmKRPXiAR/fyvi1dXuL4o= -github.com/libp2p/go-libp2p-transport-upgrader v0.4.2 h1:4JsnbfJzgZeRS9AWN7B9dPqn/LY/HoQTlO9gtdJTIYM= github.com/libp2p/go-libp2p-transport-upgrader v0.4.2/go.mod h1:NR8ne1VwfreD5VIWIU62Agt/J18ekORFU/j1i2y8zvk= +github.com/libp2p/go-libp2p-transport-upgrader v0.4.6 h1:SHt3g0FslnqIkEWF25YOB8UCOCTpGAVvHRWQYJ+veiI= +github.com/libp2p/go-libp2p-transport-upgrader v0.4.6/go.mod h1:JE0WQuQdy+uLZ5zOaI3Nw9dWGYJIA7mywEtP2lMvnyk= github.com/libp2p/go-libp2p-yamux v0.5.1 h1:sX4WQPHMhRxJE5UZTfjEuBvlQWXB5Bo3A2JK9ZJ9EM0= github.com/libp2p/go-libp2p-yamux v0.5.1/go.mod h1:dowuvDu8CRWmr0iqySMiSxK+W0iL5cMVO9S94Y6gkv4= github.com/libp2p/go-maddr-filter v0.0.1/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= @@ -1103,6 +1123,7 @@ github.com/libp2p/go-nat v0.0.5 h1:qxnwkco8RLKqVh1NmjQ+tJ8p8khNLFxuElYG/TwqW4Q= github.com/libp2p/go-nat v0.0.5/go.mod h1:B7NxsVNPZmRLvMOwiEO1scOSyjA56zxYAGv1yQgRkEU= github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.1.3/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= +github.com/libp2p/go-netroute v0.1.5/go.mod h1:V1SR3AaECRkEQCoFFzYwVYWvYIEtlxx89+O3qcpCl4A= github.com/libp2p/go-netroute v0.1.6 h1:ruPJStbYyXVYGQ81uzEDzuvbYRLKRrLvTYd33yomC38= github.com/libp2p/go-netroute v0.1.6/go.mod h1:AqhkMh0VuWmfgtxKPp3Oc1LdU5QSWS7wl0QLhSZqXxQ= github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0= @@ -1117,8 +1138,9 @@ github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQy github.com/libp2p/go-reuseport-transport v0.0.1/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-reuseport-transport v0.0.3/go.mod h1:Spv+MPft1exxARzP2Sruj2Wb5JSyHNncjf1Oi2dEbzM= -github.com/libp2p/go-reuseport-transport v0.0.4 h1:OZGz0RB620QDGpv300n1zaOcKGGAoGVf8h9txtt/1uM= github.com/libp2p/go-reuseport-transport v0.0.4/go.mod h1:trPa7r/7TJK/d+0hdBLOCGvpQQVOU74OXbNCIMkufGw= +github.com/libp2p/go-reuseport-transport v0.0.5 h1:lJzi+vSYbyJj2faPKLxNGWEIBcaV/uJmyvsUxXy2mLw= +github.com/libp2p/go-reuseport-transport v0.0.5/go.mod h1:TC62hhPc8qs5c/RoXDZG6YmjK+/YWUPC0yYmeUecbjc= github.com/libp2p/go-sockaddr v0.0.2/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= github.com/libp2p/go-sockaddr v0.1.0/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= github.com/libp2p/go-sockaddr v0.1.1 h1:yD80l2ZOdGksnOyHrhxDdTDFrf7Oy+v3FMVArIRgZxQ= @@ -1134,8 +1156,9 @@ github.com/libp2p/go-tcp-transport v0.0.4/go.mod h1:+E8HvC8ezEVOxIo3V5vCK9l1y/19 github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc= github.com/libp2p/go-tcp-transport v0.1.1/go.mod h1:3HzGvLbx6etZjnFlERyakbaYPdfjg2pWP97dFZworkY= github.com/libp2p/go-tcp-transport v0.2.0/go.mod h1:vX2U0CnWimU4h0SGSEsg++AzvBcroCGYw28kh94oLe0= -github.com/libp2p/go-tcp-transport v0.2.1 h1:ExZiVQV+h+qL16fzCWtd1HSzPsqWottJ8KXwWaVi8Ns= github.com/libp2p/go-tcp-transport v0.2.1/go.mod h1:zskiJ70MEfWz2MKxvFB/Pv+tPIB1PpPUrHIWQ8aFw7M= +github.com/libp2p/go-tcp-transport v0.2.7 h1:Z8Kc/Kb8tD84WiaH55xAlaEnkqzrp88jSEySCKV4+gg= +github.com/libp2p/go-tcp-transport v0.2.7/go.mod h1:lue9p1b3VmZj1MhhEGB/etmvF/nBQ0X9CW2DutBT3MM= github.com/libp2p/go-testutil v0.0.1/go.mod h1:iAcJc/DKJQanJ5ws2V+u5ywdL2n12X1WbbEG+Jjy69I= github.com/libp2p/go-testutil v0.1.0/go.mod h1:81b2n5HypcVyrCg/MJx4Wgfp/VHojytjVe/gLzZ2Ehc= github.com/libp2p/go-ws-transport v0.0.1/go.mod h1:p3bKjDWHEgtuKKj+2OdPYs5dAPIjtpQGHF2tJfGz7Ww= @@ -1156,8 +1179,9 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lucas-clemente/quic-go v0.11.2/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw= github.com/lucas-clemente/quic-go v0.16.0/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE= -github.com/lucas-clemente/quic-go v0.19.3 h1:eCDQqvGBB+kCTkA0XrAFtNe81FMa0/fn4QSoeAbmiF4= github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= +github.com/lucas-clemente/quic-go v0.21.2 h1:8LqqL7nBQFDUINadW0fHV/xSaCQJgmJC0Gv+qUnjd78= +github.com/lucas-clemente/quic-go v0.21.2/go.mod h1:vF5M1XqhBAHgbjKcJOXY3JZz3GP0T3FQhz/uyOUS38Q= github.com/lucasb-eyer/go-colorful v1.0.3 h1:QIbQXiugsb+q10B+MI+7DI1oQLdmnep86tWFlaaUAac= github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lufia/iostat v1.1.0/go.mod h1:rEPNA0xXgjHQjuI5Cy05sLlS2oRcSlWHRLrvh/AQ+Pg= @@ -1175,10 +1199,17 @@ github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGD github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk= github.com/marten-seemann/qtls v0.9.1/go.mod h1:T1MmAdDPyISzxlK6kjRr0pcZFBVd1OZbBb/j3cvzHhk= -github.com/marten-seemann/qtls v0.10.0 h1:ECsuYUKalRL240rRD4Ri33ISb7kAQ3qGDlrrl55b2pc= github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs= -github.com/marten-seemann/qtls-go1-15 v0.1.1 h1:LIH6K34bPVttyXnUWixk0bzH6/N07VxbSabxn5A5gZQ= github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= +github.com/marten-seemann/qtls-go1-15 v0.1.4/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= +github.com/marten-seemann/qtls-go1-15 v0.1.5 h1:Ci4EIUN6Rlb+D6GmLdej/bCQ4nPYNtVXQB+xjiXE1nk= +github.com/marten-seemann/qtls-go1-15 v0.1.5/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= +github.com/marten-seemann/qtls-go1-16 v0.1.4 h1:xbHbOGGhrenVtII6Co8akhLEdrawwB2iHl5yhJRpnco= +github.com/marten-seemann/qtls-go1-16 v0.1.4/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= +github.com/marten-seemann/qtls-go1-17 v0.1.0-rc.1 h1:/rpmWuGvceLwwWuaKPdjpR4JJEUH0tq64/I3hvzaNLM= +github.com/marten-seemann/qtls-go1-17 v0.1.0-rc.1/go.mod h1:fz4HIxByo+LlWcreM4CZOYNuz3taBQ8rN2X6FqvaWo8= +github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= +github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -1215,6 +1246,12 @@ github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7 github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= +github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= +github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= +github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKoFL8DUUmalo2yJJUCxbPKtm8OKfqr2/FTNU= +github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc= +github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= @@ -1256,8 +1293,9 @@ github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y9 github.com/multiformats/go-multiaddr v0.2.1/go.mod h1:s/Apk6IyxfvMjDafnhJgJ3/46z7tZ04iMk5wP4QMGGE= github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u0xW5UouOmQQrn6a3Y= github.com/multiformats/go-multiaddr v0.3.0/go.mod h1:dF9kph9wfJ+3VLAaeBqo9Of8x4fJxp6ggJGteB8HQTI= -github.com/multiformats/go-multiaddr v0.3.1 h1:1bxa+W7j9wZKTZREySx1vPMs2TqrYWjVZ7zE6/XLG1I= github.com/multiformats/go-multiaddr v0.3.1/go.mod h1:uPbspcUPd5AfaP6ql3ujFY+QWzmBD8uLLL4bXW0XfGc= +github.com/multiformats/go-multiaddr v0.3.3 h1:vo2OTSAqnENB2rLk79pLtr+uhj+VAzSe3uef5q0lRSs= +github.com/multiformats/go-multiaddr v0.3.3/go.mod h1:lCKNGP1EQ1eZ35Za2wlqnabm9xQkib3fyB+nZXHLag0= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.3/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= @@ -1305,6 +1343,7 @@ github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY= github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= @@ -1319,8 +1358,9 @@ github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c h1:5bFTChQxSKNwy github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c/go.mod h1:7qN3Y0BvzRUf4LofcoJplQL10lsFDb4PYlePTVwrP28= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -1330,16 +1370,19 @@ github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= -github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= +github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 h1:CznVS40zms0Dj5he4ERo+fRPtO0qxUk8lA8Xu3ddet0= github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333/go.mod h1:Ag6rSXkHIckQmjFBCweJEEt1mrTPBv8b9W4aU/NQWfI= @@ -1390,8 +1433,11 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A= github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= +github.com/prometheus/client_golang v1.10.0 h1:/o0BDeWzLWXNZ+4q5gXltUvaMpJqckTa+jTNoB+z4cg= +github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -1406,8 +1452,10 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/common v0.18.0 h1:WCVKW7aL6LEe1uryfI9dnEc2ZqNB1Fn0ok930v0iL1Y= +github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/node_exporter v1.0.0-rc.0.0.20200428091818-01054558c289/go.mod h1:FGbBv5OPKjch+jNUJmEQpMZytIdyW0NdBtWFcfSKusc= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1418,8 +1466,11 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.1.0 h1:jhMy6QXfi3y2HEzFoyuCj40z4OZIIHHPtFyCMftmvKA= github.com/prometheus/procfs v0.1.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/raulk/clock v1.1.0 h1:dpb29+UKMbLqiU/jqIJptgLR1nn23HLgMY0sTCDza5Y= github.com/raulk/clock v1.1.0/go.mod h1:3MpVxdZ/ODBQDxbN+kzshf5OSZwPjtMDx6BBXBmOeY0= github.com/raulk/go-watchdog v1.0.1 h1:qgm3DIJAeb+2byneLrQJ7kvmDLGxN2vy3apXyGaDKN4= @@ -1661,8 +1712,9 @@ go.uber.org/dig v1.10.0 h1:yLmDDj9/zuDjv3gz8GQGviXMs9TfysIUMUilCpgzUJY= go.uber.org/dig v1.10.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= go.uber.org/fx v1.9.0 h1:7OAz8ucp35AU8eydejpYG7QrbE8rLKzGhHbZlJi5LYY= go.uber.org/fx v1.9.0/go.mod h1:mFdUyAUuJ3w4jAckiKSKbldsxy1ojpAMJ+dVZg5Y0Aw= -go.uber.org/goleak v1.0.0 h1:qsup4IcBdlmsnGfqyLl4Ntn3C2XCCuKAE7DwHpScyUo= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= @@ -1714,8 +1766,9 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf h1:B2n+Zi5QeYRDAEodEu72OS36gmTWjgpXr2+cWcBW90o= +golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1798,10 +1851,13 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201022231255-08b38378de70/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6 h1:0PC75Fz/kyMGhL0e1QnypqK2kQMqKt9csD1GnMJR+Zk= golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1819,6 +1875,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1884,17 +1941,25 @@ golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200812155832-6a926be9bd1d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200926100807-9d91bd62050c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210511113859-b0526f3d8744 h1:yhBbb4IRs2HS9PPlAg6DMC6mUOKexJBNsLf4Z+6En1Q= +golang.org/x/sys v0.0.0-20210511113859-b0526f3d8744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M= @@ -1953,7 +2018,9 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200711155855-7342f9734a7d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200827010519-17fd2f27a9e3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201112185108-eeaa07dd7696/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2035,8 +2102,10 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -2063,8 +2132,9 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= diff --git a/node/builder_chain.go b/node/builder_chain.go index 1447a4df7..4d9294972 100644 --- a/node/builder_chain.go +++ b/node/builder_chain.go @@ -17,7 +17,6 @@ import ( "github.com/filecoin-project/lotus/chain/market" "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/messagesigner" - "github.com/filecoin-project/lotus/chain/metrics" "github.com/filecoin-project/lotus/chain/stmgr" rpcstmgr "github.com/filecoin-project/lotus/chain/stmgr/rpc" "github.com/filecoin-project/lotus/chain/store" @@ -174,10 +173,6 @@ func ConfigFullNode(c interface{}) Option { ), Override(new(dtypes.Graphsync), modules.Graphsync(cfg.Client.SimultaneousTransfers)), - If(cfg.Metrics.HeadNotifs, - Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)), - ), - If(cfg.Wallet.RemoteBackend != "", Override(new(*remotewallet.RemoteWallet), remotewallet.SetupRemoteWallet(cfg.Wallet.RemoteBackend)), ), diff --git a/node/builder_miner.go b/node/builder_miner.go index 0c0f9d15a..3be055de7 100644 --- a/node/builder_miner.go +++ b/node/builder_miner.go @@ -72,6 +72,7 @@ func ConfigStorageMiner(c interface{}) Option { return Options( ConfigCommon(&cfg.Common, enableLibp2pNode), + Override(new(api.MinerSubsystems), modules.ExtractEnabledMinerSubsystems(cfg.Subsystems)), Override(new(stores.LocalStorage), From(new(repo.LockedRepo))), Override(new(*stores.Local), modules.LocalStorage), Override(new(*stores.Remote), modules.RemoteStorage), diff --git a/node/config/def.go b/node/config/def.go index c5c455c68..e40b0a36e 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -10,6 +10,8 @@ import ( "github.com/filecoin-project/go-state-types/big" miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" ) @@ -100,6 +102,8 @@ func DefaultStorageMiner() *StorageMiner { PreCommitBatchWait: Duration(24 * time.Hour), // this should be less than 31.5 hours, which is the expiration of a precommit ticket PreCommitBatchSlack: Duration(3 * time.Hour), // time buffer for forceful batch submission before sectors/deals in batch would start expiring, higher value will lower the chances for message fail due to expiration + CommittedCapacitySectorLifetime: Duration(builtin.EpochDurationSeconds * policy.GetMaxSectorExpirationExtension()), + AggregateCommits: true, MinCommitBatch: miner5.MinAggregatedSectors, // per FIP13, we must have at least four proofs to aggregate, where 4 is the cross over point where aggregation wins out on single provecommit gas costs MaxCommitBatch: miner5.MaxAggregatedSectors, // maximum 819 sectors, this is the maximum aggregation per FIP13 diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 5d4a91d5f..9f1ee533b 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -245,12 +245,6 @@ see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-f Comment: ``, }, - { - Name: "Metrics", - Type: "Metrics", - - Comment: ``, - }, { Name: "Wallet", Type: "Wallet", @@ -324,20 +318,6 @@ Format: multiaddress`, Comment: ``, }, }, - "Metrics": []DocField{ - { - Name: "Nickname", - Type: "string", - - Comment: ``, - }, - { - Name: "HeadNotifs", - Type: "bool", - - Comment: ``, - }, - }, "MinerAddressConfig": []DocField{ { Name: "PreCommitControl", @@ -562,6 +542,14 @@ Note that setting this number too high in relation to deal ingestion rate may re Comment: `Upper bound on how many sectors can be sealing at the same time when creating new sectors with deals (0 = unlimited)`, }, + { + Name: "CommittedCapacitySectorLifetime", + Type: "Duration", + + Comment: `CommittedCapacitySectorLifetime is the duration a Committed Capacity (CC) sector will +live before it must be extended or converted into sector containing deals before it is +terminated. Value must be between 180-540 days inclusive`, + }, { Name: "WaitDealsDelay", Type: "Duration", diff --git a/node/config/types.go b/node/config/types.go index fe42aa27e..f2a3a26c9 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -23,7 +23,6 @@ type Common struct { type FullNode struct { Common Client Client - Metrics Metrics Wallet Wallet Fees FeeConfig Chainstore Chainstore @@ -141,6 +140,11 @@ type SealingConfig struct { // Upper bound on how many sectors can be sealing at the same time when creating new sectors with deals (0 = unlimited) MaxSealingSectorsForDeals uint64 + // CommittedCapacitySectorLifetime is the duration a Committed Capacity (CC) sector will + // live before it must be extended or converted into sector containing deals before it is + // terminated. Value must be between 180-540 days inclusive + CommittedCapacitySectorLifetime Duration + // Period of time that a newly created sector will wait for more deals to be packed in to before it starts to seal. // Sectors which are fully filled will start sealing immediately WaitDealsDelay Duration @@ -298,12 +302,6 @@ type Splitstore struct { } // // Full Node - -type Metrics struct { - Nickname string - HeadNotifs bool -} - type Client struct { UseIpfs bool IpfsOnlineMode bool diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 9db6a3775..0fbd12111 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -23,8 +23,8 @@ import ( "github.com/filecoin-project/go-address" datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/piecestore" - retrievalmarket "github.com/filecoin-project/go-fil-markets/retrievalmarket" - storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" @@ -51,6 +51,8 @@ type StorageMinerAPI struct { api.Common api.Net + EnabledSubsystems api.MinerSubsystems + Full api.FullNode LocalStore *stores.Local RemoteStore *stores.Remote @@ -703,4 +705,8 @@ func (sm *StorageMinerAPI) ComputeProof(ctx context.Context, ssi []builtin.Secto return sm.Epp.ComputeProof(ctx, ssi, rand) } +func (sm *StorageMinerAPI) RuntimeSubsystems(context.Context) (res api.MinerSubsystems, err error) { + return sm.EnabledSubsystems, nil +} + var _ api.StorageMiner = &StorageMinerAPI{} diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 3a3914e0c..5497eab58 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -1007,3 +1007,19 @@ func mutateCfg(r repo.LockedRepo, mutator func(*config.StorageMiner)) error { return multierr.Combine(typeErr, setConfigErr) } + +func ExtractEnabledMinerSubsystems(cfg config.MinerSubsystemConfig) (res api.MinerSubsystems) { + if cfg.EnableMining { + res = append(res, api.SubsystemMining) + } + if cfg.EnableSealing { + res = append(res, api.SubsystemSealing) + } + if cfg.EnableSectorStorage { + res = append(res, api.SubsystemSectorStorage) + } + if cfg.EnableMarkets { + res = append(res, api.SubsystemMarkets) + } + return res +} diff --git a/scripts/lotus-chainwatch.service b/scripts/lotus-chainwatch.service deleted file mode 100644 index e121cb1d1..000000000 --- a/scripts/lotus-chainwatch.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Chainwatch -After=lotus-daemon.service -Requires=lotus-daemon.service - -[Service] -Environment=GOLOG_FILE="/var/log/lotus/chainwatch.log" -Environment=GOLOG_LOG_FMT="json" -Environment=LOTUS_DB="" -Environment=LOTUS_PATH="%h/.lotus" -EnvironmentFile=-/etc/lotus/chainwatch.env -ExecStart=/usr/local/bin/lotus-chainwatch run - -[Install] -WantedBy=multi-user.target diff --git a/storage/miner.go b/storage/miner.go index cdacc2734..59c64eb41 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -27,7 +27,6 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" - "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/types" @@ -179,19 +178,17 @@ func (m *Miner) Run(ctx context.Context) error { adaptedAPI = NewSealingAPIAdapter(m.api) // Instantiate a precommit policy. - defaultDuration = policy.GetMaxSectorExpirationExtension() - (md.WPoStProvingPeriod * 2) + cfg = sealing.GetSealingConfigFunc(m.getSealConfig) provingBoundary = md.PeriodStart % md.WPoStProvingPeriod + provingBuffer = md.WPoStProvingPeriod * 2 // TODO: Maybe we update this policy after actor upgrades? - pcp = sealing.NewBasicPreCommitPolicy(adaptedAPI, defaultDuration, provingBoundary) + pcp = sealing.NewBasicPreCommitPolicy(adaptedAPI, cfg, provingBoundary, provingBuffer) // address selector. as = func(ctx context.Context, mi miner.MinerInfo, use api.AddrUse, goodFunds, minFunds abi.TokenAmount) (address.Address, abi.TokenAmount, error) { return m.addrSel.AddressFor(ctx, m.api, mi, use, goodFunds, minFunds) } - - // sealing configuration. - cfg = sealing.GetSealingConfigFunc(m.getSealConfig) ) // Instantiate the sealing FSM.