diff --git a/.circleci/config.yml b/.circleci/config.yml index 7987060d4..90db3a626 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2.1 orbs: - aws-cli: circleci/aws-cli@1.3.2 - docker: circleci/docker@2.1.4 + aws-cli: circleci/aws-cli@4.1.1 + docker: circleci/docker@2.3.0 executors: golang: @@ -70,8 +70,6 @@ commands: name: Restore parameters cache keys: - 'v26-2k-lotus-params' - paths: - - /var/tmp/filecoin-proof-parameters/ - run: ./lotus fetch-params 2048 - save_cache: name: Save parameters cache @@ -96,6 +94,7 @@ commands: git fetch --all install-ubuntu-deps: steps: + - run: sudo apt install curl ca-certificates gnupg - run: sudo apt-get update - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev check-go-version: @@ -143,9 +142,9 @@ jobs: Run tests with gotestsum. working_directory: ~/lotus parameters: &test-params - executor: - type: executor - default: golang + resource_class: + type: string + default: medium+ go-test-flags: type: string default: "-timeout 20m" @@ -164,7 +163,14 @@ jobs: type: string default: unit description: Test suite name to report to CircleCI. - executor: << parameters.executor >> + docker: + - image: cimg/go:1.20 + environment: + LOTUS_HARMONYDB_HOSTS: yugabyte + - image: yugabytedb/yugabyte:2.18.0.0-b65 + command: bin/yugabyted start --daemon=false + name: yugabyte + resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps - attach_workspace: @@ -182,6 +188,8 @@ jobs: command: | mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts + dockerize -wait tcp://yugabyte:5433 -timeout 3m + env gotestsum \ --format standard-verbose \ --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ @@ -209,7 +217,9 @@ jobs: Branch on github.com/filecoin-project/test-vectors to checkout and test with. If empty (the default) the commit defined by the git submodule is used. - executor: << parameters.executor >> + docker: + - image: cimg/go:1.20 + resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps - attach_workspace: @@ -396,15 +406,14 @@ jobs: Run golangci-lint. working_directory: ~/lotus parameters: - executor: - type: executor - default: golang args: type: string default: '' description: | Arguments to pass to golangci-lint - executor: << parameters.executor >> + docker: + - image: cimg/go:1.20 + resource_class: medium+ steps: - install-ubuntu-deps - attach_workspace: @@ -575,7 +584,7 @@ workflows: - build suite: itest-deals_concurrent target: "./itests/deals_concurrent_test.go" - executor: golang-2xl + resource_class: 2xlarge - test: name: test-itest-deals_invalid_utf8_label requires: @@ -768,6 +777,18 @@ workflows: - build suite: itest-get_messages_in_ts target: "./itests/get_messages_in_ts_test.go" + - test: + name: test-itest-harmonydb + requires: + - build + suite: itest-harmonydb + target: "./itests/harmonydb_test.go" + - test: + name: test-itest-harmonytask + requires: + - build + suite: itest-harmonytask + target: "./itests/harmonytask_test.go" - test: name: test-itest-lite_migration requires: @@ -970,14 +991,14 @@ workflows: - build suite: itest-wdpost_worker_config target: "./itests/wdpost_worker_config_test.go" - executor: golang-2xl + resource_class: 2xlarge - test: name: test-itest-worker requires: - build suite: itest-worker target: "./itests/worker_test.go" - executor: golang-2xl + resource_class: 2xlarge - test: name: test-itest-worker_upgrade requires: @@ -990,7 +1011,7 @@ workflows: - build suite: utest-unit-cli target: "./cli/... ./cmd/... ./api/..." - executor: golang-2xl + resource_class: 2xlarge get-params: true - test: name: test-unit-node @@ -1004,7 +1025,7 @@ workflows: - build suite: utest-unit-rest target: "./blockstore/... ./build/... ./chain/... ./conformance/... ./gateway/... ./journal/... ./lib/... ./markets/... ./paychmgr/... ./tools/..." - executor: golang-2xl + resource_class: 2xlarge - test: name: test-unit-storage requires: diff --git a/.circleci/gen.go b/.circleci/gen.go index 93f409df2..19329247a 100644 --- a/.circleci/gen.go +++ b/.circleci/gen.go @@ -10,11 +10,25 @@ import ( "text/template" ) +var GoVersion = "" // from init below. Ex: 1.19.7 + //go:generate go run ./gen.go .. //go:embed template.yml var templateFile embed.FS +func init() { + b, err := os.ReadFile("../go.mod") + if err != nil { + panic("cannot find go.mod in parent folder") + } + for _, line := range strings.Split(string(b), "\n") { + if strings.HasPrefix(line, "go ") { + GoVersion = line[3:] + } + } +} + type ( dirs = []string suite = string @@ -111,6 +125,7 @@ func main() { Networks []string ItestFiles []string UnitSuites map[string]string + GoVersion string } in := data{ Networks: []string{"mainnet", "butterflynet", "calibnet", "debug"}, @@ -125,6 +140,7 @@ func main() { } return ret }(), + GoVersion: GoVersion, } out, err := os.Create("./config.yml") diff --git a/.circleci/template.yml b/.circleci/template.yml index 0b244d013..9011f1a86 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -1,7 +1,7 @@ version: 2.1 orbs: - aws-cli: circleci/aws-cli@1.3.2 - docker: circleci/docker@2.1.4 + aws-cli: circleci/aws-cli@4.1.1 + docker: circleci/docker@2.3.0 executors: golang: @@ -70,8 +70,6 @@ commands: name: Restore parameters cache keys: - 'v26-2k-lotus-params' - paths: - - /var/tmp/filecoin-proof-parameters/ - run: ./lotus fetch-params 2048 - save_cache: name: Save parameters cache @@ -96,6 +94,7 @@ commands: git fetch --all install-ubuntu-deps: steps: + - run: sudo apt install curl ca-certificates gnupg - run: sudo apt-get update - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev check-go-version: @@ -143,9 +142,9 @@ jobs: Run tests with gotestsum. working_directory: ~/lotus parameters: &test-params - executor: - type: executor - default: golang + resource_class: + type: string + default: medium+ go-test-flags: type: string default: "-timeout 20m" @@ -164,7 +163,14 @@ jobs: type: string default: unit description: Test suite name to report to CircleCI. - executor: << parameters.executor >> + docker: + - image: cimg/go:[[ .GoVersion]] + environment: + LOTUS_HARMONYDB_HOSTS: yugabyte + - image: yugabytedb/yugabyte:2.18.0.0-b65 + command: bin/yugabyted start --daemon=false + name: yugabyte + resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps - attach_workspace: @@ -182,6 +188,8 @@ jobs: command: | mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts + dockerize -wait tcp://yugabyte:5433 -timeout 3m + env gotestsum \ --format standard-verbose \ --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ @@ -209,7 +217,9 @@ jobs: Branch on github.com/filecoin-project/test-vectors to checkout and test with. If empty (the default) the commit defined by the git submodule is used. - executor: << parameters.executor >> + docker: + - image: cimg/go:[[ .GoVersion]] + resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps - attach_workspace: @@ -396,15 +406,14 @@ jobs: Run golangci-lint. working_directory: ~/lotus parameters: - executor: - type: executor - default: golang args: type: string default: '' description: | Arguments to pass to golangci-lint - executor: << parameters.executor >> + docker: + - image: cimg/go:[[ .GoVersion]] + resource_class: medium+ steps: - install-ubuntu-deps - attach_workspace: @@ -543,7 +552,7 @@ workflows: suite: itest-[[ $name ]] target: "./itests/[[ $file ]]" [[- if or (eq $name "worker") (eq $name "deals_concurrent") (eq $name "wdpost_worker_config")]] - executor: golang-2xl + resource_class: 2xlarge [[- end]] [[- if or (eq $name "wdpost") (eq $name "sector_pledge")]] get-params: true @@ -561,11 +570,11 @@ workflows: get-params: true [[- end -]] [[- if eq $suite "unit-cli"]] - executor: golang-2xl + resource_class: 2xlarge get-params: true [[- end -]] [[- if eq $suite "unit-rest"]] - executor: golang-2xl + resource_class: 2xlarge [[- end -]] [[- end]] - test: diff --git a/.gitignore b/.gitignore index 01a3a03ff..c40a76fd0 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ /lotus-chainwatch /lotus-shed /lotus-sim +/lotus-provider /lotus-townhall /lotus-fountain /lotus-stats diff --git a/CHANGELOG.md b/CHANGELOG.md index 6866631d0..6fb092922 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,7 @@ The full list of [protocol improvements delivered in the network upgrade can be ## ☢️ Upgrade Warnings ☢️ - Read through the [changelog of the mandatory v1.24.0 release](https://github.com/filecoin-project/lotus/releases/tag/v1.24.0). Especially the `Migration` and `v12 Builtin Actor Bundle` sections. -- Please remove and clone a new Lotus repo (`git clone https://github.com/filecoin-project/lotus.git`) when upgrading to this release. +- Please remove and clone a new Lotus repo (`git clone https://github.com/filecoin-project/lotus.git`) when upgrading to this release. - This feature release requires a minimum Go version of v1.20.7 or higher to successfully build Lotus. Go version 1.21.x is not supported yet. - EthRPC providers, please check out the [new tracing API to Lotus RPC](https://github.com/filecoin-project/lotus/pull/11100) @@ -190,7 +190,7 @@ account bafk2bzaceboftg75mdiba7xbo2i3uvgtca4brhnr3u5ptihonixgpnrvhpxoa init bafk2bzacebllyegx5r6lggf6ymyetbp7amacwpuxakhtjvjtvoy2bfkzk3vms ``` -## Migration +## Migration We are expecting a heavier than normal state migration for this upgrade due to the amount of state changes introduced for miner sector info. (This is a similar migration as the Shark upgrade, however, we have introduced a couple of migration performance optimizations since then for a smoother upgrade experience.) @@ -209,7 +209,7 @@ You can check out the [tutorial for benchmarking the network migration here.](ht ## BREAKING CHANGE -There is a new protocol limit on how many partition could be submited in one PoSt - if you have any customized tooling for batching PoSts, please update accordingly. +There is a new protocol limit on how many partition could be submited in one PoSt - if you have any customized tooling for batching PoSts, please update accordingly. - feat: limit PoSted partitions to 3 ([filecoin-project/lotus#11327](https://github.com/filecoin-project/lotus/pull/11327)) ## New features @@ -221,7 +221,7 @@ There is a new protocol limit on how many partition could be submited in one PoS ## Improvements - Backport: feat: sealing: Switch to calling PreCommitSectorBatch2 ([filecoin-project/lotus#11215](https://github.com/filecoin-project/lotus/pull/11215)) -- updated the boostrap nodes +- updated the boostrap nodes ## Dependencies - github.com/filecoin-project/go-amt-ipld/v4 (v4.0.0 -> v4.2.0) @@ -231,9 +231,9 @@ There is a new protocol limit on how many partition could be submited in one PoS - chore: deps: update libp2p to v0.30.0 #11434 -## Snapshots +## Snapshots -The [Forest team](https://filecoinproject.slack.com/archives/C029LPZ5N73) at Chainsafe has launched a brand new lightweight snapshot service that is backed up by forest nodes! This is a great alternative service along with the fil-infra one, and it is compatible with lotus! We recommend lotus users to check it out [here](https://docs.filecoin.io/networks/mainnet#resources)! +The [Forest team](https://filecoinproject.slack.com/archives/C029LPZ5N73) at Chainsafe has launched a brand new lightweight snapshot service that is backed up by forest nodes! This is a great alternative service along with the fil-infra one, and it is compatible with lotus! We recommend lotus users to check it out [here](https://docs.filecoin.io/networks/mainnet#resources)! diff --git a/Dockerfile b/Dockerfile index 00930fb0f..c9750a71f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -109,6 +109,7 @@ COPY --from=lotus-builder /opt/filecoin/lotus-wallet /usr/local/bin/ COPY --from=lotus-builder /opt/filecoin/lotus-gateway /usr/local/bin/ COPY --from=lotus-builder /opt/filecoin/lotus-miner /usr/local/bin/ COPY --from=lotus-builder /opt/filecoin/lotus-worker /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/lotus-provider /usr/local/bin/ COPY --from=lotus-builder /opt/filecoin/lotus-stats /usr/local/bin/ COPY --from=lotus-builder /opt/filecoin/lotus-fountain /usr/local/bin/ @@ -117,11 +118,13 @@ RUN mkdir /var/lib/lotus RUN mkdir /var/lib/lotus-miner RUN mkdir /var/lib/lotus-worker RUN mkdir /var/lib/lotus-wallet +RUN mkdir /var/lib/lotus-provider RUN chown fc: /var/tmp/filecoin-proof-parameters RUN chown fc: /var/lib/lotus RUN chown fc: /var/lib/lotus-miner RUN chown fc: /var/lib/lotus-worker RUN chown fc: /var/lib/lotus-wallet +RUN chown fc: /var/lib/lotus-provider VOLUME /var/tmp/filecoin-proof-parameters @@ -129,6 +132,7 @@ VOLUME /var/lib/lotus VOLUME /var/lib/lotus-miner VOLUME /var/lib/lotus-worker VOLUME /var/lib/lotus-wallet +VOLUME /var/lib/lotus-provider EXPOSE 1234 EXPOSE 2345 diff --git a/Makefile b/Makefile index b94c13c0d..c3c46f71e 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,7 @@ CLEAN+=build/.update-modules deps: $(BUILD_DEPS) .PHONY: deps -build-devnets: build lotus-seed lotus-shed +build-devnets: build lotus-seed lotus-shed lotus-provider .PHONY: build-devnets debug: GOFLAGS+=-tags=debug @@ -97,6 +97,13 @@ lotus-miner: $(BUILD_DEPS) .PHONY: lotus-miner BINS+=lotus-miner +lotus-provider: $(BUILD_DEPS) + rm -f lotus-provider + $(GOCC) build $(GOFLAGS) -o lotus-provider ./cmd/lotus-provider + +lp2k: GOFLAGS+=-tags=2k +lp2k: lotus-provider + lotus-worker: $(BUILD_DEPS) rm -f lotus-worker $(GOCC) build $(GOFLAGS) -o lotus-worker ./cmd/lotus-worker @@ -115,13 +122,13 @@ lotus-gateway: $(BUILD_DEPS) .PHONY: lotus-gateway BINS+=lotus-gateway -build: lotus lotus-miner lotus-worker +build: lotus lotus-miner lotus-worker @[[ $$(type -P "lotus") ]] && echo "Caution: you have \ an existing lotus binary in your PATH. This may cause problems if you don't run 'sudo make install'" || true .PHONY: build -install: install-daemon install-miner install-worker +install: install-daemon install-miner install-worker install-provider install-daemon: install -C ./lotus /usr/local/bin/lotus @@ -129,6 +136,9 @@ install-daemon: install-miner: install -C ./lotus-miner /usr/local/bin/lotus-miner +install-provider: + install -C ./lotus-provider /usr/local/bin/lotus-provider + install-worker: install -C ./lotus-worker /usr/local/bin/lotus-worker @@ -144,6 +154,9 @@ uninstall-daemon: uninstall-miner: rm -f /usr/local/bin/lotus-miner +uninstall-provider: + rm -f /usr/local/bin/lotus-provider + uninstall-worker: rm -f /usr/local/bin/lotus-worker @@ -241,6 +254,14 @@ install-miner-service: install-miner install-daemon-service @echo @echo "lotus-miner service installed. Don't forget to run 'sudo systemctl start lotus-miner' to start it and 'sudo systemctl enable lotus-miner' for it to be enabled on startup." +install-provider-service: install-provider install-daemon-service + mkdir -p /etc/systemd/system + mkdir -p /var/log/lotus + install -C -m 0644 ./scripts/lotus-provider.service /etc/systemd/system/lotus-provider.service + systemctl daemon-reload + @echo + @echo "lotus-provider service installed. Don't forget to run 'sudo systemctl start lotus-provider' to start it and 'sudo systemctl enable lotus-provider' for it to be enabled on startup." + install-main-services: install-miner-service install-all-services: install-main-services @@ -259,6 +280,12 @@ clean-miner-service: rm -f /etc/systemd/system/lotus-miner.service systemctl daemon-reload +clean-provider-service: + -systemctl stop lotus-provider + -systemctl disable lotus-provider + rm -f /etc/systemd/system/lotus-provider.service + systemctl daemon-reload + clean-main-services: clean-daemon-service clean-all-services: clean-main-services @@ -294,7 +321,8 @@ actors-code-gen: $(GOCC) run ./chain/actors/agen $(GOCC) fmt ./... -actors-gen: actors-code-gen fiximports +actors-gen: actors-code-gen + ./scripts/fiximports .PHONY: actors-gen bundle-gen: @@ -328,7 +356,7 @@ docsgen-md-bin: api-gen actors-gen docsgen-openrpc-bin: api-gen actors-gen $(GOCC) build $(GOFLAGS) -o docgen-openrpc ./api/docgen-openrpc/cmd -docsgen-md: docsgen-md-full docsgen-md-storage docsgen-md-worker +docsgen-md: docsgen-md-full docsgen-md-storage docsgen-md-worker docsgen-md-provider docsgen-md-full: docsgen-md-bin ./docgen-md "api/api_full.go" "FullNode" "api" "./api" > documentation/en/api-v1-unstable-methods.md @@ -337,6 +365,8 @@ docsgen-md-storage: docsgen-md-bin ./docgen-md "api/api_storage.go" "StorageMiner" "api" "./api" > documentation/en/api-v0-methods-miner.md docsgen-md-worker: docsgen-md-bin ./docgen-md "api/api_worker.go" "Worker" "api" "./api" > documentation/en/api-v0-methods-worker.md +docsgen-md-provider: docsgen-md-bin + ./docgen-md "api/api_lp.go" "Provider" "api" "./api" > documentation/en/api-v0-methods-provider.md docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker docsgen-openrpc-gateway @@ -354,21 +384,23 @@ docsgen-openrpc-gateway: docsgen-openrpc-bin fiximports: ./scripts/fiximports -gen: actors-code-gen type-gen cfgdoc-gen docsgen api-gen circleci fiximports +gen: actors-code-gen type-gen cfgdoc-gen docsgen api-gen circleci + ./scripts/fiximports @echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO RUN 'make docsgen-cli'" .PHONY: gen jen: gen -snap: lotus lotus-miner lotus-worker +snap: lotus lotus-miner lotus-worker lotus-provider snapcraft # snapcraft upload ./lotus_*.snap # separate from gen because it needs binaries -docsgen-cli: lotus lotus-miner lotus-worker +docsgen-cli: lotus lotus-miner lotus-worker lotus-provider python3 ./scripts/generate-lotus-cli.py ./lotus config default > documentation/en/default-lotus-config.toml ./lotus-miner config default > documentation/en/default-lotus-miner-config.toml + ./lotus-provider config default > documentation/en/default-lotus-provider-config.toml .PHONY: docsgen-cli print-%: diff --git a/api/api_lp.go b/api/api_lp.go new file mode 100644 index 000000000..8b58379f8 --- /dev/null +++ b/api/api_lp.go @@ -0,0 +1,10 @@ +package api + +import "context" + +type LotusProvider interface { + Version(context.Context) (Version, error) //perm:admin + + // Trigger shutdown + Shutdown(context.Context) error //perm:admin +} diff --git a/api/client/client.go b/api/client/client.go index 8b159c5b1..4d51221f9 100644 --- a/api/client/client.go +++ b/api/client/client.go @@ -15,6 +15,16 @@ import ( "github.com/filecoin-project/lotus/lib/rpcenc" ) +// NewProviderRpc creates a new http jsonrpc client. +func NewProviderRpc(ctx context.Context, addr string, requestHeader http.Header) (api.LotusProvider, jsonrpc.ClientCloser, error) { + var res v1api.LotusProviderStruct + + closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin", + api.GetInternalStructs(&res), requestHeader, jsonrpc.WithErrors(api.RPCErrors)) + + return &res, closer, err +} + // NewCommonRPCV0 creates a new http jsonrpc client. func NewCommonRPCV0(ctx context.Context, addr string, requestHeader http.Header) (api.CommonNet, jsonrpc.ClientCloser, error) { var res v0api.CommonNetStruct diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index 70024d3db..b31c25b86 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -428,6 +428,10 @@ func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []r i = &api.GatewayStruct{} t = reflect.TypeOf(new(struct{ api.Gateway })).Elem() permStruct = append(permStruct, reflect.TypeOf(api.GatewayStruct{}.Internal)) + case "Provider": + i = &api.LotusProviderStruct{} + t = reflect.TypeOf(new(struct{ api.LotusProvider })).Elem() + permStruct = append(permStruct, reflect.TypeOf(api.LotusProviderStruct{}.Internal)) default: panic("unknown type") } diff --git a/api/permissioned.go b/api/permissioned.go index 72d2239ee..f189cd78f 100644 --- a/api/permissioned.go +++ b/api/permissioned.go @@ -41,6 +41,12 @@ func PermissionedWorkerAPI(a Worker) Worker { return &out } +func PermissionedAPI[T, P any](a T) *P { + var out P + permissionedProxies(a, &out) + return &out +} + func PermissionedWalletAPI(a Wallet) Wallet { var out WalletStruct permissionedProxies(a, &out) diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 589ae8f56..c07fc3a61 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -827,6 +827,19 @@ type GatewayMethods struct { type GatewayStub struct { } +type LotusProviderStruct struct { + Internal LotusProviderMethods +} + +type LotusProviderMethods struct { + Shutdown func(p0 context.Context) error `perm:"admin"` + + Version func(p0 context.Context) (Version, error) `perm:"admin"` +} + +type LotusProviderStub struct { +} + type NetStruct struct { Internal NetMethods } @@ -5188,6 +5201,28 @@ func (s *GatewayStub) Web3ClientVersion(p0 context.Context) (string, error) { return "", ErrNotSupported } +func (s *LotusProviderStruct) Shutdown(p0 context.Context) error { + if s.Internal.Shutdown == nil { + return ErrNotSupported + } + return s.Internal.Shutdown(p0) +} + +func (s *LotusProviderStub) Shutdown(p0 context.Context) error { + return ErrNotSupported +} + +func (s *LotusProviderStruct) Version(p0 context.Context) (Version, error) { + if s.Internal.Version == nil { + return *new(Version), ErrNotSupported + } + return s.Internal.Version(p0) +} + +func (s *LotusProviderStub) Version(p0 context.Context) (Version, error) { + return *new(Version), ErrNotSupported +} + func (s *NetStruct) ID(p0 context.Context) (peer.ID, error) { if s.Internal.ID == nil { return *new(peer.ID), ErrNotSupported @@ -7416,6 +7451,7 @@ var _ CommonNet = new(CommonNetStruct) var _ EthSubscriber = new(EthSubscriberStruct) var _ FullNode = new(FullNodeStruct) var _ Gateway = new(GatewayStruct) +var _ LotusProvider = new(LotusProviderStruct) var _ Net = new(NetStruct) var _ Signable = new(SignableStruct) var _ StorageMiner = new(StorageMinerStruct) diff --git a/api/v1api/latest.go b/api/v1api/latest.go index aefb1543b..b8eeed2de 100644 --- a/api/v1api/latest.go +++ b/api/v1api/latest.go @@ -12,3 +12,5 @@ type RawFullNodeAPI FullNode func PermissionedFullAPI(a FullNode) FullNode { return api.PermissionedFullAPI(a) } + +type LotusProviderStruct = api.LotusProviderStruct diff --git a/api/version.go b/api/version.go index 9c2113578..e968bf93b 100644 --- a/api/version.go +++ b/api/version.go @@ -59,6 +59,8 @@ var ( MinerAPIVersion0 = newVer(1, 5, 0) WorkerAPIVersion0 = newVer(1, 7, 0) + + ProviderAPIVersion0 = newVer(1, 0, 0) ) //nolint:varcheck,deadcode diff --git a/blockstore/splitstore/splitstore_test.go b/blockstore/splitstore/splitstore_test.go index 63e77b47e..1b821654d 100644 --- a/blockstore/splitstore/splitstore_test.go +++ b/blockstore/splitstore/splitstore_test.go @@ -3,9 +3,9 @@ package splitstore import ( "context" + "crypto/rand" "errors" "fmt" - "math/rand" "sync" "sync/atomic" "testing" diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index 3aa6bc16b..3c30e81a9 100644 Binary files a/build/openrpc/full.json.gz and b/build/openrpc/full.json.gz differ diff --git a/build/openrpc/gateway.json.gz b/build/openrpc/gateway.json.gz index 64765a403..82d638d7e 100644 Binary files a/build/openrpc/gateway.json.gz and b/build/openrpc/gateway.json.gz differ diff --git a/build/openrpc/miner.json.gz b/build/openrpc/miner.json.gz index 4702c5867..52f021026 100644 Binary files a/build/openrpc/miner.json.gz and b/build/openrpc/miner.json.gz differ diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index ad518528c..9b4f0243d 100644 Binary files a/build/openrpc/worker.json.gz and b/build/openrpc/worker.json.gz differ diff --git a/build/version.go b/build/version.go index 0758686f6..6ec1ecd7a 100644 --- a/build/version.go +++ b/build/version.go @@ -37,7 +37,7 @@ func BuildTypeString() string { } // BuildVersion is the local build version -const BuildVersion = "1.25.1-dev" +const BuildVersion = "1.25.3-dev" func UserVersion() string { if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" { diff --git a/chain/actors/policy/policy.go b/chain/actors/policy/policy.go index a0e4728fe..6d2b41154 100644 --- a/chain/actors/policy/policy.go +++ b/chain/actors/policy/policy.go @@ -867,6 +867,24 @@ func AggregatePreCommitNetworkFee(nwVer network.Version, aggregateSize int, base } } +var PoStToSealMap map[abi.RegisteredPoStProof]abi.RegisteredSealProof + +func init() { + PoStToSealMap = make(map[abi.RegisteredPoStProof]abi.RegisteredSealProof) + for sealProof, info := range abi.SealProofInfos { + PoStToSealMap[info.WinningPoStProof] = sealProof + PoStToSealMap[info.WindowPoStProof] = sealProof + } +} + +func GetSealProofFromPoStProof(postProof abi.RegisteredPoStProof) (abi.RegisteredSealProof, error) { + sealProof, exists := PoStToSealMap[postProof] + if !exists { + return 0, xerrors.New("no corresponding RegisteredSealProof for the given RegisteredPoStProof") + } + return sealProof, nil +} + func min(a, b int) int { if a < b { return a diff --git a/chain/actors/policy/policy.go.template b/chain/actors/policy/policy.go.template index 8803c97e6..d13518e0a 100644 --- a/chain/actors/policy/policy.go.template +++ b/chain/actors/policy/policy.go.template @@ -343,9 +343,26 @@ func AggregatePreCommitNetworkFee(nwVer network.Version, aggregateSize int, base } } +var PoStToSealMap map[abi.RegisteredPoStProof]abi.RegisteredSealProof +func init() { + PoStToSealMap = make(map[abi.RegisteredPoStProof]abi.RegisteredSealProof) + for sealProof, info := range abi.SealProofInfos { + PoStToSealMap[info.WinningPoStProof] = sealProof + PoStToSealMap[info.WindowPoStProof] = sealProof + } +} + +func GetSealProofFromPoStProof(postProof abi.RegisteredPoStProof) (abi.RegisteredSealProof, error) { + sealProof, exists := PoStToSealMap[postProof] + if !exists { + return 0, xerrors.New("no corresponding RegisteredSealProof for the given RegisteredPoStProof") + } + return sealProof, nil +} + func min(a, b int) int { if a < b { return a } return b -} \ No newline at end of file +} diff --git a/chain/consensus/common.go b/chain/consensus/common.go index 1d9fb3646..a7e5c40d2 100644 --- a/chain/consensus/common.go +++ b/chain/consensus/common.go @@ -362,7 +362,8 @@ func CreateBlockHeader(ctx context.Context, sm *stmgr.StateManager, pts *types.T var blsMsgCids, secpkMsgCids []cid.Cid var blsSigs []crypto.Signature nv := sm.GetNetworkVersion(ctx, bt.Epoch) - for _, msg := range bt.Messages { + for _, msgTmp := range bt.Messages { + msg := msgTmp if msg.Signature.Type == crypto.SigTypeBLS { blsSigs = append(blsSigs, msg.Signature) blsMessages = append(blsMessages, &msg.Message) diff --git a/chain/exchange/cbor_gen.go b/chain/exchange/cbor_gen.go index e66b6d798..71c75869d 100644 --- a/chain/exchange/cbor_gen.go +++ b/chain/exchange/cbor_gen.go @@ -306,9 +306,9 @@ func (t *Response) UnmarshalCBOR(r io.Reader) (err error) { return nil } -var lengthBufCompactedMessages = []byte{132} +var lengthBufCompactedMessagesCBOR = []byte{132} -func (t *CompactedMessages) MarshalCBOR(w io.Writer) error { +func (t *CompactedMessagesCBOR) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err @@ -316,12 +316,12 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error { cw := cbg.NewCborWriter(w) - if _, err := cw.Write(lengthBufCompactedMessages); err != nil { + if _, err := cw.Write(lengthBufCompactedMessagesCBOR); err != nil { return err } // t.Bls ([]*types.Message) (slice) - if len(t.Bls) > cbg.MaxLength { + if len(t.Bls) > 150000 { return xerrors.Errorf("Slice value in field t.Bls was too long") } @@ -334,7 +334,7 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error { } } - // t.BlsIncludes ([][]uint64) (slice) + // t.BlsIncludes ([]exchange.messageIndices) (slice) if len(t.BlsIncludes) > cbg.MaxLength { return xerrors.Errorf("Slice value in field t.BlsIncludes was too long") } @@ -343,24 +343,13 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error { return err } for _, v := range t.BlsIncludes { - if len(v) > cbg.MaxLength { - return xerrors.Errorf("Slice value in field v was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(v))); err != nil { + if err := v.MarshalCBOR(cw); err != nil { return err } - for _, v := range v { - - if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(v)); err != nil { - return err - } - - } } // t.Secpk ([]*types.SignedMessage) (slice) - if len(t.Secpk) > cbg.MaxLength { + if len(t.Secpk) > 150000 { return xerrors.Errorf("Slice value in field t.Secpk was too long") } @@ -373,7 +362,7 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error { } } - // t.SecpkIncludes ([][]uint64) (slice) + // t.SecpkIncludes ([]exchange.messageIndices) (slice) if len(t.SecpkIncludes) > cbg.MaxLength { return xerrors.Errorf("Slice value in field t.SecpkIncludes was too long") } @@ -382,26 +371,15 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error { return err } for _, v := range t.SecpkIncludes { - if len(v) > cbg.MaxLength { - return xerrors.Errorf("Slice value in field v was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(v))); err != nil { + if err := v.MarshalCBOR(cw); err != nil { return err } - for _, v := range v { - - if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(v)); err != nil { - return err - } - - } } return nil } -func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) { - *t = CompactedMessages{} +func (t *CompactedMessagesCBOR) UnmarshalCBOR(r io.Reader) (err error) { + *t = CompactedMessagesCBOR{} cr := cbg.NewCborReader(r) @@ -430,7 +408,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) { return err } - if extra > cbg.MaxLength { + if extra > 150000 { return fmt.Errorf("t.Bls: array too large (%d)", extra) } @@ -471,7 +449,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) { } } - // t.BlsIncludes ([][]uint64) (slice) + // t.BlsIncludes ([]exchange.messageIndices) (slice) maj, extra, err = cr.ReadHeader() if err != nil { @@ -487,7 +465,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) { } if extra > 0 { - t.BlsIncludes = make([][]uint64, extra) + t.BlsIncludes = make([]messageIndices, extra) } for i := 0; i < int(extra); i++ { @@ -499,47 +477,13 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) { _ = extra _ = err - maj, extra, err = cr.ReadHeader() - if err != nil { - return err - } + { - if extra > cbg.MaxLength { - return fmt.Errorf("t.BlsIncludes[i]: array too large (%d)", extra) - } - - if maj != cbg.MajArray { - return fmt.Errorf("expected cbor array") - } - - if extra > 0 { - t.BlsIncludes[i] = make([]uint64, extra) - } - - for j := 0; j < int(extra); j++ { - { - var maj byte - var extra uint64 - var err error - _ = maj - _ = extra - _ = err - - { - - maj, extra, err = cr.ReadHeader() - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.BlsIncludes[i][j] = uint64(extra) - - } + if err := t.BlsIncludes[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.BlsIncludes[i]: %w", err) } - } + } } } @@ -550,7 +494,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) { return err } - if extra > cbg.MaxLength { + if extra > 150000 { return fmt.Errorf("t.Secpk: array too large (%d)", extra) } @@ -591,7 +535,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) { } } - // t.SecpkIncludes ([][]uint64) (slice) + // t.SecpkIncludes ([]exchange.messageIndices) (slice) maj, extra, err = cr.ReadHeader() if err != nil { @@ -607,7 +551,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) { } if extra > 0 { - t.SecpkIncludes = make([][]uint64, extra) + t.SecpkIncludes = make([]messageIndices, extra) } for i := 0; i < int(extra); i++ { @@ -619,47 +563,13 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) { _ = extra _ = err - maj, extra, err = cr.ReadHeader() - if err != nil { - return err - } + { - if extra > cbg.MaxLength { - return fmt.Errorf("t.SecpkIncludes[i]: array too large (%d)", extra) - } - - if maj != cbg.MajArray { - return fmt.Errorf("expected cbor array") - } - - if extra > 0 { - t.SecpkIncludes[i] = make([]uint64, extra) - } - - for j := 0; j < int(extra); j++ { - { - var maj byte - var extra uint64 - var err error - _ = maj - _ = extra - _ = err - - { - - maj, extra, err = cr.ReadHeader() - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.SecpkIncludes[i][j] = uint64(extra) - - } + if err := t.SecpkIncludes[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.SecpkIncludes[i]: %w", err) } - } + } } } diff --git a/chain/exchange/client.go b/chain/exchange/client.go index 120b554a1..769c375ca 100644 --- a/chain/exchange/client.go +++ b/chain/exchange/client.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "fmt" + "io" "math/rand" "time" @@ -23,6 +24,10 @@ import ( "github.com/filecoin-project/lotus/lib/peermgr" ) +// Set the max exchange message size to 120MiB. Purely based on gas numbers, we can include ~8MiB of +// messages per block, so I've set this to 120MiB to be _very_ safe. +const maxExchangeMessageSize = (15 * 8) << 20 + // client implements exchange.Client, using the libp2p ChainExchange protocol // as the fetching mechanism. type client struct { @@ -434,10 +439,11 @@ func (c *client) sendRequestToPeer(ctx context.Context, peer peer.ID, req *Reque log.Warnw("CloseWrite err", "error", err) } - // Read response. + // Read response, limiting the size of the response to maxExchangeMessageSize as we allow a + // lot of messages (10k+) but they'll mostly be quite small. var res Response err = cborutil.ReadCborRPC( - bufio.NewReader(incrt.New(stream, ReadResMinSpeed, ReadResDeadline)), + bufio.NewReader(io.LimitReader(incrt.New(stream, ReadResMinSpeed, ReadResDeadline), maxExchangeMessageSize)), &res) if err != nil { c.peerTracker.logFailure(peer, build.Clock.Since(connectionStart), req.Length) diff --git a/chain/exchange/protocol.go b/chain/exchange/protocol.go index 5e12d31cc..cd25f4a43 100644 --- a/chain/exchange/protocol.go +++ b/chain/exchange/protocol.go @@ -154,6 +154,8 @@ type BSTipSet struct { // FIXME: The logic to decompress this structure should belong // // to itself, not to the consumer. +// +// NOTE: Max messages is: BlockMessageLimit (10k) * MaxTipsetSize (15) = 150k type CompactedMessages struct { Bls []*types.Message BlsIncludes [][]uint64 diff --git a/chain/exchange/protocol_encoding.go b/chain/exchange/protocol_encoding.go new file mode 100644 index 000000000..040dd0d40 --- /dev/null +++ b/chain/exchange/protocol_encoding.go @@ -0,0 +1,125 @@ +package exchange + +import ( + "fmt" + "io" + + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/build" + types "github.com/filecoin-project/lotus/chain/types" +) + +// Type used for encoding/decoding compacted messages. This is a ustom type as we need custom limits. +// - Max messages is 150,000 as that's 15 times the max block size (in messages). It needs to be +// large enough to cover a full tipset full of full blocks. +type CompactedMessagesCBOR struct { + Bls []*types.Message `cborgen:"maxlen=150000"` + BlsIncludes []messageIndices + + Secpk []*types.SignedMessage `cborgen:"maxlen=150000"` + SecpkIncludes []messageIndices +} + +// Unmarshal into the "decoding" struct, then copy into the actual struct. +func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) { + var c CompactedMessagesCBOR + if err := c.UnmarshalCBOR(r); err != nil { + return err + } + t.Bls = c.Bls + t.BlsIncludes = make([][]uint64, len(c.BlsIncludes)) + for i, v := range c.BlsIncludes { + t.BlsIncludes[i] = v.v + } + t.Secpk = c.Secpk + t.SecpkIncludes = make([][]uint64, len(c.SecpkIncludes)) + for i, v := range c.SecpkIncludes { + t.SecpkIncludes[i] = v.v + } + return nil +} + +// Copy into the encoding struct, then marshal. +func (t *CompactedMessages) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + var c CompactedMessagesCBOR + c.Bls = t.Bls + c.BlsIncludes = make([]messageIndices, len(t.BlsIncludes)) + for i, v := range t.BlsIncludes { + c.BlsIncludes[i].v = v + } + c.Secpk = t.Secpk + c.SecpkIncludes = make([]messageIndices, len(t.SecpkIncludes)) + for i, v := range t.SecpkIncludes { + c.SecpkIncludes[i].v = v + } + return c.MarshalCBOR(w) +} + +// this needs to be a struct or cborgen will peak into it and ignore the Unmarshal/Marshal functions +type messageIndices struct { + v []uint64 +} + +func (t *messageIndices) UnmarshalCBOR(r io.Reader) (err error) { + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra > uint64(build.BlockMessageLimit) { + return fmt.Errorf("cbor input had wrong number of fields") + } + + if extra > 0 { + t.v = make([]uint64, extra) + } + + for i := 0; i < int(extra); i++ { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.v[i] = extra + + } + return nil +} + +func (t *messageIndices) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if len(t.v) > build.BlockMessageLimit { + return xerrors.Errorf("Slice value in field v was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.v))); err != nil { + return err + } + for _, v := range t.v { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, v); err != nil { + return err + } + } + return nil +} diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index 0bac282d2..df8900cab 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -251,7 +251,8 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sys vm.Syscal } params := &markettypes.PublishStorageDealsParams{} - for _, preseal := range m.Sectors { + for _, presealTmp := range m.Sectors { + preseal := presealTmp preseal.Deal.VerifiedDeal = true preseal.Deal.EndEpoch = minerInfos[i].presealExp p := markettypes.ClientDealProposal{ diff --git a/chain/messagepool/block_proba_test.go b/chain/messagepool/block_proba_test.go index 6d121d222..2dc1dc25d 100644 --- a/chain/messagepool/block_proba_test.go +++ b/chain/messagepool/block_proba_test.go @@ -5,7 +5,6 @@ import ( "math" "math/rand" "testing" - "time" ) func TestBlockProbability(t *testing.T) { @@ -23,7 +22,6 @@ func TestBlockProbability(t *testing.T) { func TestWinnerProba(t *testing.T) { //stm: @OTHER_IMPLEMENTATION_BLOCK_PROB_002 - rand.Seed(time.Now().UnixNano()) const N = 1000000 winnerProba := noWinnersProb() sum := 0 diff --git a/chain/state/statetree.go b/chain/state/statetree.go index a0356f44c..61d7d500a 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -438,7 +438,8 @@ func (st *StateTree) Flush(ctx context.Context) (cid.Cid, error) { return cid.Undef, xerrors.Errorf("tried to flush state tree with snapshots on the stack") } - for addr, sto := range st.snaps.layers[0].actors { + for addr, stoTmp := range st.snaps.layers[0].actors { + sto := stoTmp if sto.Delete { if err := st.root.Delete(abi.AddrKey(addr)); err != nil { return cid.Undef, err diff --git a/chain/sync.go b/chain/sync.go index 1b9a302f7..4dccc2036 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -886,6 +886,35 @@ func (syncer *Syncer) syncFork(ctx context.Context, incoming *types.TipSet, know } } + incomingParentsTsk := incoming.Parents() + commonParent := false + for _, incomingParent := range incomingParentsTsk.Cids() { + if known.Contains(incomingParent) { + commonParent = true + } + } + + if commonParent { + // known contains at least one of incoming's Parents => the common ancestor is known's Parents (incoming's Grandparents) + // in this case, we need to return {incoming.Parents()} + incomingParents, err := syncer.store.LoadTipSet(ctx, incomingParentsTsk) + if err != nil { + // fallback onto the network + tips, err := syncer.Exchange.GetBlocks(ctx, incoming.Parents(), 1) + if err != nil { + return nil, xerrors.Errorf("failed to fetch incomingParents from the network: %w", err) + } + + if len(tips) == 0 { + return nil, xerrors.Errorf("network didn't return any tipsets") + } + + incomingParents = tips[0] + } + + return []*types.TipSet{incomingParents}, nil + } + // TODO: Does this mean we always ask for ForkLengthThreshold blocks from the network, even if we just need, like, 2? Yes. // Would it not be better to ask in smaller chunks, given that an ~ForkLengthThreshold is very rare? tips, err := syncer.Exchange.GetBlocks(ctx, incoming.Parents(), int(build.ForkLengthThreshold)) diff --git a/chain/types/ethtypes/eth_types.go b/chain/types/ethtypes/eth_types.go index b933329f4..bff15ed24 100644 --- a/chain/types/ethtypes/eth_types.go +++ b/chain/types/ethtypes/eth_types.go @@ -229,13 +229,25 @@ type EthCall struct { } func (c *EthCall) UnmarshalJSON(b []byte) error { - type TempEthCall EthCall - var params TempEthCall + type EthCallRaw EthCall // Avoid a recursive call. + type EthCallDecode struct { + // The field should be "input" by spec, but many clients use "data" so we support + // both, but prefer "input". + Input *EthBytes `json:"input"` + EthCallRaw + } + var params EthCallDecode if err := json.Unmarshal(b, ¶ms); err != nil { return err } - *c = EthCall(params) + + // If input is specified, prefer it. + if params.Input != nil { + params.Data = *params.Input + } + + *c = EthCall(params.EthCallRaw) return nil } diff --git a/chain/types/ethtypes/eth_types_test.go b/chain/types/ethtypes/eth_types_test.go index 4a73184c2..2a1b2df55 100644 --- a/chain/types/ethtypes/eth_types_test.go +++ b/chain/types/ethtypes/eth_types_test.go @@ -194,11 +194,40 @@ func TestMaskedIDInF4(t *testing.T) { } func TestUnmarshalEthCall(t *testing.T) { - data := `{"from":"0x4D6D86b31a112a05A473c4aE84afaF873f632325","to":"0xFe01CC39f5Ae8553D6914DBb9dC27D219fa22D7f","gas":"0x5","gasPrice":"0x6","value":"0x123","data":""}` + data := `{"from":"0x4D6D86b31a112a05A473c4aE84afaF873f632325","to":"0xFe01CC39f5Ae8553D6914DBb9dC27D219fa22D7f","gas":"0x5","gasPrice":"0x6","value":"0x123","data":"0xFF"}` var c EthCall err := c.UnmarshalJSON([]byte(data)) require.Nil(t, err) + require.EqualValues(t, []byte{0xff}, c.Data) +} + +func TestUnmarshalEthCallInput(t *testing.T) { + data := `{"from":"0x4D6D86b31a112a05A473c4aE84afaF873f632325","to":"0xFe01CC39f5Ae8553D6914DBb9dC27D219fa22D7f","gas":"0x5","gasPrice":"0x6","value":"0x123","input":"0xFF"}` + + var c EthCall + err := c.UnmarshalJSON([]byte(data)) + require.Nil(t, err) + require.EqualValues(t, []byte{0xff}, c.Data) +} + +func TestUnmarshalEthCallInputAndData(t *testing.T) { + data := `{"from":"0x4D6D86b31a112a05A473c4aE84afaF873f632325","to":"0xFe01CC39f5Ae8553D6914DBb9dC27D219fa22D7f","gas":"0x5","gasPrice":"0x6","value":"0x123","data":"0xFE","input":"0xFF"}` + + var c EthCall + err := c.UnmarshalJSON([]byte(data)) + require.Nil(t, err) + require.EqualValues(t, []byte{0xff}, c.Data) +} + +func TestUnmarshalEthCallInputAndDataEmpty(t *testing.T) { + // Even if the input is empty, it should be used when specified. + data := `{"from":"0x4D6D86b31a112a05A473c4aE84afaF873f632325","to":"0xFe01CC39f5Ae8553D6914DBb9dC27D219fa22D7f","gas":"0x5","gasPrice":"0x6","value":"0x123","data":"0xFE","input":""}` + + var c EthCall + err := c.UnmarshalJSON([]byte(data)) + require.Nil(t, err) + require.EqualValues(t, []byte{}, c.Data) } func TestUnmarshalEthBytes(t *testing.T) { diff --git a/chain/types/fil.go b/chain/types/fil.go index 60a2940c6..2a0ccb460 100644 --- a/chain/types/fil.go +++ b/chain/types/fil.go @@ -12,6 +12,9 @@ import ( type FIL BigInt func (f FIL) String() string { + if f.Int == nil { + return "0 FIL" + } return f.Unitless() + " FIL" } diff --git a/chain/vectors/gen/main.go b/chain/vectors/gen/main.go index ce9f1baf8..f4b7c82da 100644 --- a/chain/vectors/gen/main.go +++ b/chain/vectors/gen/main.go @@ -2,6 +2,7 @@ package main import ( "context" + crand "crypto/rand" "encoding/json" "fmt" "math/rand" @@ -145,7 +146,10 @@ func MakeUnsignedMessageVectors() []vectors.UnsignedMessageVector { } params := make([]byte, 32) - rand.Read(params) + _, err = crand.Read(params) + if err != nil { + panic(err) + } msg := &types.Message{ To: to, diff --git a/cli/helper.go b/cli/helper.go index 81a5bb033..fb1899e0a 100644 --- a/cli/helper.go +++ b/cli/helper.go @@ -1,6 +1,7 @@ package cli import ( + "errors" "fmt" "io" "os" @@ -8,7 +9,6 @@ import ( "syscall" ufcli "github.com/urfave/cli/v2" - "golang.org/x/xerrors" ) type PrintHelpErr struct { @@ -52,7 +52,7 @@ func RunApp(app *ufcli.App) { fmt.Fprintf(os.Stderr, "ERROR: %s\n\n", err) // nolint:errcheck } var phe *PrintHelpErr - if xerrors.As(err, &phe) { + if errors.As(err, &phe) { _ = ufcli.ShowCommandHelp(phe.Ctx, phe.Ctx.Command.Name) } os.Exit(1) diff --git a/cli/net.go b/cli/net.go index f25799e95..99ee92aef 100644 --- a/cli/net.go +++ b/cli/net.go @@ -847,7 +847,8 @@ var NetStatCmd = &cli.Command{ }) for _, stat := range stats { - printScope(&stat.stat, name+stat.name) + tmp := stat.stat + printScope(&tmp, name+stat.name) } } diff --git a/cli/util/api.go b/cli/util/api.go index 1d6928c3f..fe1ac1536 100644 --- a/cli/util/api.go +++ b/cli/util/api.go @@ -119,7 +119,7 @@ func GetAPIInfoMulti(ctx *cli.Context, t repo.RepoType) ([]APIInfo, error) { } } - return []APIInfo{}, fmt.Errorf("could not determine API endpoint for node type: %v", t.Type()) + return []APIInfo{}, fmt.Errorf("could not determine API endpoint for node type: %v. Try setting environment variable: %s", t.Type(), primaryEnv) } func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { @@ -164,6 +164,28 @@ func GetRawAPIMulti(ctx *cli.Context, t repo.RepoType, version string) ([]HttpHe return httpHeads, nil } +func GetRawAPIMultiV2(ctx *cli.Context, ainfoCfg []string, version string) ([]HttpHead, error) { + var httpHeads []HttpHead + + if len(ainfoCfg) == 0 { + return httpHeads, xerrors.Errorf("could not get API info: none configured. \nConsider getting base.toml with './lotus-provider config get base >/tmp/base.toml' \nthen adding \n[APIs] \n ChainApiInfo = [\" result_from lotus auth api-info --perm=admin \"]\n and updating it with './lotus-provider config set /tmp/base.toml'") + } + for _, i := range ainfoCfg { + ainfo := ParseApiInfo(i) + addr, err := ainfo.DialArgs(version) + if err != nil { + return httpHeads, xerrors.Errorf("could not get DialArgs: %w", err) + } + httpHeads = append(httpHeads, HttpHead{addr: addr, header: ainfo.AuthHeader()}) + } + + if IsVeryVerbose { + _, _ = fmt.Fprintf(ctx.App.Writer, "using raw API %s endpoint: %s\n", version, httpHeads[0].addr) + } + + return httpHeads, nil +} + func GetRawAPI(ctx *cli.Context, t repo.RepoType, version string) (string, http.Header, error) { heads, err := GetRawAPIMulti(ctx, t, version) if err != nil { @@ -393,6 +415,68 @@ func GetFullNodeAPIV1(ctx *cli.Context, opts ...GetFullNodeOption) (v1api.FullNo return &v1API, finalCloser, nil } +func GetFullNodeAPIV1LotusProvider(ctx *cli.Context, ainfoCfg []string, opts ...GetFullNodeOption) (v1api.FullNode, jsonrpc.ClientCloser, error) { + if tn, ok := ctx.App.Metadata["testnode-full"]; ok { + return tn.(v1api.FullNode), func() {}, nil + } + + var options GetFullNodeOptions + for _, opt := range opts { + opt(&options) + } + + var rpcOpts []jsonrpc.Option + if options.ethSubHandler != nil { + rpcOpts = append(rpcOpts, jsonrpc.WithClientHandler("Filecoin", options.ethSubHandler), jsonrpc.WithClientHandlerAlias("eth_subscription", "Filecoin.EthSubscription")) + } + + heads, err := GetRawAPIMultiV2(ctx, ainfoCfg, "v1") + if err != nil { + return nil, nil, err + } + + if IsVeryVerbose { + _, _ = fmt.Fprintln(ctx.App.Writer, "using full node API v1 endpoint:", heads[0].addr) + } + + var fullNodes []api.FullNode + var closers []jsonrpc.ClientCloser + + for _, head := range heads { + v1api, closer, err := client.NewFullNodeRPCV1(ctx.Context, head.addr, head.header, rpcOpts...) + if err != nil { + log.Warnf("Not able to establish connection to node with addr: %s, Reason: %s", head.addr, err.Error()) + continue + } + fullNodes = append(fullNodes, v1api) + closers = append(closers, closer) + } + + // When running in cluster mode and trying to establish connections to multiple nodes, fail + // if less than 2 lotus nodes are actually running + if len(heads) > 1 && len(fullNodes) < 2 { + return nil, nil, xerrors.Errorf("Not able to establish connection to more than a single node") + } + + finalCloser := func() { + for _, c := range closers { + c() + } + } + + var v1API api.FullNodeStruct + FullNodeProxy(fullNodes, &v1API) + + v, err := v1API.Version(ctx.Context) + if err != nil { + return nil, nil, err + } + if !v.APIVersion.EqMajorMinor(api.FullAPIVersion1) { + return nil, nil, xerrors.Errorf("Remote API version didn't match (expected %s, remote %s)", api.FullAPIVersion1, v.APIVersion) + } + return &v1API, finalCloser, nil +} + type GetStorageMinerOptions struct { PreferHttp bool } diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 1db788498..7d3c0cde0 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -3,10 +3,10 @@ package main import ( "bytes" "context" + "crypto/rand" "encoding/json" "fmt" "math/big" - "math/rand" "os" "path/filepath" "sync" @@ -547,7 +547,10 @@ var sealBenchCmd = &cli.Command{ } var challenge [32]byte - rand.Read(challenge[:]) + _, err = rand.Read(challenge[:]) + if err != nil { + return err + } beforePost := time.Now() @@ -777,9 +780,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par start := time.Now() log.Infof("[%d] Writing piece into sector...", i) - r := rand.New(rand.NewSource(100 + int64(i))) - - pi, err := sb.AddPiece(context.TODO(), sid, nil, abi.PaddedPieceSize(sectorSize).Unpadded(), r) + pi, err := sb.AddPiece(context.TODO(), sid, nil, abi.PaddedPieceSize(sectorSize).Unpadded(), rand.Reader) if err != nil { return nil, nil, err } diff --git a/cmd/lotus-bench/simple.go b/cmd/lotus-bench/simple.go index 09df22078..35d909ffb 100644 --- a/cmd/lotus-bench/simple.go +++ b/cmd/lotus-bench/simple.go @@ -308,7 +308,36 @@ var simplePreCommit2 = &cli.Command{ Name: "synthetic", Usage: "generate synthetic PoRep proofs", }, + &cli.StringFlag{ + Name: "external-pc2", + Usage: "command for computing PC2 externally", + }, }, + Description: `Compute PreCommit2 inputs and seal a sector. + +--external-pc2 can be used to compute the PreCommit2 inputs externally. +The flag behaves similarly to the related lotus-worker flag, using it in +lotus-bench may be useful for testing if the external PreCommit2 command is +invoked correctly. + +The command will be called with a number of environment variables set: +* EXTSEAL_PC2_SECTOR_NUM: the sector number +* EXTSEAL_PC2_SECTOR_MINER: the miner id +* EXTSEAL_PC2_PROOF_TYPE: the proof type +* EXTSEAL_PC2_SECTOR_SIZE: the sector size in bytes +* EXTSEAL_PC2_CACHE: the path to the cache directory +* EXTSEAL_PC2_SEALED: the path to the sealed sector file (initialized with unsealed data by the caller) +* EXTSEAL_PC2_PC1OUT: output from rust-fil-proofs precommit1 phase (base64 encoded json) + +The command is expected to: +* Create cache sc-02-data-tree-r* files +* Create cache sc-02-data-tree-c* files +* Create cache p_aux / t_aux files +* Transform the sealed file in place + +Example invocation of lotus-bench as external executor: +'./lotus-bench simple precommit2 --sector-size $EXTSEAL_PC2_SECTOR_SIZE $EXTSEAL_PC2_SEALED $EXTSEAL_PC2_CACHE $EXTSEAL_PC2_PC1OUT' +`, ArgsUsage: "[sealed] [cache] [pc1 out]", Action: func(cctx *cli.Context) error { ctx := cctx.Context @@ -333,7 +362,18 @@ var simplePreCommit2 = &cli.Command{ storiface.FTSealed: cctx.Args().Get(0), storiface.FTCache: cctx.Args().Get(1), } - sealer, err := ffiwrapper.New(pp) + + var opts []ffiwrapper.FFIWrapperOpt + + if cctx.IsSet("external-pc2") { + extSeal := ffiwrapper.ExternalSealer{ + PreCommit2: ffiwrapper.MakeExternPrecommit2(cctx.String("external-pc2")), + } + + opts = append(opts, ffiwrapper.WithExternalSealCalls(extSeal)) + } + + sealer, err := ffiwrapper.New(pp, opts...) if err != nil { return err } diff --git a/cmd/lotus-miner/init.go b/cmd/lotus-miner/init.go index c109e85b9..1a4a98fc4 100644 --- a/cmd/lotus-miner/init.go +++ b/cmd/lotus-miner/init.go @@ -120,6 +120,11 @@ var initCmd = &cli.Command{ Name: "from", Usage: "select which address to send actor creation message from", }, + &cli.Uint64Flag{ + Name: "confidence", + Usage: "number of block confirmations to wait for", + Value: build.MessageConfidence, + }, }, Subcommands: []*cli.Command{ restoreCmd, @@ -146,6 +151,8 @@ var initCmd = &cli.Command{ return xerrors.Errorf("failed to parse gas-price flag: %s", err) } + confidence := cctx.Uint64("confidence") + symlink := cctx.Bool("symlink-imported-sectors") if symlink { log.Info("will attempt to symlink to imported sectors") @@ -265,7 +272,7 @@ var initCmd = &cli.Command{ } } - if err := storageMinerInit(ctx, cctx, api, r, ssize, gasPrice); err != nil { + if err := storageMinerInit(ctx, cctx, api, r, ssize, gasPrice, confidence); err != nil { log.Errorf("Failed to initialize lotus-miner: %+v", err) path, err := homedir.Expand(repoPath) if err != nil { @@ -414,7 +421,7 @@ func findMarketDealID(ctx context.Context, api v1api.FullNode, deal markettypes. return 0, xerrors.New("deal not found") } -func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode, r repo.Repo, ssize abi.SectorSize, gasPrice types.BigInt) error { +func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode, r repo.Repo, ssize abi.SectorSize, gasPrice types.BigInt, confidence uint64) error { lr, err := r.Lock(repo.StorageMiner) if err != nil { return err @@ -463,7 +470,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode wsts := statestore.New(namespace.Wrap(mds, modules.WorkerCallsPrefix)) smsts := statestore.New(namespace.Wrap(mds, modules.ManagerWorkPrefix)) - si := paths.NewIndex(nil) + si := paths.NewMemIndex(nil) lstor, err := paths.NewLocal(ctx, lr, si, nil) if err != nil { @@ -501,7 +508,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode return xerrors.Errorf("failed to start up genesis miner: %w", err) } - cerr := configureStorageMiner(ctx, api, a, peerid, gasPrice) + cerr := configureStorageMiner(ctx, api, a, peerid, gasPrice, confidence) if err := m.Stop(ctx); err != nil { log.Error("failed to shut down miner: ", err) @@ -541,13 +548,13 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode } } - if err := configureStorageMiner(ctx, api, a, peerid, gasPrice); err != nil { + if err := configureStorageMiner(ctx, api, a, peerid, gasPrice, confidence); err != nil { return xerrors.Errorf("failed to configure miner: %w", err) } addr = a } else { - a, err := createStorageMiner(ctx, api, ssize, peerid, gasPrice, cctx) + a, err := createStorageMiner(ctx, api, ssize, peerid, gasPrice, confidence, cctx) if err != nil { return xerrors.Errorf("creating miner failed: %w", err) } @@ -589,7 +596,7 @@ func makeHostKey(lr repo.LockedRepo) (crypto.PrivKey, error) { return pk, nil } -func configureStorageMiner(ctx context.Context, api v1api.FullNode, addr address.Address, peerid peer.ID, gasPrice types.BigInt) error { +func configureStorageMiner(ctx context.Context, api v1api.FullNode, addr address.Address, peerid peer.ID, gasPrice types.BigInt, confidence uint64) error { mi, err := api.StateMinerInfo(ctx, addr, types.EmptyTSK) if err != nil { return xerrors.Errorf("getWorkerAddr returned bad address: %w", err) @@ -615,7 +622,7 @@ func configureStorageMiner(ctx context.Context, api v1api.FullNode, addr address } log.Info("Waiting for message: ", smsg.Cid()) - ret, err := api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence, lapi.LookbackNoLimit, true) + ret, err := api.StateWaitMsg(ctx, smsg.Cid(), confidence, lapi.LookbackNoLimit, true) if err != nil { return err } @@ -627,7 +634,7 @@ func configureStorageMiner(ctx context.Context, api v1api.FullNode, addr address return nil } -func createStorageMiner(ctx context.Context, api v1api.FullNode, ssize abi.SectorSize, peerid peer.ID, gasPrice types.BigInt, cctx *cli.Context) (address.Address, error) { +func createStorageMiner(ctx context.Context, api v1api.FullNode, ssize abi.SectorSize, peerid peer.ID, gasPrice types.BigInt, confidence uint64, cctx *cli.Context) (address.Address, error) { var err error var owner address.Address if cctx.String("owner") != "" { @@ -679,7 +686,7 @@ func createStorageMiner(ctx context.Context, api v1api.FullNode, ssize abi.Secto log.Infof("Initializing worker account %s, message: %s", worker, signed.Cid()) log.Infof("Waiting for confirmation") - mw, err := api.StateWaitMsg(ctx, signed.Cid(), build.MessageConfidence, lapi.LookbackNoLimit, true) + mw, err := api.StateWaitMsg(ctx, signed.Cid(), confidence, lapi.LookbackNoLimit, true) if err != nil { return address.Undef, xerrors.Errorf("waiting for worker init: %w", err) } @@ -703,7 +710,7 @@ func createStorageMiner(ctx context.Context, api v1api.FullNode, ssize abi.Secto log.Infof("Initializing owner account %s, message: %s", worker, signed.Cid()) log.Infof("Waiting for confirmation") - mw, err := api.StateWaitMsg(ctx, signed.Cid(), build.MessageConfidence, lapi.LookbackNoLimit, true) + mw, err := api.StateWaitMsg(ctx, signed.Cid(), confidence, lapi.LookbackNoLimit, true) if err != nil { return address.Undef, xerrors.Errorf("waiting for owner init: %w", err) } @@ -752,7 +759,7 @@ func createStorageMiner(ctx context.Context, api v1api.FullNode, ssize abi.Secto log.Infof("Pushed CreateMiner message: %s", signed.Cid()) log.Infof("Waiting for confirmation") - mw, err := api.StateWaitMsg(ctx, signed.Cid(), build.MessageConfidence, lapi.LookbackNoLimit, true) + mw, err := api.StateWaitMsg(ctx, signed.Cid(), confidence, lapi.LookbackNoLimit, true) if err != nil { return address.Undef, xerrors.Errorf("waiting for createMiner message: %w", err) } diff --git a/cmd/lotus-miner/init_restore.go b/cmd/lotus-miner/init_restore.go index 7e28729bb..272754c23 100644 --- a/cmd/lotus-miner/init_restore.go +++ b/cmd/lotus-miner/init_restore.go @@ -80,8 +80,7 @@ var restoreCmd = &cli.Command{ } log.Info("Configuring miner actor") - - if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero()); err != nil { + if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero(), cctx.Uint64("confidence")); err != nil { return err } diff --git a/cmd/lotus-miner/init_service.go b/cmd/lotus-miner/init_service.go index 235e4e4c8..876313941 100644 --- a/cmd/lotus-miner/init_service.go +++ b/cmd/lotus-miner/init_service.go @@ -105,7 +105,7 @@ var serviceCmd = &cli.Command{ if es.Contains(MarketsService) { log.Info("Configuring miner actor") - if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero()); err != nil { + if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero(), cctx.Uint64("confidence")); err != nil { return err } } diff --git a/cmd/lotus-miner/proving.go b/cmd/lotus-miner/proving.go index 3ecc58ba7..2fc1427b5 100644 --- a/cmd/lotus-miner/proving.go +++ b/cmd/lotus-miner/proving.go @@ -559,7 +559,8 @@ var provingCheckProvableCmd = &cli.Command{ for parIdx, par := range partitions { sectors := make(map[abi.SectorNumber]struct{}) - sectorInfos, err := api.StateMinerSectors(ctx, addr, &par.LiveSectors, types.EmptyTSK) + tmp := par.LiveSectors + sectorInfos, err := api.StateMinerSectors(ctx, addr, &tmp, types.EmptyTSK) if err != nil { return err } diff --git a/cmd/lotus-miner/sectors.go b/cmd/lotus-miner/sectors.go index 07cc2e795..3e4439eb8 100644 --- a/cmd/lotus-miner/sectors.go +++ b/cmd/lotus-miner/sectors.go @@ -2290,7 +2290,7 @@ var sectorsCompactPartitionsCmd = &cli.Command{ if len(parts) <= 0 { return fmt.Errorf("must include at least one partition to compact") } - fmt.Printf("compacting %d paritions\n", len(parts)) + fmt.Printf("compacting %d partitions\n", len(parts)) var makeMsgForPartitions func(partitionsBf bitfield.BitField) ([]*types.Message, error) makeMsgForPartitions = func(partitionsBf bitfield.BitField) ([]*types.Message, error) { diff --git a/cmd/lotus-provider/config.go b/cmd/lotus-provider/config.go new file mode 100644 index 000000000..44ba49beb --- /dev/null +++ b/cmd/lotus-provider/config.go @@ -0,0 +1,241 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path" + "strings" + + "github.com/BurntSushi/toml" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/cmd/lotus-provider/deps" + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" + "github.com/filecoin-project/lotus/node/config" +) + +var configCmd = &cli.Command{ + Name: "config", + Usage: "Manage node config by layers. The layer 'base' will always be applied. ", + Subcommands: []*cli.Command{ + configDefaultCmd, + configSetCmd, + configGetCmd, + configListCmd, + configViewCmd, + configRmCmd, + configMigrateCmd, + }, +} + +var configDefaultCmd = &cli.Command{ + Name: "default", + Aliases: []string{"defaults"}, + Usage: "Print default node config", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "no-comment", + Usage: "don't comment default values", + }, + }, + Action: func(cctx *cli.Context) error { + comment := !cctx.Bool("no-comment") + cfg, err := getDefaultConfig(comment) + if err != nil { + return err + } + fmt.Print(cfg) + + return nil + }, +} + +func getDefaultConfig(comment bool) (string, error) { + c := config.DefaultLotusProvider() + cb, err := config.ConfigUpdate(c, nil, config.Commented(comment), config.DefaultKeepUncommented(), config.NoEnv()) + if err != nil { + return "", err + } + return string(cb), nil +} + +var configSetCmd = &cli.Command{ + Name: "set", + Aliases: []string{"add", "update", "create"}, + Usage: "Set a config layer or the base by providing a filename or stdin.", + ArgsUsage: "a layer's file name", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "title", + Usage: "title of the config layer (req'd for stdin)", + }, + }, + Action: func(cctx *cli.Context) error { + args := cctx.Args() + + db, err := deps.MakeDB(cctx) + if err != nil { + return err + } + + name := cctx.String("title") + var stream io.Reader = os.Stdin + if args.Len() != 1 { + if cctx.String("title") == "" { + return errors.New("must have a title for stdin, or a file name") + } + } else { + stream, err = os.Open(args.First()) + if err != nil { + return fmt.Errorf("cannot open file %s: %w", args.First(), err) + } + if name == "" { + name = strings.Split(path.Base(args.First()), ".")[0] + } + } + bytes, err := io.ReadAll(stream) + if err != nil { + return fmt.Errorf("cannot read stream/file %w", err) + } + + lp := config.DefaultLotusProvider() // ensure it's toml + _, err = toml.Decode(string(bytes), lp) + if err != nil { + return fmt.Errorf("cannot decode file: %w", err) + } + _ = lp + + err = setConfig(db, name, string(bytes)) + + if err != nil { + return fmt.Errorf("unable to save config layer: %w", err) + } + + fmt.Println("Layer " + name + " created/updated") + return nil + }, +} + +func setConfig(db *harmonydb.DB, name, config string) error { + _, err := db.Exec(context.Background(), + `INSERT INTO harmony_config (title, config) VALUES ($1, $2) + ON CONFLICT (title) DO UPDATE SET config = excluded.config`, name, config) + return err +} + +var configGetCmd = &cli.Command{ + Name: "get", + Aliases: []string{"cat", "show"}, + Usage: "Get a config layer by name. You may want to pipe the output to a file, or use 'less'", + ArgsUsage: "layer name", + Action: func(cctx *cli.Context) error { + args := cctx.Args() + if args.Len() != 1 { + return fmt.Errorf("want 1 layer arg, got %d", args.Len()) + } + db, err := deps.MakeDB(cctx) + if err != nil { + return err + } + + cfg, err := getConfig(db, args.First()) + if err != nil { + return err + } + fmt.Println(cfg) + + return nil + }, +} + +func getConfig(db *harmonydb.DB, layer string) (string, error) { + var cfg string + err := db.QueryRow(context.Background(), `SELECT config FROM harmony_config WHERE title=$1`, layer).Scan(&cfg) + if err != nil { + return "", err + } + return cfg, nil +} + +var configListCmd = &cli.Command{ + Name: "list", + Aliases: []string{"ls"}, + Usage: "List config layers you can get.", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + db, err := deps.MakeDB(cctx) + if err != nil { + return err + } + var res []string + err = db.Select(context.Background(), &res, `SELECT title FROM harmony_config ORDER BY title`) + if err != nil { + return fmt.Errorf("unable to read from db: %w", err) + } + for _, r := range res { + fmt.Println(r) + } + + return nil + }, +} + +var configRmCmd = &cli.Command{ + Name: "remove", + Aliases: []string{"rm", "del", "delete"}, + Usage: "Remove a named config layer.", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + args := cctx.Args() + if args.Len() != 1 { + return errors.New("must have exactly 1 arg for the layer name") + } + db, err := deps.MakeDB(cctx) + if err != nil { + return err + } + ct, err := db.Exec(context.Background(), `DELETE FROM harmony_config WHERE title=$1`, args.First()) + if err != nil { + return fmt.Errorf("unable to read from db: %w", err) + } + if ct == 0 { + return fmt.Errorf("no layer named %s", args.First()) + } + + return nil + }, +} +var configViewCmd = &cli.Command{ + Name: "interpret", + Aliases: []string{"view", "stacked", "stack"}, + Usage: "Interpret stacked config layers by this version of lotus-provider, with system-generated comments.", + ArgsUsage: "a list of layers to be interpreted as the final config", + Flags: []cli.Flag{ + &cli.StringSliceFlag{ + Name: "layers", + Usage: "comma or space separated list of layers to be interpreted", + Value: cli.NewStringSlice("base"), + Required: true, + }, + }, + Action: func(cctx *cli.Context) error { + db, err := deps.MakeDB(cctx) + if err != nil { + return err + } + lp, err := deps.GetConfig(cctx, db) + if err != nil { + return err + } + cb, err := config.ConfigUpdate(lp, config.DefaultLotusProvider(), config.Commented(true), config.DefaultKeepUncommented(), config.NoEnv()) + if err != nil { + return xerrors.Errorf("cannot interpret config: %w", err) + } + fmt.Println(string(cb)) + return nil + }, +} diff --git a/cmd/lotus-provider/deps/deps.go b/cmd/lotus-provider/deps/deps.go new file mode 100644 index 000000000..7a8db855f --- /dev/null +++ b/cmd/lotus-provider/deps/deps.go @@ -0,0 +1,282 @@ +// Package deps provides the dependencies for the lotus provider node. +package deps + +import ( + "context" + "database/sql" + "encoding/base64" + "errors" + "fmt" + "net" + "net/http" + "os" + "strings" + + "github.com/BurntSushi/toml" + "github.com/gbrlsnchs/jwt/v3" + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + logging "github.com/ipfs/go-log/v2" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-jsonrpc/auth" + "github.com/filecoin-project/go-statestore" + + "github.com/filecoin-project/lotus/api" + cliutil "github.com/filecoin-project/lotus/cli/util" + "github.com/filecoin-project/lotus/journal" + "github.com/filecoin-project/lotus/journal/alerting" + "github.com/filecoin-project/lotus/journal/fsjournal" + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" + "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/provider" + "github.com/filecoin-project/lotus/storage/ctladdr" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" + "github.com/filecoin-project/lotus/storage/sealer/storiface" +) + +var log = logging.Logger("lotus-provider/deps") + +func MakeDB(cctx *cli.Context) (*harmonydb.DB, error) { + dbConfig := config.HarmonyDB{ + Username: cctx.String("db-user"), + Password: cctx.String("db-password"), + Hosts: strings.Split(cctx.String("db-host"), ","), + Database: cctx.String("db-name"), + Port: cctx.String("db-port"), + } + return harmonydb.NewFromConfig(dbConfig) +} + +type JwtPayload struct { + Allow []auth.Permission +} + +func StorageAuth(apiKey string) (sealer.StorageAuth, error) { + if apiKey == "" { + return nil, xerrors.Errorf("no api key provided") + } + + rawKey, err := base64.StdEncoding.DecodeString(apiKey) + if err != nil { + return nil, xerrors.Errorf("decoding api key: %w", err) + } + + key := jwt.NewHS256(rawKey) + + p := JwtPayload{ + Allow: []auth.Permission{"admin"}, + } + + token, err := jwt.Sign(&p, key) + if err != nil { + return nil, err + } + + headers := http.Header{} + headers.Add("Authorization", "Bearer "+string(token)) + return sealer.StorageAuth(headers), nil +} + +func GetDeps(ctx context.Context, cctx *cli.Context) (*Deps, error) { + var deps Deps + return &deps, deps.PopulateRemainingDeps(ctx, cctx, true) +} + +type Deps struct { + Cfg *config.LotusProviderConfig + DB *harmonydb.DB + Full api.FullNode + Verif storiface.Verifier + LW *sealer.LocalWorker + As *ctladdr.AddressSelector + Maddrs []dtypes.MinerAddress + Stor *paths.Remote + Si *paths.DBIndex + LocalStore *paths.Local + ListenAddr string +} + +const ( + FlagRepoPath = "repo-path" +) + +func (deps *Deps) PopulateRemainingDeps(ctx context.Context, cctx *cli.Context, makeRepo bool) error { + var err error + if makeRepo { + // Open repo + repoPath := cctx.String(FlagRepoPath) + fmt.Println("repopath", repoPath) + r, err := repo.NewFS(repoPath) + if err != nil { + return err + } + + ok, err := r.Exists() + if err != nil { + return err + } + if !ok { + if err := r.Init(repo.Provider); err != nil { + return err + } + } + } + + if deps.Cfg == nil { + deps.DB, err = MakeDB(cctx) + if err != nil { + return err + } + } + + if deps.Cfg == nil { + // The config feeds into task runners & their helpers + deps.Cfg, err = GetConfig(cctx, deps.DB) + if err != nil { + return err + } + } + + log.Debugw("config", "config", deps.Cfg) + + if deps.Verif == nil { + deps.Verif = ffiwrapper.ProofVerifier + } + + if deps.As == nil { + deps.As, err = provider.AddressSelector(&deps.Cfg.Addresses)() + if err != nil { + return err + } + } + + if deps.Si == nil { + de, err := journal.ParseDisabledEvents(deps.Cfg.Journal.DisabledEvents) + if err != nil { + return err + } + j, err := fsjournal.OpenFSJournalPath(cctx.String("journal"), de) + if err != nil { + return err + } + go func() { + <-ctx.Done() + _ = j.Close() + }() + + al := alerting.NewAlertingSystem(j) + deps.Si = paths.NewDBIndex(al, deps.DB) + } + + if deps.Full == nil { + var fullCloser func() + cfgApiInfo := deps.Cfg.Apis.ChainApiInfo + if v := os.Getenv("FULLNODE_API_INFO"); v != "" { + cfgApiInfo = []string{v} + } + deps.Full, fullCloser, err = cliutil.GetFullNodeAPIV1LotusProvider(cctx, cfgApiInfo) + if err != nil { + return err + } + + go func() { + <-ctx.Done() + fullCloser() + }() + } + + bls := &paths.BasicLocalStorage{ + PathToJSON: cctx.String("storage-json"), + } + + if deps.ListenAddr == "" { + listenAddr := cctx.String("listen") + const unspecifiedAddress = "0.0.0.0" + addressSlice := strings.Split(listenAddr, ":") + if ip := net.ParseIP(addressSlice[0]); ip != nil { + if ip.String() == unspecifiedAddress { + rip, err := deps.DB.GetRoutableIP() + if err != nil { + return err + } + deps.ListenAddr = rip + ":" + addressSlice[1] + } + } + } + if deps.LocalStore == nil { + deps.LocalStore, err = paths.NewLocal(ctx, bls, deps.Si, []string{"http://" + deps.ListenAddr + "/remote"}) + if err != nil { + return err + } + } + + sa, err := StorageAuth(deps.Cfg.Apis.StorageRPCSecret) + if err != nil { + return xerrors.Errorf(`'%w' while parsing the config toml's + [Apis] + StorageRPCSecret=%v +Get it with: jq .PrivateKey ~/.lotus-miner/keystore/MF2XI2BNNJ3XILLQOJUXMYLUMU`, err, deps.Cfg.Apis.StorageRPCSecret) + } + if deps.Stor == nil { + deps.Stor = paths.NewRemote(deps.LocalStore, deps.Si, http.Header(sa), 10, &paths.DefaultPartialFileHandler{}) + } + if deps.LW == nil { + wstates := statestore.New(dssync.MutexWrap(ds.NewMapDatastore())) + + // todo localWorker isn't the abstraction layer we want to use here, we probably want to go straight to ffiwrapper + // maybe with a lotus-provider specific abstraction. LocalWorker does persistent call tracking which we probably + // don't need (ehh.. maybe we do, the async callback system may actually work decently well with harmonytask) + deps.LW = sealer.NewLocalWorker(sealer.WorkerConfig{}, deps.Stor, deps.LocalStore, deps.Si, nil, wstates) + } + if len(deps.Maddrs) == 0 { + for _, s := range deps.Cfg.Addresses.MinerAddresses { + addr, err := address.NewFromString(s) + if err != nil { + return err + } + deps.Maddrs = append(deps.Maddrs, dtypes.MinerAddress(addr)) + } + } + fmt.Println("last line of populate") + return nil +} + +func GetConfig(cctx *cli.Context, db *harmonydb.DB) (*config.LotusProviderConfig, error) { + lp := config.DefaultLotusProvider() + have := []string{} + layers := cctx.StringSlice("layers") + for _, layer := range layers { + text := "" + err := db.QueryRow(cctx.Context, `SELECT config FROM harmony_config WHERE title=$1`, layer).Scan(&text) + if err != nil { + if strings.Contains(err.Error(), sql.ErrNoRows.Error()) { + return nil, fmt.Errorf("missing layer '%s' ", layer) + } + if layer == "base" { + return nil, errors.New(`lotus-provider defaults to a layer named 'base'. + Either use 'migrate' command or edit a base.toml and upload it with: lotus-provider config set base.toml`) + } + return nil, fmt.Errorf("could not read layer '%s': %w", layer, err) + } + meta, err := toml.Decode(text, &lp) + if err != nil { + return nil, fmt.Errorf("could not read layer, bad toml %s: %w", layer, err) + } + for _, k := range meta.Keys() { + have = append(have, strings.Join(k, " ")) + } + log.Infow("Using layer", "layer", layer, "config", lp) + } + _ = have // FUTURE: verify that required fields are here. + // If config includes 3rd-party config, consider JSONSchema as a way that + // 3rd-parties can dynamically include config requirements and we can + // validate the config. Because of layering, we must validate @ startup. + return lp, nil +} diff --git a/cmd/lotus-provider/main.go b/cmd/lotus-provider/main.go new file mode 100644 index 000000000..1b025303c --- /dev/null +++ b/cmd/lotus-provider/main.go @@ -0,0 +1,165 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "runtime/pprof" + "syscall" + + "github.com/fatih/color" + logging "github.com/ipfs/go-log/v2" + "github.com/mitchellh/go-homedir" + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/lotus/build" + lcli "github.com/filecoin-project/lotus/cli" + cliutil "github.com/filecoin-project/lotus/cli/util" + "github.com/filecoin-project/lotus/cmd/lotus-provider/deps" + "github.com/filecoin-project/lotus/lib/lotuslog" + "github.com/filecoin-project/lotus/lib/tracing" + "github.com/filecoin-project/lotus/node/repo" +) + +var log = logging.Logger("main") + +func SetupCloseHandler() { + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + fmt.Println("\r- Ctrl+C pressed in Terminal") + _ = pprof.Lookup("goroutine").WriteTo(os.Stdout, 1) + panic(1) + }() +} + +func main() { + SetupCloseHandler() + + lotuslog.SetupLogLevels() + + local := []*cli.Command{ + //initCmd, + runCmd, + stopCmd, + configCmd, + testCmd, + webCmd, + //backupCmd, + //lcli.WithCategory("chain", actorCmd), + //lcli.WithCategory("storage", sectorsCmd), + //lcli.WithCategory("storage", provingCmd), + //lcli.WithCategory("storage", storageCmd), + //lcli.WithCategory("storage", sealingCmd), + } + + jaeger := tracing.SetupJaegerTracing("lotus") + defer func() { + if jaeger != nil { + _ = jaeger.ForceFlush(context.Background()) + } + }() + + for _, cmd := range local { + cmd := cmd + originBefore := cmd.Before + cmd.Before = func(cctx *cli.Context) error { + if jaeger != nil { + _ = jaeger.Shutdown(cctx.Context) + } + jaeger = tracing.SetupJaegerTracing("lotus/" + cmd.Name) + + if cctx.IsSet("color") { + color.NoColor = !cctx.Bool("color") + } + + if originBefore != nil { + return originBefore(cctx) + } + + return nil + } + } + + app := &cli.App{ + Name: "lotus-provider", + Usage: "Filecoin decentralized storage network provider", + Version: build.UserVersion(), + EnableBashCompletion: true, + Flags: []cli.Flag{ + &cli.BoolFlag{ + // examined in the Before above + Name: "color", + Usage: "use color in display output", + DefaultText: "depends on output being a TTY", + }, + &cli.StringFlag{ + Name: "panic-reports", + EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"}, + Hidden: true, + Value: "~/.lotusprovider", // should follow --repo default + }, + &cli.StringFlag{ + Name: "db-host", + EnvVars: []string{"LOTUS_DB_HOST"}, + Usage: "Command separated list of hostnames for yugabyte cluster", + Value: "yugabyte", + }, + &cli.StringFlag{ + Name: "db-name", + EnvVars: []string{"LOTUS_DB_NAME", "LOTUS_HARMONYDB_HOSTS"}, + Value: "yugabyte", + }, + &cli.StringFlag{ + Name: "db-user", + EnvVars: []string{"LOTUS_DB_USER", "LOTUS_HARMONYDB_USERNAME"}, + Value: "yugabyte", + }, + &cli.StringFlag{ + Name: "db-password", + EnvVars: []string{"LOTUS_DB_PASSWORD", "LOTUS_HARMONYDB_PASSWORD"}, + Value: "yugabyte", + }, + &cli.StringFlag{ + Name: "db-port", + EnvVars: []string{"LOTUS_DB_PORT", "LOTUS_HARMONYDB_PORT"}, + Hidden: true, + Value: "5433", + }, + &cli.StringFlag{ + Name: "layers", + EnvVars: []string{"LOTUS_LAYERS", "LOTUS_CONFIG_LAYERS"}, + Value: "base", + }, + &cli.StringFlag{ + Name: deps.FlagRepoPath, + EnvVars: []string{"LOTUS_REPO_PATH"}, + Value: "~/.lotusprovider", + }, + cliutil.FlagVeryVerbose, + }, + Commands: append(local, lcli.CommonCommands...), + Before: func(c *cli.Context) error { + return nil + }, + After: func(c *cli.Context) error { + if r := recover(); r != nil { + p, err := homedir.Expand(c.String(FlagMinerRepo)) + if err != nil { + log.Errorw("could not expand repo path for panic report", "error", err) + panic(r) + } + + // Generate report in LOTUS_PATH and re-raise panic + build.GeneratePanicReport(c.String("panic-reports"), p, c.App.Name) + panic(r) + } + return nil + }, + } + app.Setup() + app.Metadata["repoType"] = repo.Provider + lcli.RunApp(app) +} diff --git a/cmd/lotus-provider/migrate.go b/cmd/lotus-provider/migrate.go new file mode 100644 index 000000000..3869c7dfb --- /dev/null +++ b/cmd/lotus-provider/migrate.go @@ -0,0 +1,247 @@ +package main + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "os" + "path" + "strings" + + "github.com/BurntSushi/toml" + "github.com/fatih/color" + "github.com/ipfs/go-datastore" + "github.com/samber/lo" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + + cliutil "github.com/filecoin-project/lotus/cli/util" + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" + "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/node/modules" + "github.com/filecoin-project/lotus/node/repo" +) + +var configMigrateCmd = &cli.Command{ + Name: "from-miner", + Usage: "Express a database config (for lotus-provider) from an existing miner.", + Description: "Express a database config (for lotus-provider) from an existing miner.", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: FlagMinerRepo, + Aliases: []string{FlagMinerRepoDeprecation}, + EnvVars: []string{"LOTUS_MINER_PATH", "LOTUS_STORAGE_PATH"}, + Value: "~/.lotusminer", + Usage: fmt.Sprintf("Specify miner repo path. flag(%s) and env(LOTUS_STORAGE_PATH) are DEPRECATION, will REMOVE SOON", FlagMinerRepoDeprecation), + }, + &cli.StringFlag{ + Name: "repo", + EnvVars: []string{"LOTUS_PATH"}, + Hidden: true, + Value: "~/.lotus", + }, + &cli.StringFlag{ + Name: "to-layer", + Aliases: []string{"t"}, + Usage: "The layer name for this data push. 'base' is recommended for single-miner setup.", + }, + &cli.BoolFlag{ + Name: "overwrite", + Aliases: []string{"o"}, + Usage: "Use this with --to-layer to replace an existing layer", + }, + }, + Action: fromMiner, +} + +const ( + FlagMinerRepo = "miner-repo" +) + +const FlagMinerRepoDeprecation = "storagerepo" + +func fromMiner(cctx *cli.Context) (err error) { + ctx := context.Background() + cliCommandColor := color.New(color.FgHiBlue).SprintFunc() + configColor := color.New(color.FgHiGreen).SprintFunc() + + r, err := repo.NewFS(cctx.String(FlagMinerRepo)) + if err != nil { + return err + } + + ok, err := r.Exists() + if err != nil { + return err + } + + if !ok { + return fmt.Errorf("repo not initialized") + } + + lr, err := r.LockRO(repo.StorageMiner) + if err != nil { + return fmt.Errorf("locking repo: %w", err) + } + defer func() { _ = lr.Close() }() + + cfgNode, err := lr.Config() + if err != nil { + return fmt.Errorf("getting node config: %w", err) + } + smCfg := cfgNode.(*config.StorageMiner) + + db, err := harmonydb.NewFromConfig(smCfg.HarmonyDB) + if err != nil { + return fmt.Errorf("could not reach the database. Ensure the Miner config toml's HarmonyDB entry"+ + " is setup to reach Yugabyte correctly: %w", err) + } + + var titles []string + err = db.Select(ctx, &titles, `SELECT title FROM harmony_config WHERE LENGTH(config) > 0`) + if err != nil { + return fmt.Errorf("miner cannot reach the db. Ensure the config toml's HarmonyDB entry"+ + " is setup to reach Yugabyte correctly: %s", err.Error()) + } + name := cctx.String("to-layer") + if name == "" { + name = fmt.Sprintf("mig%d", len(titles)) + } else { + if lo.Contains(titles, name) && !cctx.Bool("overwrite") { + return errors.New("the overwrite flag is needed to replace existing layer: " + name) + } + } + msg := "Layer " + configColor(name) + ` created. ` + + // Copy over identical settings: + + buf, err := os.ReadFile(path.Join(lr.Path(), "config.toml")) + if err != nil { + return fmt.Errorf("could not read config.toml: %w", err) + } + var lpCfg config.LotusProviderConfig + _, err = toml.Decode(string(buf), &lpCfg) + if err != nil { + return fmt.Errorf("could not decode toml: %w", err) + } + + // Populate Miner Address + mmeta, err := lr.Datastore(ctx, "/metadata") + if err != nil { + return xerrors.Errorf("opening miner metadata datastore: %w", err) + } + defer func() { + _ = mmeta.Close() + }() + + maddrBytes, err := mmeta.Get(ctx, datastore.NewKey("miner-address")) + if err != nil { + return xerrors.Errorf("getting miner address datastore entry: %w", err) + } + + addr, err := address.NewFromBytes(maddrBytes) + if err != nil { + return xerrors.Errorf("parsing miner actor address: %w", err) + } + + lpCfg.Addresses.MinerAddresses = []string{addr.String()} + + ks, err := lr.KeyStore() + if err != nil { + return xerrors.Errorf("keystore err: %w", err) + } + js, err := ks.Get(modules.JWTSecretName) + if err != nil { + return xerrors.Errorf("error getting JWTSecretName: %w", err) + } + + lpCfg.Apis.StorageRPCSecret = base64.StdEncoding.EncodeToString(js.PrivateKey) + + // Populate API Key + _, header, err := cliutil.GetRawAPI(cctx, repo.FullNode, "v0") + if err != nil { + return fmt.Errorf("cannot read API: %w", err) + } + + ainfo, err := cliutil.GetAPIInfo(&cli.Context{}, repo.FullNode) + if err != nil { + return xerrors.Errorf(`could not get API info for FullNode: %w + Set the environment variable to the value of "lotus auth api-info --perm=admin"`, err) + } + lpCfg.Apis.ChainApiInfo = []string{header.Get("Authorization")[7:] + ":" + ainfo.Addr} + + // Enable WindowPoSt + lpCfg.Subsystems.EnableWindowPost = true + msg += "\nBefore running lotus-provider, ensure any miner/worker answering of WindowPost is disabled by " + + "(on Miner) " + configColor("DisableBuiltinWindowPoSt=true") + " and (on Workers) not enabling windowpost on CLI or via " + + "environment variable " + configColor("LOTUS_WORKER_WINDOWPOST") + "." + + // Express as configTOML + configTOML := &bytes.Buffer{} + if err = toml.NewEncoder(configTOML).Encode(lpCfg); err != nil { + return err + } + + if !lo.Contains(titles, "base") { + cfg, err := getDefaultConfig(true) + if err != nil { + return xerrors.Errorf("Cannot get default config: %w", err) + } + _, err = db.Exec(ctx, "INSERT INTO harmony_config (title, config) VALUES ('base', $1)", cfg) + + if err != nil { + return err + } + } + + if cctx.Bool("overwrite") { + i, err := db.Exec(ctx, "DELETE FROM harmony_config WHERE title=$1", name) + if i != 0 { + fmt.Println("Overwriting existing layer") + } + if err != nil { + fmt.Println("Got error while deleting existing layer: " + err.Error()) + } + } + + _, err = db.Exec(ctx, "INSERT INTO harmony_config (title, config) VALUES ($1, $2)", name, configTOML.String()) + if err != nil { + return err + } + + dbSettings := "" + def := config.DefaultStorageMiner().HarmonyDB + if def.Hosts[0] != smCfg.HarmonyDB.Hosts[0] { + dbSettings += ` --db-host="` + strings.Join(smCfg.HarmonyDB.Hosts, ",") + `"` + } + if def.Port != smCfg.HarmonyDB.Port { + dbSettings += " --db-port=" + smCfg.HarmonyDB.Port + } + if def.Username != smCfg.HarmonyDB.Username { + dbSettings += ` --db-user="` + smCfg.HarmonyDB.Username + `"` + } + if def.Password != smCfg.HarmonyDB.Password { + dbSettings += ` --db-password="` + smCfg.HarmonyDB.Password + `"` + } + if def.Database != smCfg.HarmonyDB.Database { + dbSettings += ` --db-name="` + smCfg.HarmonyDB.Database + `"` + } + + var layerMaybe string + if name != "base" { + layerMaybe = "--layer=" + name + } + + msg += ` +To work with the config: +` + cliCommandColor(`lotus-provider `+dbSettings+` config help `) + msg += ` +To run Lotus Provider: in its own machine or cgroup without other files, use the command: +` + cliCommandColor(`lotus-provider `+dbSettings+` run `+layerMaybe) + fmt.Println(msg) + return nil +} diff --git a/cmd/lotus-provider/proving.go b/cmd/lotus-provider/proving.go new file mode 100644 index 000000000..379bfdf85 --- /dev/null +++ b/cmd/lotus-provider/proving.go @@ -0,0 +1,207 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "os" + "time" + + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/dline" + + "github.com/filecoin-project/lotus/cmd/lotus-provider/deps" + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" + "github.com/filecoin-project/lotus/provider" +) + +var testCmd = &cli.Command{ + Name: "test", + Usage: "Utility functions for testing", + Subcommands: []*cli.Command{ + //provingInfoCmd, + wdPostCmd, + }, +} + +var wdPostCmd = &cli.Command{ + Name: "window-post", + Aliases: []string{"wd", "windowpost", "wdpost"}, + Usage: "Compute a proof-of-spacetime for a sector (requires the sector to be pre-sealed). These will not send to the chain.", + Subcommands: []*cli.Command{ + wdPostHereCmd, + wdPostTaskCmd, + }, +} + +// wdPostTaskCmd writes to harmony_task and wdpost_partition_tasks, then waits for the result. +// It is intended to be used to test the windowpost scheduler. +// The end of the compute task puts the task_id onto wdpost_proofs, which is read by the submit task. +// The submit task will not send test tasks to the chain, and instead will write the result to harmony_test. +// The result is read by this command, and printed to stdout. +var wdPostTaskCmd = &cli.Command{ + Name: "task", + Aliases: []string{"scheduled", "schedule", "async", "asynchronous"}, + Usage: "Test the windowpost scheduler by running it on the next available lotus-provider. ", + Flags: []cli.Flag{ + &cli.Uint64Flag{ + Name: "deadline", + Usage: "deadline to compute WindowPoSt for ", + Value: 0, + }, + &cli.StringSliceFlag{ + Name: "layers", + Usage: "list of layers to be interpreted (atop defaults). Default: base", + Value: cli.NewStringSlice("base"), + }, + }, + Action: func(cctx *cli.Context) error { + ctx := context.Background() + + deps, err := deps.GetDeps(ctx, cctx) + if err != nil { + return err + } + + ts, err := deps.Full.ChainHead(ctx) + if err != nil { + return xerrors.Errorf("cannot get chainhead %w", err) + } + ht := ts.Height() + + addr, err := address.NewFromString(deps.Cfg.Addresses.MinerAddresses[0]) + if err != nil { + return xerrors.Errorf("cannot get miner address %w", err) + } + maddr, err := address.IDFromAddress(addr) + if err != nil { + return xerrors.Errorf("cannot get miner id %w", err) + } + var id int64 + + retryDelay := time.Millisecond * 10 + retryAddTask: + _, err = deps.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + err = tx.QueryRow(`INSERT INTO harmony_task (name, posted_time, added_by) VALUES ('WdPost', CURRENT_TIMESTAMP, 123) RETURNING id`).Scan(&id) + if err != nil { + log.Error("inserting harmony_task: ", err) + return false, xerrors.Errorf("inserting harmony_task: %w", err) + } + _, err = tx.Exec(`INSERT INTO wdpost_partition_tasks + (task_id, sp_id, proving_period_start, deadline_index, partition_index) VALUES ($1, $2, $3, $4, $5)`, + id, maddr, ht, cctx.Uint64("deadline"), 0) + if err != nil { + log.Error("inserting wdpost_partition_tasks: ", err) + return false, xerrors.Errorf("inserting wdpost_partition_tasks: %w", err) + } + _, err = tx.Exec("INSERT INTO harmony_test (task_id) VALUES ($1)", id) + if err != nil { + return false, xerrors.Errorf("inserting into harmony_tests: %w", err) + } + return true, nil + }) + if err != nil { + if harmonydb.IsErrSerialization(err) { + time.Sleep(retryDelay) + retryDelay *= 2 + goto retryAddTask + } + return xerrors.Errorf("writing SQL transaction: %w", err) + } + fmt.Printf("Inserted task %v. Waiting for success ", id) + var result sql.NullString + for { + time.Sleep(time.Second) + err = deps.DB.QueryRow(ctx, `SELECT result FROM harmony_test WHERE task_id=$1`, id).Scan(&result) + if err != nil { + return xerrors.Errorf("reading result from harmony_test: %w", err) + } + if result.Valid { + break + } + fmt.Print(".") + } + log.Infof("Result: %s", result.String) + return nil + }, +} + +// This command is intended to be used to verify PoSt compute performance. +// It will not send any messages to the chain. Since it can compute any deadline, output may be incorrectly timed for the chain. +// The entire processing happens in this process while you wait. It does not use the scheduler. +var wdPostHereCmd = &cli.Command{ + Name: "here", + Aliases: []string{"cli"}, + Usage: "Compute WindowPoSt for performance and configuration testing.", + Description: `Note: This command is intended to be used to verify PoSt compute performance. +It will not send any messages to the chain. Since it can compute any deadline, output may be incorrectly timed for the chain.`, + ArgsUsage: "[deadline index]", + Flags: []cli.Flag{ + &cli.Uint64Flag{ + Name: "deadline", + Usage: "deadline to compute WindowPoSt for ", + Value: 0, + }, + &cli.StringSliceFlag{ + Name: "layers", + Usage: "list of layers to be interpreted (atop defaults). Default: base", + Value: cli.NewStringSlice("base"), + }, + &cli.StringFlag{ + Name: "storage-json", + Usage: "path to json file containing storage config", + Value: "~/.lotus-provider/storage.json", + }, + &cli.Uint64Flag{ + Name: "partition", + Usage: "partition to compute WindowPoSt for", + Value: 0, + }, + }, + Action: func(cctx *cli.Context) error { + + ctx := context.Background() + deps, err := deps.GetDeps(ctx, cctx) + if err != nil { + return err + } + + wdPostTask, wdPoStSubmitTask, derlareRecoverTask, err := provider.WindowPostScheduler(ctx, deps.Cfg.Fees, deps.Cfg.Proving, deps.Full, deps.Verif, deps.LW, nil, + deps.As, deps.Maddrs, deps.DB, deps.Stor, deps.Si, deps.Cfg.Subsystems.WindowPostMaxTasks) + if err != nil { + return err + } + _, _ = wdPoStSubmitTask, derlareRecoverTask + + if len(deps.Maddrs) == 0 { + return errors.New("no miners to compute WindowPoSt for") + } + head, err := deps.Full.ChainHead(ctx) + if err != nil { + return xerrors.Errorf("failed to get chain head: %w", err) + } + + di := dline.NewInfo(head.Height(), cctx.Uint64("deadline"), 0, 0, 0, 10 /*challenge window*/, 0, 0) + + for _, maddr := range deps.Maddrs { + out, err := wdPostTask.DoPartition(ctx, head, address.Address(maddr), di, cctx.Uint64("partition")) + if err != nil { + fmt.Println("Error computing WindowPoSt for miner", maddr, err) + continue + } + fmt.Println("Computed WindowPoSt for miner", maddr, ":") + err = json.NewEncoder(os.Stdout).Encode(out) + if err != nil { + fmt.Println("Could not encode WindowPoSt output for miner", maddr, err) + continue + } + } + + return nil + }, +} diff --git a/cmd/lotus-provider/rpc/rpc.go b/cmd/lotus-provider/rpc/rpc.go new file mode 100644 index 000000000..d77d8c81e --- /dev/null +++ b/cmd/lotus-provider/rpc/rpc.go @@ -0,0 +1,156 @@ +// Package rpc provides all direct access to this node. +package rpc + +import ( + "context" + "encoding/base64" + "encoding/json" + "net" + "net/http" + "time" + + "github.com/gbrlsnchs/jwt/v3" + "github.com/gorilla/mux" + logging "github.com/ipfs/go-log/v2" + "go.opencensus.io/tag" + "golang.org/x/sync/errgroup" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-jsonrpc" + "github.com/filecoin-project/go-jsonrpc/auth" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/cmd/lotus-provider/deps" + "github.com/filecoin-project/lotus/cmd/lotus-provider/web" + "github.com/filecoin-project/lotus/lib/rpcenc" + "github.com/filecoin-project/lotus/metrics" + "github.com/filecoin-project/lotus/metrics/proxy" + "github.com/filecoin-project/lotus/storage/paths" +) + +var log = logging.Logger("lp/rpc") + +func LotusProviderHandler( + authv func(ctx context.Context, token string) ([]auth.Permission, error), + remote http.HandlerFunc, + a api.LotusProvider, + permissioned bool) http.Handler { + mux := mux.NewRouter() + readerHandler, readerServerOpt := rpcenc.ReaderParamDecoder() + rpcServer := jsonrpc.NewServer(jsonrpc.WithServerErrors(api.RPCErrors), readerServerOpt) + + wapi := proxy.MetricedAPI[api.LotusProvider, api.LotusProviderStruct](a) + if permissioned { + wapi = api.PermissionedAPI[api.LotusProvider, api.LotusProviderStruct](wapi) + } + + rpcServer.Register("Filecoin", wapi) + rpcServer.AliasMethod("rpc.discover", "Filecoin.Discover") + + mux.Handle("/rpc/v0", rpcServer) + mux.Handle("/rpc/streams/v0/push/{uuid}", readerHandler) + mux.PathPrefix("/remote").HandlerFunc(remote) + mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof + + if !permissioned { + return mux + } + + ah := &auth.Handler{ + Verify: authv, + Next: mux.ServeHTTP, + } + return ah +} + +type ProviderAPI struct { + *deps.Deps + ShutdownChan chan struct{} +} + +func (p *ProviderAPI) Version(context.Context) (api.Version, error) { + return api.ProviderAPIVersion0, nil +} + +// Trigger shutdown +func (p *ProviderAPI) Shutdown(context.Context) error { + close(p.ShutdownChan) + return nil +} + +func ListenAndServe(ctx context.Context, dependencies *deps.Deps, shutdownChan chan struct{}) error { + + fh := &paths.FetchHandler{Local: dependencies.LocalStore, PfHandler: &paths.DefaultPartialFileHandler{}} + remoteHandler := func(w http.ResponseWriter, r *http.Request) { + if !auth.HasPerm(r.Context(), nil, api.PermAdmin) { + w.WriteHeader(401) + _ = json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing admin permission"}) + return + } + + fh.ServeHTTP(w, r) + } + // local APIs + { + // debugging + mux := mux.NewRouter() + mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof + mux.PathPrefix("/remote").HandlerFunc(remoteHandler) + } + + var authVerify func(context.Context, string) ([]auth.Permission, error) + { + privateKey, err := base64.StdEncoding.DecodeString(dependencies.Cfg.Apis.StorageRPCSecret) + if err != nil { + return xerrors.Errorf("decoding storage rpc secret: %w", err) + } + authVerify = func(ctx context.Context, token string) ([]auth.Permission, error) { + var payload deps.JwtPayload + if _, err := jwt.Verify([]byte(token), jwt.NewHS256(privateKey), &payload); err != nil { + return nil, xerrors.Errorf("JWT Verification failed: %w", err) + } + + return payload.Allow, nil + } + } + // Serve the RPC. + srv := &http.Server{ + Handler: LotusProviderHandler( + authVerify, + remoteHandler, + &ProviderAPI{dependencies, shutdownChan}, + true), + ReadHeaderTimeout: time.Minute * 3, + BaseContext: func(listener net.Listener) context.Context { + ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, "lotus-worker")) + return ctx + }, + Addr: dependencies.ListenAddr, + } + + log.Infof("Setting up RPC server at %s", dependencies.ListenAddr) + eg := errgroup.Group{} + eg.Go(srv.ListenAndServe) + + if dependencies.Cfg.Subsystems.EnableWebGui { + web, err := web.GetSrv(ctx, dependencies) + if err != nil { + return err + } + + go func() { + <-ctx.Done() + log.Warn("Shutting down...") + if err := srv.Shutdown(context.TODO()); err != nil { + log.Errorf("shutting down RPC server failed: %s", err) + } + if err := web.Shutdown(context.Background()); err != nil { + log.Errorf("shutting down web server failed: %s", err) + } + log.Warn("Graceful shutdown successful") + }() + log.Infof("Setting up web server at %s", dependencies.Cfg.Subsystems.GuiAddress) + eg.Go(web.ListenAndServe) + } + return eg.Wait() +} diff --git a/cmd/lotus-provider/run.go b/cmd/lotus-provider/run.go new file mode 100644 index 000000000..0f18f2843 --- /dev/null +++ b/cmd/lotus-provider/run.go @@ -0,0 +1,194 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/BurntSushi/toml" + "github.com/pkg/errors" + "github.com/urfave/cli/v2" + "go.opencensus.io/stats" + "go.opencensus.io/tag" + + "github.com/filecoin-project/lotus/build" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/cmd/lotus-provider/deps" + "github.com/filecoin-project/lotus/cmd/lotus-provider/rpc" + "github.com/filecoin-project/lotus/cmd/lotus-provider/tasks" + "github.com/filecoin-project/lotus/lib/ulimit" + "github.com/filecoin-project/lotus/metrics" + "github.com/filecoin-project/lotus/node" + "github.com/filecoin-project/lotus/node/config" +) + +type stackTracer interface { + StackTrace() errors.StackTrace +} + +var runCmd = &cli.Command{ + Name: "run", + Usage: "Start a lotus provider process", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "listen", + Usage: "host address and port the worker api will listen on", + Value: "0.0.0.0:12300", + EnvVars: []string{"LOTUS_WORKER_LISTEN"}, + }, + &cli.BoolFlag{ + Name: "nosync", + Usage: "don't check full-node sync status", + }, + &cli.BoolFlag{ + Name: "halt-after-init", + Usage: "only run init, then return", + Hidden: true, + }, + &cli.BoolFlag{ + Name: "manage-fdlimit", + Usage: "manage open file limit", + Value: true, + }, + &cli.StringSliceFlag{ + Name: "layers", + Usage: "list of layers to be interpreted (atop defaults). Default: base", + Value: cli.NewStringSlice("base"), + }, + &cli.StringFlag{ + Name: "storage-json", + Usage: "path to json file containing storage config", + Value: "~/.lotus-provider/storage.json", + }, + &cli.StringFlag{ + Name: "journal", + Usage: "path to journal files", + Value: "~/.lotus-provider/", + }, + }, + Action: func(cctx *cli.Context) (err error) { + defer func() { + if err != nil { + if err, ok := err.(stackTracer); ok { + for _, f := range err.StackTrace() { + fmt.Printf("%+s:%d\n", f, f) + } + } + } + }() + if !cctx.Bool("enable-gpu-proving") { + err := os.Setenv("BELLMAN_NO_GPU", "true") + if err != nil { + return err + } + } + + ctx, _ := tag.New(lcli.DaemonContext(cctx), + tag.Insert(metrics.Version, build.BuildVersion), + tag.Insert(metrics.Commit, build.CurrentCommit), + tag.Insert(metrics.NodeType, "provider"), + ) + shutdownChan := make(chan struct{}) + { + var ctxclose func() + ctx, ctxclose = context.WithCancel(ctx) + go func() { + <-shutdownChan + ctxclose() + }() + } + // Register all metric views + /* + if err := view.Register( + metrics.MinerNodeViews..., + ); err != nil { + log.Fatalf("Cannot register the view: %v", err) + } + */ + // Set the metric to one so it is published to the exporter + stats.Record(ctx, metrics.LotusInfo.M(1)) + + if cctx.Bool("manage-fdlimit") { + if _, _, err := ulimit.ManageFdLimit(); err != nil { + log.Errorf("setting file descriptor limit: %s", err) + } + } + + dependencies := &deps.Deps{} + err = dependencies.PopulateRemainingDeps(ctx, cctx, true) + if err != nil { + fmt.Println("err", err) + return err + } + fmt.Println("ef") + + taskEngine, err := tasks.StartTasks(ctx, dependencies) + fmt.Println("gh") + + if err != nil { + return nil + } + defer taskEngine.GracefullyTerminate(time.Hour) + + err = rpc.ListenAndServe(ctx, dependencies, shutdownChan) // Monitor for shutdown. + if err != nil { + return err + } + finishCh := node.MonitorShutdown(shutdownChan) //node.ShutdownHandler{Component: "rpc server", StopFunc: rpcStopper}, + //node.ShutdownHandler{Component: "provider", StopFunc: stop}, + + <-finishCh + return nil + }, +} + +var webCmd = &cli.Command{ + Name: "web", + Usage: "Start lotus provider web interface", + Description: `Start an instance of lotus provider web interface. + This creates the 'web' layer if it does not exist, then calls run with that layer.`, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "listen", + Usage: "Address to listen on", + Value: "127.0.0.1:4701", + }, + &cli.StringSliceFlag{ + Name: "layers", + Usage: "list of layers to be interpreted (atop defaults). Default: base. Web will be added", + Value: cli.NewStringSlice("base"), + }, + &cli.BoolFlag{ + Name: "nosync", + Usage: "don't check full-node sync status", + }, + }, + Action: func(cctx *cli.Context) error { + db, err := deps.MakeDB(cctx) + if err != nil { + return err + } + + webtxt, err := getConfig(db, "web") + if err != nil || webtxt == "" { + cfg := config.DefaultLotusProvider() + cfg.Subsystems.EnableWebGui = true + var b bytes.Buffer + if err = toml.NewEncoder(&b).Encode(cfg); err != nil { + return err + } + if err = setConfig(db, "web", b.String()); err != nil { + return err + } + } + layers := append([]string{"web"}, cctx.StringSlice("layers")...) + err = cctx.Set("layers", strings.Join(layers, ",")) + if err != nil { + return err + } + return runCmd.Action(cctx) + }, +} diff --git a/cmd/lotus-provider/stop.go b/cmd/lotus-provider/stop.go new file mode 100644 index 000000000..3376d762a --- /dev/null +++ b/cmd/lotus-provider/stop.go @@ -0,0 +1,29 @@ +package main + +import ( + _ "net/http/pprof" + + "github.com/urfave/cli/v2" + + lcli "github.com/filecoin-project/lotus/cli" +) + +var stopCmd = &cli.Command{ + Name: "stop", + Usage: "Stop a running lotus provider", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + api, closer, err := lcli.GetAPI(cctx) + if err != nil { + return err + } + defer closer() + + err = api.Shutdown(lcli.ReqContext(cctx)) + if err != nil { + return err + } + + return nil + }, +} diff --git a/cmd/lotus-provider/tasks/tasks.go b/cmd/lotus-provider/tasks/tasks.go new file mode 100644 index 000000000..2c4cd58bf --- /dev/null +++ b/cmd/lotus-provider/tasks/tasks.go @@ -0,0 +1,58 @@ +// Package tasks contains tasks that can be run by the lotus-provider command. +package tasks + +import ( + "context" + + logging "github.com/ipfs/go-log/v2" + "github.com/samber/lo" + + "github.com/filecoin-project/lotus/cmd/lotus-provider/deps" + "github.com/filecoin-project/lotus/lib/harmony/harmonytask" + "github.com/filecoin-project/lotus/provider" + "github.com/filecoin-project/lotus/provider/lpmessage" + "github.com/filecoin-project/lotus/provider/lpwinning" +) + +var log = logging.Logger("lotus-provider/deps") + +func StartTasks(ctx context.Context, dependencies *deps.Deps) (*harmonytask.TaskEngine, error) { + cfg := dependencies.Cfg + db := dependencies.DB + full := dependencies.Full + verif := dependencies.Verif + lw := dependencies.LW + as := dependencies.As + maddrs := dependencies.Maddrs + stor := dependencies.Stor + si := dependencies.Si + var activeTasks []harmonytask.TaskInterface + + sender, sendTask := lpmessage.NewSender(full, full, db) + activeTasks = append(activeTasks, sendTask) + + /////////////////////////////////////////////////////////////////////// + ///// Task Selection + /////////////////////////////////////////////////////////////////////// + { + + if cfg.Subsystems.EnableWindowPost { + wdPostTask, wdPoStSubmitTask, derlareRecoverTask, err := provider.WindowPostScheduler(ctx, cfg.Fees, cfg.Proving, full, verif, lw, sender, + as, maddrs, db, stor, si, cfg.Subsystems.WindowPostMaxTasks) + if err != nil { + return nil, err + } + activeTasks = append(activeTasks, wdPostTask, wdPoStSubmitTask, derlareRecoverTask) + } + + if cfg.Subsystems.EnableWinningPost { + winPoStTask := lpwinning.NewWinPostTask(cfg.Subsystems.WinningPostMaxTasks, db, lw, verif, full, maddrs) + activeTasks = append(activeTasks, winPoStTask) + } + } + log.Infow("This lotus_provider instance handles", + "miner_addresses", maddrs, + "tasks", lo.Map(activeTasks, func(t harmonytask.TaskInterface, _ int) string { return t.TypeDetails().Name })) + + return harmonytask.New(db, activeTasks, dependencies.ListenAddr) +} diff --git a/cmd/lotus-provider/web/api/debug/debug.go b/cmd/lotus-provider/web/api/debug/debug.go new file mode 100644 index 000000000..845684519 --- /dev/null +++ b/cmd/lotus-provider/web/api/debug/debug.go @@ -0,0 +1,229 @@ +// Package debug provides the API for various debug endpoints in lotus-provider. +package debug + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "sort" + "sync" + "time" + + "github.com/BurntSushi/toml" + "github.com/gorilla/mux" + logging "github.com/ipfs/go-log/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api/client" + "github.com/filecoin-project/lotus/build" + cliutil "github.com/filecoin-project/lotus/cli/util" + "github.com/filecoin-project/lotus/cmd/lotus-provider/deps" +) + +var log = logging.Logger("lp/web/debug") + +type debug struct { + *deps.Deps +} + +func Routes(r *mux.Router, deps *deps.Deps) { + d := debug{deps} + r.HandleFunc("/chain-state-sse", d.chainStateSSE) +} + +type rpcInfo struct { + Address string + CLayers []string + Reachable bool + SyncState string + Version string +} + +func (d *debug) chainStateSSE(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type") + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + + ctx := r.Context() + + for { + + type minimalApiInfo struct { + Apis struct { + ChainApiInfo []string + } + } + + rpcInfos := map[string]minimalApiInfo{} // config name -> api info + confNameToAddr := map[string]string{} // config name -> api address + + err := forEachConfig[minimalApiInfo](d, func(name string, info minimalApiInfo) error { + if len(info.Apis.ChainApiInfo) == 0 { + return nil + } + + rpcInfos[name] = info + + for _, addr := range info.Apis.ChainApiInfo { + ai := cliutil.ParseApiInfo(addr) + confNameToAddr[name] = ai.Addr + } + + return nil + }) + if err != nil { + log.Errorw("getting api info", "error", err) + return + } + + dedup := map[string]bool{} // for dedup by address + + infos := map[string]rpcInfo{} // api address -> rpc info + var infosLk sync.Mutex + + var wg sync.WaitGroup + for _, info := range rpcInfos { + ai := cliutil.ParseApiInfo(info.Apis.ChainApiInfo[0]) + if dedup[ai.Addr] { + continue + } + dedup[ai.Addr] = true + wg.Add(1) + go func() { + defer wg.Done() + var clayers []string + for layer, a := range confNameToAddr { + if a == ai.Addr { + clayers = append(clayers, layer) + } + } + + myinfo := rpcInfo{ + Address: ai.Addr, + Reachable: false, + CLayers: clayers, + } + defer func() { + infosLk.Lock() + defer infosLk.Unlock() + infos[ai.Addr] = myinfo + }() + da, err := ai.DialArgs("v1") + if err != nil { + log.Warnw("DialArgs", "error", err) + return + } + + ah := ai.AuthHeader() + + v1api, closer, err := client.NewFullNodeRPCV1(ctx, da, ah) + if err != nil { + log.Warnf("Not able to establish connection to node with addr: %s", ai.Addr) + return + } + defer closer() + + ver, err := v1api.Version(ctx) + if err != nil { + log.Warnw("Version", "error", err) + return + } + + head, err := v1api.ChainHead(ctx) + if err != nil { + log.Warnw("ChainHead", "error", err) + return + } + + var syncState string + switch { + case time.Now().Unix()-int64(head.MinTimestamp()) < int64(build.BlockDelaySecs*3/2): // within 1.5 epochs + syncState = "ok" + case time.Now().Unix()-int64(head.MinTimestamp()) < int64(build.BlockDelaySecs*5): // within 5 epochs + syncState = fmt.Sprintf("slow (%s behind)", time.Since(time.Unix(int64(head.MinTimestamp()), 0)).Truncate(time.Second)) + default: + syncState = fmt.Sprintf("behind (%s behind)", time.Since(time.Unix(int64(head.MinTimestamp()), 0)).Truncate(time.Second)) + } + + myinfo = rpcInfo{ + Address: ai.Addr, + CLayers: clayers, + Reachable: true, + Version: ver.Version, + SyncState: syncState, + } + }() + } + wg.Wait() + + var infoList []rpcInfo + for _, i := range infos { + infoList = append(infoList, i) + } + sort.Slice(infoList, func(i, j int) bool { + return infoList[i].Address < infoList[j].Address + }) + + fmt.Fprintf(w, "data: ") + err = json.NewEncoder(w).Encode(&infoList) + if err != nil { + log.Warnw("json encode", "error", err) + return + } + fmt.Fprintf(w, "\n\n") + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + + time.Sleep(time.Duration(build.BlockDelaySecs) * time.Second) + + select { // stop running if there is reader. + case <-ctx.Done(): + return + default: + } + } +} + +func forEachConfig[T any](a *debug, cb func(name string, v T) error) error { + confs, err := a.loadConfigs(context.Background()) + if err != nil { + return err + } + + for name, tomlStr := range confs { // todo for-each-config + var info T + if err := toml.Unmarshal([]byte(tomlStr), &info); err != nil { + return xerrors.Errorf("unmarshaling %s config: %w", name, err) + } + + if err := cb(name, info); err != nil { + return xerrors.Errorf("cb: %w", err) + } + } + + return nil +} + +func (d *debug) loadConfigs(ctx context.Context) (map[string]string, error) { + //err := db.QueryRow(cctx.Context, `SELECT config FROM harmony_config WHERE title=$1`, layer).Scan(&text) + + rows, err := d.DB.Query(ctx, `SELECT title, config FROM harmony_config`) + if err != nil { + return nil, xerrors.Errorf("getting db configs: %w", err) + } + + configs := make(map[string]string) + for rows.Next() { + var title, config string + if err := rows.Scan(&title, &config); err != nil { + return nil, xerrors.Errorf("scanning db configs: %w", err) + } + configs[title] = config + } + + return configs, nil +} diff --git a/cmd/lotus-provider/web/api/routes.go b/cmd/lotus-provider/web/api/routes.go new file mode 100644 index 000000000..9bb6fb67c --- /dev/null +++ b/cmd/lotus-provider/web/api/routes.go @@ -0,0 +1,13 @@ +// Package api provides the HTTP API for the lotus provider web gui. +package api + +import ( + "github.com/gorilla/mux" + + "github.com/filecoin-project/lotus/cmd/lotus-provider/deps" + "github.com/filecoin-project/lotus/cmd/lotus-provider/web/api/debug" +) + +func Routes(r *mux.Router, deps *deps.Deps) { + debug.Routes(r.PathPrefix("/debug").Subrouter(), deps) +} diff --git a/cmd/lotus-provider/web/hapi/routes.go b/cmd/lotus-provider/web/hapi/routes.go new file mode 100644 index 000000000..b07ab60a5 --- /dev/null +++ b/cmd/lotus-provider/web/hapi/routes.go @@ -0,0 +1,35 @@ +package hapi + +import ( + "embed" + "html/template" + + "github.com/gorilla/mux" + logging "github.com/ipfs/go-log/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/cmd/lotus-provider/deps" +) + +//go:embed web/* +var templateFS embed.FS + +func Routes(r *mux.Router, deps *deps.Deps) error { + t, err := template.ParseFS(templateFS, "web/*") + if err != nil { + return xerrors.Errorf("parse templates: %w", err) + } + + a := &app{ + db: deps.DB, + t: t, + } + + r.HandleFunc("/simpleinfo/actorsummary", a.actorSummary) + r.HandleFunc("/simpleinfo/machines", a.indexMachines) + r.HandleFunc("/simpleinfo/tasks", a.indexTasks) + r.HandleFunc("/simpleinfo/taskhistory", a.indexTasksHistory) + return nil +} + +var log = logging.Logger("lpweb") diff --git a/cmd/lotus-provider/web/hapi/simpleinfo.go b/cmd/lotus-provider/web/hapi/simpleinfo.go new file mode 100644 index 000000000..a14735a84 --- /dev/null +++ b/cmd/lotus-provider/web/hapi/simpleinfo.go @@ -0,0 +1,187 @@ +package hapi + +import ( + "context" + "html/template" + "net/http" + "os" + "sync" + "time" + + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" +) + +type app struct { + db *harmonydb.DB + t *template.Template + + actorInfoLk sync.Mutex + actorInfos []actorInfo +} + +type actorInfo struct { + Address string + CLayers []string + + QualityAdjustedPower string + RawBytePower string + + Deadlines []actorDeadline +} + +type actorDeadline struct { + Empty bool + Current bool + Proven bool + PartFaulty bool + Faulty bool +} + +func (a *app) actorSummary(w http.ResponseWriter, r *http.Request) { + a.actorInfoLk.Lock() + defer a.actorInfoLk.Unlock() + + a.executeTemplate(w, "actor_summary", a.actorInfos) +} + +func (a *app) indexMachines(w http.ResponseWriter, r *http.Request) { + s, err := a.clusterMachineSummary(r.Context()) + if err != nil { + log.Errorf("cluster machine summary: %v", err) + http.Error(w, "internal server error", http.StatusInternalServerError) + return + } + + a.executeTemplate(w, "cluster_machines", s) +} + +func (a *app) indexTasks(w http.ResponseWriter, r *http.Request) { + s, err := a.clusterTaskSummary(r.Context()) + if err != nil { + log.Errorf("cluster task summary: %v", err) + http.Error(w, "internal server error", http.StatusInternalServerError) + return + } + + a.executeTemplate(w, "cluster_tasks", s) +} + +func (a *app) indexTasksHistory(w http.ResponseWriter, r *http.Request) { + s, err := a.clusterTaskHistorySummary(r.Context()) + if err != nil { + log.Errorf("cluster task history summary: %v", err) + http.Error(w, "internal server error", http.StatusInternalServerError) + return + } + + a.executeTemplate(w, "cluster_task_history", s) +} + +var templateDev = os.Getenv("LOTUS_WEB_DEV") == "1" + +func (a *app) executeTemplate(w http.ResponseWriter, name string, data interface{}) { + if templateDev { + fs := os.DirFS("./cmd/lotus-provider/web/hapi/web") + a.t = template.Must(template.ParseFS(fs, "web/*")) + } + if err := a.t.ExecuteTemplate(w, name, data); err != nil { + log.Errorf("execute template %s: %v", name, err) + http.Error(w, "internal server error", http.StatusInternalServerError) + } +} + +type machineSummary struct { + Address string + ID int64 + SinceContact string +} + +type taskSummary struct { + Name string + SincePosted string + Owner *string + ID int64 +} + +type taskHistorySummary struct { + Name string + TaskID int64 + + Posted, Start, End string + + Result bool + Err string + + CompletedBy string +} + +func (a *app) clusterMachineSummary(ctx context.Context) ([]machineSummary, error) { + rows, err := a.db.Query(ctx, "SELECT id, host_and_port, last_contact FROM harmony_machines") + if err != nil { + return nil, err // Handle error + } + defer rows.Close() + + var summaries []machineSummary + for rows.Next() { + var m machineSummary + var lastContact time.Time + + if err := rows.Scan(&m.ID, &m.Address, &lastContact); err != nil { + return nil, err // Handle error + } + + m.SinceContact = time.Since(lastContact).Round(time.Second).String() + + summaries = append(summaries, m) + } + return summaries, nil +} + +func (a *app) clusterTaskSummary(ctx context.Context) ([]taskSummary, error) { + rows, err := a.db.Query(ctx, "SELECT id, name, update_time, owner_id FROM harmony_task") + if err != nil { + return nil, err // Handle error + } + defer rows.Close() + + var summaries []taskSummary + for rows.Next() { + var t taskSummary + var posted time.Time + + if err := rows.Scan(&t.ID, &t.Name, &posted, &t.Owner); err != nil { + return nil, err // Handle error + } + + t.SincePosted = time.Since(posted).Round(time.Second).String() + + summaries = append(summaries, t) + } + return summaries, nil +} + +func (a *app) clusterTaskHistorySummary(ctx context.Context) ([]taskHistorySummary, error) { + rows, err := a.db.Query(ctx, "SELECT id, name, task_id, posted, work_start, work_end, result, err, completed_by_host_and_port FROM harmony_task_history ORDER BY work_end DESC LIMIT 15") + if err != nil { + return nil, err // Handle error + } + defer rows.Close() + + var summaries []taskHistorySummary + for rows.Next() { + var t taskHistorySummary + var posted, start, end time.Time + + if err := rows.Scan(&t.TaskID, &t.Name, &t.TaskID, &posted, &start, &end, &t.Result, &t.Err, &t.CompletedBy); err != nil { + return nil, err // Handle error + } + + t.Posted = posted.Round(time.Second).Format("02 Jan 06 15:04") + t.Start = start.Round(time.Second).Format("02 Jan 06 15:04") + t.End = end.Round(time.Second).Format("02 Jan 06 15:04") + + summaries = append(summaries, t) + } + return summaries, nil +} diff --git a/cmd/lotus-provider/web/hapi/web/actor_summary.gohtml b/cmd/lotus-provider/web/hapi/web/actor_summary.gohtml new file mode 100644 index 000000000..31992fb23 --- /dev/null +++ b/cmd/lotus-provider/web/hapi/web/actor_summary.gohtml @@ -0,0 +1,20 @@ +{{define "actor_summary"}} +{{range .}} + + {{.Address}} + + {{range .CLayers}} + {{.}} + {{end}} + + {{.QualityAdjustedPower}} + +
+ {{range .Deadlines}} +
+ {{end}} +
+ + +{{end}} +{{end}} \ No newline at end of file diff --git a/cmd/lotus-provider/web/hapi/web/chain_rpcs.gohtml b/cmd/lotus-provider/web/hapi/web/chain_rpcs.gohtml new file mode 100644 index 000000000..5705da395 --- /dev/null +++ b/cmd/lotus-provider/web/hapi/web/chain_rpcs.gohtml @@ -0,0 +1,15 @@ +{{define "chain_rpcs"}} +{{range .}} + + {{.Address}} + + {{range .CLayers}} + {{.}} + {{end}} + + {{if .Reachable}}ok{{else}}FAIL{{end}} + {{if eq "ok" .SyncState}}ok{{else}}{{.SyncState}}{{end}} + {{.Version}} + +{{end}} +{{end}} diff --git a/cmd/lotus-provider/web/hapi/web/cluster_machines.gohtml b/cmd/lotus-provider/web/hapi/web/cluster_machines.gohtml new file mode 100644 index 000000000..f94f53bf8 --- /dev/null +++ b/cmd/lotus-provider/web/hapi/web/cluster_machines.gohtml @@ -0,0 +1,10 @@ +{{define "cluster_machines"}} +{{range .}} + + {{.Address}} + {{.ID}} + todo + {{.SinceContact}} + +{{end}} +{{end}} diff --git a/cmd/lotus-provider/web/hapi/web/cluster_task_history.gohtml b/cmd/lotus-provider/web/hapi/web/cluster_task_history.gohtml new file mode 100644 index 000000000..8f04ef5c5 --- /dev/null +++ b/cmd/lotus-provider/web/hapi/web/cluster_task_history.gohtml @@ -0,0 +1,14 @@ +{{define "cluster_task_history"}} + {{range .}} + + {{.Name}} + {{.TaskID}} + {{.CompletedBy}} + {{.Posted}} + {{.Start}} + {{.End}} + {{if .Result}}success{{else}}error{{end}} + {{.Err}} + + {{end}} +{{end}} diff --git a/cmd/lotus-provider/web/hapi/web/cluster_tasks.gohtml b/cmd/lotus-provider/web/hapi/web/cluster_tasks.gohtml new file mode 100644 index 000000000..690ab8cff --- /dev/null +++ b/cmd/lotus-provider/web/hapi/web/cluster_tasks.gohtml @@ -0,0 +1,10 @@ +{{define "cluster_tasks"}} + {{range .}} + + {{.Name}} + {{.ID}} + {{.SincePosted}} + {{.Owner}} + + {{end}} +{{end}} diff --git a/cmd/lotus-provider/web/srv.go b/cmd/lotus-provider/web/srv.go new file mode 100644 index 000000000..55d20cc9a --- /dev/null +++ b/cmd/lotus-provider/web/srv.go @@ -0,0 +1,84 @@ +// Package web defines the HTTP web server for static files and endpoints. +package web + +import ( + "context" + "embed" + "io" + "io/fs" + "net" + "net/http" + "os" + "path" + "strings" + "time" + + "github.com/gorilla/mux" + "go.opencensus.io/tag" + + "github.com/filecoin-project/lotus/cmd/lotus-provider/deps" + "github.com/filecoin-project/lotus/cmd/lotus-provider/web/api" + "github.com/filecoin-project/lotus/cmd/lotus-provider/web/hapi" + "github.com/filecoin-project/lotus/metrics" +) + +//go:embed static +var static embed.FS + +var basePath = "/static/" + +// An dev mode hack for no-restart changes to static and templates. +// You still need to recomplie the binary for changes to go code. +var webDev = os.Getenv("LOTUS_WEB_DEV") == "1" + +func GetSrv(ctx context.Context, deps *deps.Deps) (*http.Server, error) { + mx := mux.NewRouter() + err := hapi.Routes(mx.PathPrefix("/hapi").Subrouter(), deps) + if err != nil { + return nil, err + } + api.Routes(mx.PathPrefix("/api").Subrouter(), deps) + + basePath := basePath + + var static fs.FS = static + if webDev { + basePath = "cmd/lotus-provider/web/static" + static = os.DirFS(basePath) + } + + mx.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // If the request is for a directory, redirect to the index file. + if strings.HasSuffix(r.URL.Path, "/") { + r.URL.Path += "index.html" + } + + file, err := static.Open(path.Join(basePath, r.URL.Path)[1:]) + if err != nil { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("404 Not Found")) + return + } + defer func() { _ = file.Close() }() + + fileInfo, err := file.Stat() + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte("500 Internal Server Error")) + return + } + + http.ServeContent(w, r, fileInfo.Name(), fileInfo.ModTime(), file.(io.ReadSeeker)) + }) + + return &http.Server{ + Handler: http.HandlerFunc(mx.ServeHTTP), + BaseContext: func(listener net.Listener) context.Context { + ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, "lotus-provider")) + return ctx + }, + Addr: deps.Cfg.Subsystems.GuiAddress, + ReadTimeout: time.Minute * 3, + ReadHeaderTimeout: time.Minute * 3, // lint + }, nil +} diff --git a/cmd/lotus-provider/web/static/chain-connectivity.js b/cmd/lotus-provider/web/static/chain-connectivity.js new file mode 100644 index 000000000..ea7349c41 --- /dev/null +++ b/cmd/lotus-provider/web/static/chain-connectivity.js @@ -0,0 +1,73 @@ +import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/all/lit-all.min.js'; +window.customElements.define('chain-connectivity', class MyElement extends LitElement { + constructor() { + super(); + this.data = []; + this.loadData(); + } + loadData() { + const eventSource = new EventSource('/api/debug/chain-state-sse'); + eventSource.onmessage = (event) => { + this.data = JSON.parse(event.data); + super.requestUpdate(); + }; + eventSource.onerror = (error) => { + console.error('Error:', error); + loadData(); + }; + }; + + static get styles() { + return [css` + :host { + box-sizing: border-box; /* Don't forgert this to include padding/border inside width calculation */ + } + table { + border-collapse: collapse; + } + + table td, table th { + border-left: 1px solid #f0f0f0; + padding: 1px 5px; + } + + table tr td:first-child, table tr th:first-child { + border-left: none; + } + + .success { + color: green; + } + .warning { + color: yellow; + } + .error { + color: red; + } + `]; + } + render = () => html` + + + + + + + + + + + ${this.data.map(item => html` + + + + + + + `)} + + + + +
RPC AddressReachabilitySync StatusVersion
${item.Address}${item.Reachable ? html`ok` : html`FAIL`}${item.SyncState === "ok" ? html`ok` : html`${item.SyncState}`}${item.Version}
Data incoming...
` +}); diff --git a/cmd/lotus-provider/web/static/index.html b/cmd/lotus-provider/web/static/index.html new file mode 100644 index 000000000..98f7336ad --- /dev/null +++ b/cmd/lotus-provider/web/static/index.html @@ -0,0 +1,193 @@ + + + Lotus Provider Cluster Overview + + + + + +
+
+

Lotus Provider Cluster

+
+
+ version [todo] +
+
+
+
+
+

Chain Connectivity

+ +
+
+
+

Actor Summary

+ + + + + + + + + + + +
AddressConfig LayersQaPDeadlines
+
+
+
+

Cluster Machines

+ + + + + + + + + + + +
HostIDConfig LayersLast Contact
+
+
+
+

Recently Finished Tasks

+ + + + + + + + + + + + + + + +
NameIDExecutorPostedStartEndOutcomeMessage
+
+
+
+

Cluster Tasks

+ + + + + + + + + + + +
TaskIDPostedOwner
+
+
+ + \ No newline at end of file diff --git a/cmd/lotus-shed/invariants.go b/cmd/lotus-shed/invariants.go index e74a0dd24..5c6fb2d4f 100644 --- a/cmd/lotus-shed/invariants.go +++ b/cmd/lotus-shed/invariants.go @@ -3,7 +3,8 @@ package main import ( "context" "fmt" - "io" + "os" + "path/filepath" "strconv" "time" @@ -21,6 +22,8 @@ import ( v9 "github.com/filecoin-project/go-state-types/builtin/v9" "github.com/filecoin-project/lotus/blockstore" + badgerbs "github.com/filecoin-project/lotus/blockstore/badger" + "github.com/filecoin-project/lotus/blockstore/splitstore" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/consensus" "github.com/filecoin-project/lotus/chain/consensus/filcns" @@ -73,24 +76,52 @@ var invariantsCmd = &cli.Command{ defer lkrepo.Close() //nolint:errcheck - bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) + cold, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { - return fmt.Errorf("failed to open blockstore: %w", err) + return fmt.Errorf("failed to open universal blockstore %w", err) } - defer func() { - if c, ok := bs.(io.Closer); ok { - if err := c.Close(); err != nil { - log.Warnf("failed to close blockstore: %s", err) - } - } - }() + path, err := lkrepo.SplitstorePath() + if err != nil { + return err + } + + path = filepath.Join(path, "hot.badger") + if err := os.MkdirAll(path, 0755); err != nil { + return err + } + + opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, lkrepo.Readonly()) + if err != nil { + return err + } + + hot, err := badgerbs.Open(opts) + if err != nil { + return err + } mds, err := lkrepo.Datastore(context.Background(), "/metadata") if err != nil { return err } + cfg := &splitstore.Config{ + MarkSetType: "map", + DiscardColdBlocks: true, + } + ss, err := splitstore.Open(path, mds, hot, cold, cfg) + if err != nil { + return err + } + defer func() { + if err := ss.Close(); err != nil { + log.Warnf("failed to close blockstore: %s", err) + + } + }() + bs := ss + cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil) defer cs.Close() //nolint:errcheck diff --git a/cmd/lotus-shed/miner.go b/cmd/lotus-shed/miner.go index a8bb93744..2f9b4ecf1 100644 --- a/cmd/lotus-shed/miner.go +++ b/cmd/lotus-shed/miner.go @@ -553,7 +553,7 @@ var sendInvalidWindowPoStCmd = &cli.Command{ return xerrors.Errorf("serializing params: %w", err) } - fmt.Printf("submitting bad PoST for %d paritions\n", len(partitionIndices)) + fmt.Printf("submitting bad PoST for %d partitions\n", len(partitionIndices)) smsg, err := api.MpoolPushMessage(ctx, &types.Message{ From: minfo.Worker, To: maddr, diff --git a/cmd/lotus-shed/terminations.go b/cmd/lotus-shed/terminations.go index c5f35995a..563c1ba3a 100644 --- a/cmd/lotus-shed/terminations.go +++ b/cmd/lotus-shed/terminations.go @@ -157,7 +157,8 @@ var terminationsCmd = &cli.Command{ } for _, t := range termParams.Terminations { - sectors, err := minerSt.LoadSectors(&t.Sectors) + tmp := t.Sectors + sectors, err := minerSt.LoadSectors(&tmp) if err != nil { return err } diff --git a/cmd/lotus-sim/simulation/stages/funding_stage.go b/cmd/lotus-sim/simulation/stages/funding_stage.go index f75a9910d..4ce4afae1 100644 --- a/cmd/lotus-sim/simulation/stages/funding_stage.go +++ b/cmd/lotus-sim/simulation/stages/funding_stage.go @@ -166,7 +166,8 @@ func (fs *FundingStage) PackMessages(ctx context.Context, bb *blockbuilder.Block ) }() - for _, actor := range targets { + for _, actorTmp := range targets { + actor := actorTmp switch { case builtin.IsAccountActor(actor.Code): if _, err := bb.PushMessage(&types.Message{ diff --git a/cmd/lotus-worker/main.go b/cmd/lotus-worker/main.go index 257dac800..41af11bdd 100644 --- a/cmd/lotus-worker/main.go +++ b/cmd/lotus-worker/main.go @@ -39,6 +39,7 @@ import ( "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage/paths" "github.com/filecoin-project/lotus/storage/sealer" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" "github.com/filecoin-project/lotus/storage/sealer/sealtasks" "github.com/filecoin-project/lotus/storage/sealer/storiface" ) @@ -284,7 +285,36 @@ var runCmd = &cli.Command{ Value: true, DefaultText: "inherits --addpiece", }, + &cli.StringFlag{ + Name: "external-pc2", + Usage: "command for computing PC2 externally", + }, }, + Description: `Run lotus-worker. + +--external-pc2 can be used to compute the PreCommit2 inputs externally. +The flag behaves similarly to the related lotus-worker flag, using it in +lotus-bench may be useful for testing if the external PreCommit2 command is +invoked correctly. + +The command will be called with a number of environment variables set: +* EXTSEAL_PC2_SECTOR_NUM: the sector number +* EXTSEAL_PC2_SECTOR_MINER: the miner id +* EXTSEAL_PC2_PROOF_TYPE: the proof type +* EXTSEAL_PC2_SECTOR_SIZE: the sector size in bytes +* EXTSEAL_PC2_CACHE: the path to the cache directory +* EXTSEAL_PC2_SEALED: the path to the sealed sector file (initialized with unsealed data by the caller) +* EXTSEAL_PC2_PC1OUT: output from rust-fil-proofs precommit1 phase (base64 encoded json) + +The command is expected to: +* Create cache sc-02-data-tree-r* files +* Create cache sc-02-data-tree-c* files +* Create cache p_aux / t_aux files +* Transform the sealed file in place + +Example invocation of lotus-bench as external executor: +'./lotus-bench simple precommit2 --sector-size $EXTSEAL_PC2_SECTOR_SIZE $EXTSEAL_PC2_SEALED $EXTSEAL_PC2_CACHE $EXTSEAL_PC2_PC1OUT' +`, Before: func(cctx *cli.Context) error { if cctx.IsSet("address") { log.Warnf("The '--address' flag is deprecated, it has been replaced by '--listen'") @@ -623,18 +653,32 @@ var runCmd = &cli.Command{ fh.ServeHTTP(w, r) } + // Parse ffi executor flags + + var ffiOpts []ffiwrapper.FFIWrapperOpt + + if cctx.IsSet("external-pc2") { + extSeal := ffiwrapper.ExternalSealer{ + PreCommit2: ffiwrapper.MakeExternPrecommit2(cctx.String("external-pc2")), + } + + ffiOpts = append(ffiOpts, ffiwrapper.WithExternalSealCalls(extSeal)) + } + // Create / expose the worker wsts := statestore.New(namespace.Wrap(ds, modules.WorkerCallsPrefix)) workerApi := &sealworker.Worker{ - LocalWorker: sealer.NewLocalWorker(sealer.WorkerConfig{ - TaskTypes: taskTypes, - NoSwap: cctx.Bool("no-swap"), - MaxParallelChallengeReads: cctx.Int("post-parallel-reads"), - ChallengeReadTimeout: cctx.Duration("post-read-timeout"), - Name: cctx.String("name"), - }, remote, localStore, nodeApi, nodeApi, wsts), + LocalWorker: sealer.NewLocalWorkerWithExecutor( + sealer.FFIExec(ffiOpts...), + sealer.WorkerConfig{ + TaskTypes: taskTypes, + NoSwap: cctx.Bool("no-swap"), + MaxParallelChallengeReads: cctx.Int("post-parallel-reads"), + ChallengeReadTimeout: cctx.Duration("post-read-timeout"), + Name: cctx.String("name"), + }, os.LookupEnv, remote, localStore, nodeApi, nodeApi, wsts), LocalStore: localStore, Storage: lr, } diff --git a/cmd/lotus-worker/sealworker/rpc.go b/cmd/lotus-worker/sealworker/rpc.go index 97f78942e..4e720ef64 100644 --- a/cmd/lotus-worker/sealworker/rpc.go +++ b/cmd/lotus-worker/sealworker/rpc.go @@ -26,7 +26,11 @@ import ( var log = logging.Logger("sealworker") -func WorkerHandler(authv func(ctx context.Context, token string) ([]auth.Permission, error), remote http.HandlerFunc, a api.Worker, permissioned bool) http.Handler { +func WorkerHandler( + authv func(ctx context.Context, token string) ([]auth.Permission, error), + remote http.HandlerFunc, + a api.Worker, + permissioned bool) http.Handler { mux := mux.NewRouter() readerHandler, readerServerOpt := rpcenc.ReaderParamDecoder() rpcServer := jsonrpc.NewServer(jsonrpc.WithServerErrors(api.RPCErrors), readerServerOpt) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 44da4139a..5d8096d1f 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -269,6 +269,26 @@ var DaemonCmd = &cli.Command{ } } + if cctx.Bool("remove-existing-chain") { + lr, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return xerrors.Errorf("error opening fs repo: %w", err) + } + + exists, err := lr.Exists() + if err != nil { + return err + } + if !exists { + return xerrors.Errorf("lotus repo doesn't exist") + } + + err = removeExistingChain(cctx, lr) + if err != nil { + return err + } + } + chainfile := cctx.String("import-chain") snapshot := cctx.String("import-snapshot") willImportChain := false diff --git a/documentation/en/api-v0-methods-provider.md b/documentation/en/api-v0-methods-provider.md new file mode 100644 index 000000000..fc4a2daf7 --- /dev/null +++ b/documentation/en/api-v0-methods-provider.md @@ -0,0 +1,25 @@ +# Groups +* [](#) + * [Shutdown](#Shutdown) + * [Version](#Version) +## + + +### Shutdown + + +Perms: admin + +Inputs: `null` + +Response: `{}` + +### Version + + +Perms: admin + +Inputs: `null` + +Response: `131840` + diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index f3fb8a9e1..2ea89e6ce 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -7,7 +7,7 @@ USAGE: lotus-miner [global options] command [command options] [arguments...] VERSION: - 1.25.1-dev + 1.25.3-dev COMMANDS: init Initialize a lotus miner repo @@ -66,6 +66,7 @@ OPTIONS: --no-local-storage don't use storageminer repo for sector storage (default: false) --gas-premium value set gas premium for initialization messages in AttoFIL (default: "0") --from value select which address to send actor creation message from + --confidence value number of block confirmations to wait for (default: 5) --help, -h show help ``` @@ -231,8 +232,19 @@ OPTIONS: --help, -h show help ``` -#### lotus-miner actor set-addresses, set-addrs +### lotus-miner actor set-addresses ``` +NAME: + lotus-miner actor set-addresses - set addresses that your miner can be publicly dialed on + +USAGE: + lotus-miner actor set-addresses [command options] + +OPTIONS: + --from value optionally specify the account to send the message from + --gas-limit value set gas limit (default: 0) + --unset unset address (default: false) + --help, -h show help ``` ### lotus-miner actor withdraw @@ -1161,8 +1173,20 @@ OPTIONS: --help, -h show help ``` -##### lotus-miner proving compute windowed-post, window-post +#### lotus-miner proving compute windowed-post ``` +NAME: + lotus-miner proving compute windowed-post - Compute WindowPoSt for a specific deadline + +USAGE: + lotus-miner proving compute windowed-post [command options] [deadline index] + +DESCRIPTION: + Note: This command is intended to be used to verify PoSt compute performance. + It will not send any messages to the chain. + +OPTIONS: + --help, -h show help ``` ### lotus-miner proving recover-faults diff --git a/documentation/en/cli-lotus-provider.md b/documentation/en/cli-lotus-provider.md new file mode 100644 index 000000000..5e5864107 --- /dev/null +++ b/documentation/en/cli-lotus-provider.md @@ -0,0 +1,430 @@ +# lotus-provider +``` +NAME: + lotus-provider - Filecoin decentralized storage network provider + +USAGE: + lotus-provider [global options] command [command options] [arguments...] + +VERSION: + 1.25.3-dev + +COMMANDS: + run Start a lotus provider process + stop Stop a running lotus provider + config Manage node config by layers. The layer 'base' will always be applied. + test Utility functions for testing + web Start lotus provider web interface + version Print version + help, h Shows a list of commands or help for one command + DEVELOPER: + auth Manage RPC permissions + log Manage logging + wait-api Wait for lotus api to come online + fetch-params Fetch proving parameters + +GLOBAL OPTIONS: + --color use color in display output (default: depends on output being a TTY) + --db-host value Command separated list of hostnames for yugabyte cluster (default: "yugabyte") [$LOTUS_DB_HOST] + --db-name value (default: "yugabyte") [$LOTUS_DB_NAME, $LOTUS_HARMONYDB_HOSTS] + --db-user value (default: "yugabyte") [$LOTUS_DB_USER, $LOTUS_HARMONYDB_USERNAME] + --db-password value (default: "yugabyte") [$LOTUS_DB_PASSWORD, $LOTUS_HARMONYDB_PASSWORD] + --layers value (default: "base") [$LOTUS_LAYERS, $LOTUS_CONFIG_LAYERS] + --repo-path value (default: "~/.lotusprovider") [$LOTUS_REPO_PATH] + --vv enables very verbose mode, useful for debugging the CLI (default: false) + --help, -h show help + --version, -v print the version +``` + +## lotus-provider run +``` +NAME: + lotus-provider run - Start a lotus provider process + +USAGE: + lotus-provider run [command options] [arguments...] + +OPTIONS: + --listen value host address and port the worker api will listen on (default: "0.0.0.0:12300") [$LOTUS_WORKER_LISTEN] + --nosync don't check full-node sync status (default: false) + --manage-fdlimit manage open file limit (default: true) + --layers value [ --layers value ] list of layers to be interpreted (atop defaults). Default: base (default: "base") + --storage-json value path to json file containing storage config (default: "~/.lotus-provider/storage.json") + --journal value path to journal files (default: "~/.lotus-provider/") + --help, -h show help +``` + +## lotus-provider stop +``` +NAME: + lotus-provider stop - Stop a running lotus provider + +USAGE: + lotus-provider stop [command options] [arguments...] + +OPTIONS: + --help, -h show help +``` + +## lotus-provider config +``` +NAME: + lotus-provider config - Manage node config by layers. The layer 'base' will always be applied. + +USAGE: + lotus-provider config command [command options] [arguments...] + +COMMANDS: + default, defaults Print default node config + set, add, update, create Set a config layer or the base by providing a filename or stdin. + get, cat, show Get a config layer by name. You may want to pipe the output to a file, or use 'less' + list, ls List config layers you can get. + interpret, view, stacked, stack Interpret stacked config layers by this version of lotus-provider, with system-generated comments. + remove, rm, del, delete Remove a named config layer. + from-miner Express a database config (for lotus-provider) from an existing miner. + help, h Shows a list of commands or help for one command + +OPTIONS: + --help, -h show help +``` + +### lotus-provider config default +``` +NAME: + lotus-provider config default - Print default node config + +USAGE: + lotus-provider config default [command options] [arguments...] + +OPTIONS: + --no-comment don't comment default values (default: false) + --help, -h show help +``` + +### lotus-provider config set +``` +NAME: + lotus-provider config set - Set a config layer or the base by providing a filename or stdin. + +USAGE: + lotus-provider config set [command options] a layer's file name + +OPTIONS: + --title value title of the config layer (req'd for stdin) + --help, -h show help +``` + +### lotus-provider config get +``` +NAME: + lotus-provider config get - Get a config layer by name. You may want to pipe the output to a file, or use 'less' + +USAGE: + lotus-provider config get [command options] layer name + +OPTIONS: + --help, -h show help +``` + +### lotus-provider config list +``` +NAME: + lotus-provider config list - List config layers you can get. + +USAGE: + lotus-provider config list [command options] [arguments...] + +OPTIONS: + --help, -h show help +``` + +### lotus-provider config interpret +``` +NAME: + lotus-provider config interpret - Interpret stacked config layers by this version of lotus-provider, with system-generated comments. + +USAGE: + lotus-provider config interpret [command options] a list of layers to be interpreted as the final config + +OPTIONS: + --layers value [ --layers value ] comma or space separated list of layers to be interpreted (default: "base") + --help, -h show help +``` + +### lotus-provider config remove +``` +NAME: + lotus-provider config remove - Remove a named config layer. + +USAGE: + lotus-provider config remove [command options] [arguments...] + +OPTIONS: + --help, -h show help +``` + +### lotus-provider config from-miner +``` +NAME: + lotus-provider config from-miner - Express a database config (for lotus-provider) from an existing miner. + +USAGE: + lotus-provider config from-miner [command options] [arguments...] + +DESCRIPTION: + Express a database config (for lotus-provider) from an existing miner. + +OPTIONS: + --miner-repo value, --storagerepo value Specify miner repo path. flag(storagerepo) and env(LOTUS_STORAGE_PATH) are DEPRECATION, will REMOVE SOON (default: "~/.lotusminer") [$LOTUS_MINER_PATH, $LOTUS_STORAGE_PATH] + --to-layer value, -t value The layer name for this data push. 'base' is recommended for single-miner setup. + --overwrite, -o Use this with --to-layer to replace an existing layer (default: false) + --help, -h show help +``` + +## lotus-provider test +``` +NAME: + lotus-provider test - Utility functions for testing + +USAGE: + lotus-provider test command [command options] [arguments...] + +COMMANDS: + window-post, wd, windowpost, wdpost Compute a proof-of-spacetime for a sector (requires the sector to be pre-sealed). These will not send to the chain. + help, h Shows a list of commands or help for one command + +OPTIONS: + --help, -h show help +``` + +### lotus-provider test window-post +``` +NAME: + lotus-provider test window-post - Compute a proof-of-spacetime for a sector (requires the sector to be pre-sealed). These will not send to the chain. + +USAGE: + lotus-provider test window-post command [command options] [arguments...] + +COMMANDS: + here, cli Compute WindowPoSt for performance and configuration testing. + task, scheduled, schedule, async, asynchronous Test the windowpost scheduler by running it on the next available lotus-provider. + help, h Shows a list of commands or help for one command + +OPTIONS: + --help, -h show help +``` + +#### lotus-provider test window-post here +``` +NAME: + lotus-provider test window-post here - Compute WindowPoSt for performance and configuration testing. + +USAGE: + lotus-provider test window-post here [command options] [deadline index] + +DESCRIPTION: + Note: This command is intended to be used to verify PoSt compute performance. + It will not send any messages to the chain. Since it can compute any deadline, output may be incorrectly timed for the chain. + +OPTIONS: + --deadline value deadline to compute WindowPoSt for (default: 0) + --layers value [ --layers value ] list of layers to be interpreted (atop defaults). Default: base (default: "base") + --storage-json value path to json file containing storage config (default: "~/.lotus-provider/storage.json") + --partition value partition to compute WindowPoSt for (default: 0) + --help, -h show help +``` + +#### lotus-provider test window-post task +``` +NAME: + lotus-provider test window-post task - Test the windowpost scheduler by running it on the next available lotus-provider. + +USAGE: + lotus-provider test window-post task [command options] [arguments...] + +OPTIONS: + --deadline value deadline to compute WindowPoSt for (default: 0) + --layers value [ --layers value ] list of layers to be interpreted (atop defaults). Default: base (default: "base") + --help, -h show help +``` + +## lotus-provider web +``` +NAME: + lotus-provider web - Start lotus provider web interface + +USAGE: + lotus-provider web [command options] [arguments...] + +DESCRIPTION: + Start an instance of lotus provider web interface. + This creates the 'web' layer if it does not exist, then calls run with that layer. + +OPTIONS: + --listen value Address to listen on (default: "127.0.0.1:4701") + --layers value [ --layers value ] list of layers to be interpreted (atop defaults). Default: base. Web will be added (default: "base") + --nosync don't check full-node sync status (default: false) + --help, -h show help +``` + +## lotus-provider version +``` +NAME: + lotus-provider version - Print version + +USAGE: + lotus-provider version [command options] [arguments...] + +OPTIONS: + --help, -h show help +``` + +## lotus-provider auth +``` +NAME: + lotus-provider auth - Manage RPC permissions + +USAGE: + lotus-provider auth command [command options] [arguments...] + +COMMANDS: + create-token Create token + api-info Get token with API info required to connect to this node + help, h Shows a list of commands or help for one command + +OPTIONS: + --help, -h show help +``` + +### lotus-provider auth create-token +``` +NAME: + lotus-provider auth create-token - Create token + +USAGE: + lotus-provider auth create-token [command options] [arguments...] + +OPTIONS: + --perm value permission to assign to the token, one of: read, write, sign, admin + --help, -h show help +``` + +### lotus-provider auth api-info +``` +NAME: + lotus-provider auth api-info - Get token with API info required to connect to this node + +USAGE: + lotus-provider auth api-info [command options] [arguments...] + +OPTIONS: + --perm value permission to assign to the token, one of: read, write, sign, admin + --help, -h show help +``` + +## lotus-provider log +``` +NAME: + lotus-provider log - Manage logging + +USAGE: + lotus-provider log command [command options] [arguments...] + +COMMANDS: + list List log systems + set-level Set log level + alerts Get alert states + help, h Shows a list of commands or help for one command + +OPTIONS: + --help, -h show help +``` + +### lotus-provider log list +``` +NAME: + lotus-provider log list - List log systems + +USAGE: + lotus-provider log list [command options] [arguments...] + +OPTIONS: + --help, -h show help +``` + +### lotus-provider log set-level +``` +NAME: + lotus-provider log set-level - Set log level + +USAGE: + lotus-provider log set-level [command options] [level] + +DESCRIPTION: + Set the log level for logging systems: + + The system flag can be specified multiple times. + + eg) log set-level --system chain --system chainxchg debug + + Available Levels: + debug + info + warn + error + + Environment Variables: + GOLOG_LOG_LEVEL - Default log level for all log systems + GOLOG_LOG_FMT - Change output log format (json, nocolor) + GOLOG_FILE - Write logs to file + GOLOG_OUTPUT - Specify whether to output to file, stderr, stdout or a combination, i.e. file+stderr + + +OPTIONS: + --system value [ --system value ] limit to log system + --help, -h show help +``` + +### lotus-provider log alerts +``` +NAME: + lotus-provider log alerts - Get alert states + +USAGE: + lotus-provider log alerts [command options] [arguments...] + +OPTIONS: + --all get all (active and inactive) alerts (default: false) + --help, -h show help +``` + +## lotus-provider wait-api +``` +NAME: + lotus-provider wait-api - Wait for lotus api to come online + +USAGE: + lotus-provider wait-api [command options] [arguments...] + +CATEGORY: + DEVELOPER + +OPTIONS: + --timeout value duration to wait till fail (default: 30s) + --help, -h show help +``` + +## lotus-provider fetch-params +``` +NAME: + lotus-provider fetch-params - Fetch proving parameters + +USAGE: + lotus-provider fetch-params [command options] [sectorSize] + +CATEGORY: + DEVELOPER + +OPTIONS: + --help, -h show help +``` diff --git a/documentation/en/cli-lotus-worker.md b/documentation/en/cli-lotus-worker.md index b01b721eb..0e0fee157 100644 --- a/documentation/en/cli-lotus-worker.md +++ b/documentation/en/cli-lotus-worker.md @@ -7,7 +7,7 @@ USAGE: lotus-worker [global options] command [command options] [arguments...] VERSION: - 1.25.1-dev + 1.25.3-dev COMMANDS: run Start lotus worker @@ -34,6 +34,33 @@ NAME: USAGE: lotus-worker run [command options] [arguments...] +DESCRIPTION: + Run lotus-worker. + + --external-pc2 can be used to compute the PreCommit2 inputs externally. + The flag behaves similarly to the related lotus-worker flag, using it in + lotus-bench may be useful for testing if the external PreCommit2 command is + invoked correctly. + + The command will be called with a number of environment variables set: + * EXTSEAL_PC2_SECTOR_NUM: the sector number + * EXTSEAL_PC2_SECTOR_MINER: the miner id + * EXTSEAL_PC2_PROOF_TYPE: the proof type + * EXTSEAL_PC2_SECTOR_SIZE: the sector size in bytes + * EXTSEAL_PC2_CACHE: the path to the cache directory + * EXTSEAL_PC2_SEALED: the path to the sealed sector file (initialized with unsealed data by the caller) + * EXTSEAL_PC2_PC1OUT: output from rust-fil-proofs precommit1 phase (base64 encoded json) + + The command is expected to: + * Create cache sc-02-data-tree-r* files + * Create cache sc-02-data-tree-c* files + * Create cache p_aux / t_aux files + * Transform the sealed file in place + + Example invocation of lotus-bench as external executor: + './lotus-bench simple precommit2 --sector-size $EXTSEAL_PC2_SECTOR_SIZE $EXTSEAL_PC2_SEALED $EXTSEAL_PC2_CACHE $EXTSEAL_PC2_PC1OUT' + + OPTIONS: --listen value host address and port the worker api will listen on (default: "0.0.0.0:3456") [$LOTUS_WORKER_LISTEN] --no-local-storage don't use storageminer repo for sector storage (default: false) [$LOTUS_WORKER_NO_LOCAL_STORAGE] @@ -57,6 +84,7 @@ OPTIONS: --timeout value used when 'listen' is unspecified. must be a valid duration recognized by golang's time.ParseDuration function (default: "30m") [$LOTUS_WORKER_TIMEOUT] --http-server-timeout value (default: "30s") --data-cid Run the data-cid task. true|false (default: inherits --addpiece) + --external-pc2 value command for computing PC2 externally --help, -h show help ``` diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index a6a6ea30a..ff62980dc 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -7,7 +7,7 @@ USAGE: lotus [global options] command [command options] [arguments...] VERSION: - 1.25.1-dev + 1.25.3-dev COMMANDS: daemon Start a lotus daemon process @@ -1807,8 +1807,16 @@ OPTIONS: --help, -h show help ``` -#### lotus state sector, sector-info +### lotus state sector ``` +NAME: + lotus state sector - Get miner sector info + +USAGE: + lotus state sector [command options] [minerAddress] [sectorNumber] + +OPTIONS: + --help, -h show help ``` ### lotus state get-actor @@ -1937,12 +1945,29 @@ OPTIONS: --help, -h show help ``` -#### lotus state wait-msg, wait-message +### lotus state wait-msg ``` +NAME: + lotus state wait-msg - Wait for a message to appear on chain + +USAGE: + lotus state wait-msg [command options] [messageCid] + +OPTIONS: + --timeout value (default: "10m") + --help, -h show help ``` -#### lotus state search-msg, search-message +### lotus state search-msg ``` +NAME: + lotus state search-msg - Search to see whether a message has appeared on chain + +USAGE: + lotus state search-msg [command options] [messageCid] + +OPTIONS: + --help, -h show help ``` ### lotus state miner-info @@ -2080,8 +2105,17 @@ OPTIONS: --help, -h show help ``` -#### lotus chain get-block, getblock +### lotus chain get-block ``` +NAME: + lotus chain get-block - Get a block and print its details + +USAGE: + lotus chain get-block [command options] [blockCid] + +OPTIONS: + --raw print just the raw block header (default: false) + --help, -h show help ``` ### lotus chain read-obj @@ -2132,16 +2166,46 @@ OPTIONS: --help, -h show help ``` -##### lotus chain getmessage, get-message, get-msg +### lotus chain getmessage ``` +NAME: + lotus chain getmessage - Get and print a message by its cid + +USAGE: + lotus chain getmessage [command options] [messageCid] + +OPTIONS: + --help, -h show help ``` -#### lotus chain sethead, set-head +### lotus chain sethead ``` +NAME: + lotus chain sethead - manually set the local nodes head tipset (Caution: normally only used for recovery) + +USAGE: + lotus chain sethead [command options] [tipsetkey] + +OPTIONS: + --genesis reset head to genesis (default: false) + --epoch value reset head to given epoch (default: 0) + --help, -h show help ``` -#### lotus chain list, love +### lotus chain list ``` +NAME: + lotus chain list - View a segment of the chain + +USAGE: + lotus chain list [command options] [arguments...] + +OPTIONS: + --height value (default: current head) + --count value (default: 30) + --format value specify the format to print out tipsets (default: ": (