cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #11405 from filecoin-project/feat/sturdypost

Browse Source 
feat: sturdypost work branch
This commit is contained in:
Andrew Jackson (Ajax) 2023-12-04 18:18:04 -06:00 committed by GitHub
commit 5ea06f4e2d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
129 changed files with 9159 additions and 415 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
.circleci/config.yml
View File

@ -1,7 +1,7 @@
version: 2.1
orbs:
aws-cli: circleci/aws-cli@1.3.2
docker: circleci/docker@2.1.4
aws-cli: circleci/aws-cli@4.1.1
docker: circleci/docker@2.3.0
executors:
golang:
@ -70,8 +70,6 @@ commands:
name: Restore parameters cache
keys:
- 'v26-2k-lotus-params'
paths:
- /var/tmp/filecoin-proof-parameters/
- run: ./lotus fetch-params 2048
- save_cache:
name: Save parameters cache
@ -96,6 +94,7 @@ commands:
git fetch --all
install-ubuntu-deps:
steps:
- run: sudo apt install curl ca-certificates gnupg
- run: sudo apt-get update
- run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
check-go-version:
@ -143,9 +142,9 @@ jobs:
Run tests with gotestsum.
working_directory: ~/lotus
parameters: &test-params
executor:
type: executor
default: golang
resource_class:
type: string
default: medium+
go-test-flags:
type: string
default: "-timeout 20m"
@ -164,7 +163,14 @@ jobs:
type: string
default: unit
description: Test suite name to report to CircleCI.
executor: << parameters.executor >>
docker:
- image: cimg/go:1.20
environment:
LOTUS_HARMONYDB_HOSTS: yugabyte
- image: yugabytedb/yugabyte:2.18.0.0-b65
command: bin/yugabyted start --daemon=false
name: yugabyte
resource_class: << parameters.resource_class >>
steps:
- install-ubuntu-deps
- attach_workspace:
@ -182,6 +188,8 @@ jobs:
command: |
mkdir -p /tmp/test-reports/<< parameters.suite >>
mkdir -p /tmp/test-artifacts
dockerize -wait tcp://yugabyte:5433 -timeout 3m
env
gotestsum \
--format standard-verbose \
--junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \
@ -209,7 +217,9 @@ jobs:
Branch on github.com/filecoin-project/test-vectors to checkout and
test with. If empty (the default) the commit defined by the git
submodule is used.
executor: << parameters.executor >>
docker:
- image: cimg/go:1.20
resource_class: << parameters.resource_class >>
steps:
- install-ubuntu-deps
- attach_workspace:
@ -396,15 +406,14 @@ jobs:
Run golangci-lint.
working_directory: ~/lotus
parameters:
executor:
type: executor
default: golang
args:
type: string
default: ''
description: |
Arguments to pass to golangci-lint
executor: << parameters.executor >>
docker:
- image: cimg/go:1.20
resource_class: medium+
steps:
- install-ubuntu-deps
- attach_workspace:
@ -575,7 +584,7 @@ workflows:
- build
suite: itest-deals_concurrent
target: "./itests/deals_concurrent_test.go"
executor: golang-2xl
resource_class: 2xlarge
- test:
name: test-itest-deals_invalid_utf8_label
requires:
@ -768,6 +777,18 @@ workflows:
- build
suite: itest-get_messages_in_ts
target: "./itests/get_messages_in_ts_test.go"
- test:
name: test-itest-harmonydb
requires:
- build
suite: itest-harmonydb
target: "./itests/harmonydb_test.go"
- test:
name: test-itest-harmonytask
requires:
- build
suite: itest-harmonytask
target: "./itests/harmonytask_test.go"
- test:
name: test-itest-lite_migration
requires:
@ -976,14 +997,14 @@ workflows:
- build
suite: itest-wdpost_worker_config
target: "./itests/wdpost_worker_config_test.go"
executor: golang-2xl
resource_class: 2xlarge
- test:
name: test-itest-worker
requires:
- build
suite: itest-worker
target: "./itests/worker_test.go"
executor: golang-2xl
resource_class: 2xlarge
- test:
name: test-itest-worker_upgrade
requires:
@ -996,7 +1017,7 @@ workflows:
- build
suite: utest-unit-cli
target: "./cli/... ./cmd/... ./api/..."
executor: golang-2xl
resource_class: 2xlarge
get-params: true
- test:
name: test-unit-node
@ -1010,7 +1031,7 @@ workflows:
- build
suite: utest-unit-rest
target: "./blockstore/... ./build/... ./chain/... ./conformance/... ./gateway/... ./journal/... ./lib/... ./markets/... ./paychmgr/... ./tools/..."
executor: golang-2xl
resource_class: 2xlarge
- test:
name: test-unit-storage
requires:

16
.circleci/gen.go
View File

@ -10,11 +10,25 @@ import (
"text/template"
)
var GoVersion = "" // from init below. Ex: 1.19.7
//go:generate go run ./gen.go ..
//go:embed template.yml
var templateFile embed.FS
func init() {
b, err := os.ReadFile("../go.mod")
if err != nil {
panic("cannot find go.mod in parent folder")
}
for _, line := range strings.Split(string(b), "\n") {
if strings.HasPrefix(line, "go ") {
GoVersion = line[3:]
}
}
}
type (
dirs = []string
suite = string
@ -111,6 +125,7 @@ func main() {
Networks []string
ItestFiles []string
UnitSuites map[string]string
GoVersion string
}
in := data{
Networks: []string{"mainnet", "butterflynet", "calibnet", "debug"},
@ -125,6 +140,7 @@ func main() {
}
return ret
}(),
GoVersion: GoVersion,
}
out, err := os.Create("./config.yml")

41
.circleci/template.yml
View File

@ -1,7 +1,7 @@
version: 2.1
orbs:
aws-cli: circleci/aws-cli@1.3.2
docker: circleci/docker@2.1.4
aws-cli: circleci/aws-cli@4.1.1
docker: circleci/docker@2.3.0
executors:
golang:
@ -70,8 +70,6 @@ commands:
name: Restore parameters cache
keys:
- 'v26-2k-lotus-params'
paths:
- /var/tmp/filecoin-proof-parameters/
- run: ./lotus fetch-params 2048
- save_cache:
name: Save parameters cache
@ -96,6 +94,7 @@ commands:
git fetch --all
install-ubuntu-deps:
steps:
- run: sudo apt install curl ca-certificates gnupg
- run: sudo apt-get update
- run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
check-go-version:
@ -143,9 +142,9 @@ jobs:
Run tests with gotestsum.
working_directory: ~/lotus
parameters: &test-params
executor:
type: executor
default: golang
resource_class:
type: string
default: medium+
go-test-flags:
type: string
default: "-timeout 20m"
@ -164,7 +163,14 @@ jobs:
type: string
default: unit
description: Test suite name to report to CircleCI.
executor: << parameters.executor >>
docker:
- image: cimg/go:[[ .GoVersion]]
environment:
LOTUS_HARMONYDB_HOSTS: yugabyte
- image: yugabytedb/yugabyte:2.18.0.0-b65
command: bin/yugabyted start --daemon=false
name: yugabyte
resource_class: << parameters.resource_class >>
steps:
- install-ubuntu-deps
- attach_workspace:
@ -182,6 +188,8 @@ jobs:
command: |
mkdir -p /tmp/test-reports/<< parameters.suite >>
mkdir -p /tmp/test-artifacts
dockerize -wait tcp://yugabyte:5433 -timeout 3m
env
gotestsum \
--format standard-verbose \
--junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \
@ -209,7 +217,9 @@ jobs:
Branch on github.com/filecoin-project/test-vectors to checkout and
test with. If empty (the default) the commit defined by the git
submodule is used.
executor: << parameters.executor >>
docker:
- image: cimg/go:[[ .GoVersion]]
resource_class: << parameters.resource_class >>
steps:
- install-ubuntu-deps
- attach_workspace:
@ -396,15 +406,14 @@ jobs:
Run golangci-lint.
working_directory: ~/lotus
parameters:
executor:
type: executor
default: golang
args:
type: string
default: ''
description: |
Arguments to pass to golangci-lint
executor: << parameters.executor >>
docker:
- image: cimg/go:[[ .GoVersion]]
resource_class: medium+
steps:
- install-ubuntu-deps
- attach_workspace:
@ -543,7 +552,7 @@ workflows:
suite: itest-[[ $name ]]
target: "./itests/[[ $file ]]"
[[- if or (eq $name "worker") (eq $name "deals_concurrent") (eq $name "wdpost_worker_config")]]
executor: golang-2xl
resource_class: 2xlarge
[[- end]]
[[- if or (eq $name "wdpost") (eq $name "sector_pledge")]]
get-params: true
@ -561,11 +570,11 @@ workflows:
get-params: true
[[- end -]]
[[- if eq $suite "unit-cli"]]
executor: golang-2xl
resource_class: 2xlarge
get-params: true
[[- end -]]
[[- if eq $suite "unit-rest"]]
executor: golang-2xl
resource_class: 2xlarge
[[- end -]]
[[- end]]
- test:

1
.gitignore vendored
View File

@ -6,6 +6,7 @@
/lotus-chainwatch
/lotus-shed
/lotus-sim
/lotus-provider
/lotus-townhall
/lotus-fountain
/lotus-stats

4
Dockerfile
View File

@ -109,6 +109,7 @@ COPY --from=lotus-builder /opt/filecoin/lotus-wallet /usr/local/bin/
COPY --from=lotus-builder /opt/filecoin/lotus-gateway /usr/local/bin/
COPY --from=lotus-builder /opt/filecoin/lotus-miner /usr/local/bin/
COPY --from=lotus-builder /opt/filecoin/lotus-worker /usr/local/bin/
COPY --from=lotus-builder /opt/filecoin/lotus-provider /usr/local/bin/
COPY --from=lotus-builder /opt/filecoin/lotus-stats /usr/local/bin/
COPY --from=lotus-builder /opt/filecoin/lotus-fountain /usr/local/bin/
@ -117,11 +118,13 @@ RUN mkdir /var/lib/lotus
RUN mkdir /var/lib/lotus-miner
RUN mkdir /var/lib/lotus-worker
RUN mkdir /var/lib/lotus-wallet
RUN mkdir /var/lib/lotus-provider
RUN chown fc: /var/tmp/filecoin-proof-parameters
RUN chown fc: /var/lib/lotus
RUN chown fc: /var/lib/lotus-miner
RUN chown fc: /var/lib/lotus-worker
RUN chown fc: /var/lib/lotus-wallet
RUN chown fc: /var/lib/lotus-provider
VOLUME /var/tmp/filecoin-proof-parameters
@ -129,6 +132,7 @@ VOLUME /var/lib/lotus
VOLUME /var/lib/lotus-miner
VOLUME /var/lib/lotus-worker
VOLUME /var/lib/lotus-wallet
VOLUME /var/lib/lotus-provider
EXPOSE 1234
EXPOSE 2345

42
Makefile
View File

@ -66,7 +66,7 @@ CLEAN+=build/.update-modules
deps: $(BUILD_DEPS)
.PHONY: deps
build-devnets: build lotus-seed lotus-shed
build-devnets: build lotus-seed lotus-shed lotus-provider
.PHONY: build-devnets
debug: GOFLAGS+=-tags=debug
@ -97,6 +97,13 @@ lotus-miner: $(BUILD_DEPS)
.PHONY: lotus-miner
BINS+=lotus-miner
lotus-provider: $(BUILD_DEPS)
rm -f lotus-provider
$(GOCC) build $(GOFLAGS) -o lotus-provider ./cmd/lotus-provider
lp2k: GOFLAGS+=-tags=2k
lp2k: lotus-provider
lotus-worker: $(BUILD_DEPS)
rm -f lotus-worker
$(GOCC) build $(GOFLAGS) -o lotus-worker ./cmd/lotus-worker
@ -121,7 +128,7 @@ an existing lotus binary in your PATH. This may cause problems if you don't run
.PHONY: build
install: install-daemon install-miner install-worker
install: install-daemon install-miner install-worker install-provider
install-daemon:
install -C ./lotus /usr/local/bin/lotus
@ -129,6 +136,9 @@ install-daemon:
install-miner:
install -C ./lotus-miner /usr/local/bin/lotus-miner
install-provider:
install -C ./lotus-provider /usr/local/bin/lotus-provider
install-worker:
install -C ./lotus-worker /usr/local/bin/lotus-worker
@ -144,6 +154,9 @@ uninstall-daemon:
uninstall-miner:
rm -f /usr/local/bin/lotus-miner
uninstall-provider:
rm -f /usr/local/bin/lotus-provider
uninstall-worker:
rm -f /usr/local/bin/lotus-worker
@ -241,6 +254,14 @@ install-miner-service: install-miner install-daemon-service
@echo
@echo "lotus-miner service installed. Don't forget to run 'sudo systemctl start lotus-miner' to start it and 'sudo systemctl enable lotus-miner' for it to be enabled on startup."
install-provider-service: install-provider install-daemon-service
mkdir -p /etc/systemd/system
mkdir -p /var/log/lotus
install -C -m 0644 ./scripts/lotus-provider.service /etc/systemd/system/lotus-provider.service
systemctl daemon-reload
@echo
@echo "lotus-provider service installed. Don't forget to run 'sudo systemctl start lotus-provider' to start it and 'sudo systemctl enable lotus-provider' for it to be enabled on startup."
install-main-services: install-miner-service
install-all-services: install-main-services
@ -259,6 +280,12 @@ clean-miner-service:
rm -f /etc/systemd/system/lotus-miner.service
systemctl daemon-reload
clean-provider-service:
-systemctl stop lotus-provider
-systemctl disable lotus-provider
rm -f /etc/systemd/system/lotus-provider.service
systemctl daemon-reload
clean-main-services: clean-daemon-service
clean-all-services: clean-main-services
@ -294,7 +321,8 @@ actors-code-gen:
$(GOCC) run ./chain/actors/agen
$(GOCC) fmt ./...
actors-gen: actors-code-gen fiximports
actors-gen: actors-code-gen
./scripts/fiximports
.PHONY: actors-gen
bundle-gen:
@ -354,21 +382,23 @@ docsgen-openrpc-gateway: docsgen-openrpc-bin
fiximports:
./scripts/fiximports
gen: actors-code-gen type-gen cfgdoc-gen docsgen api-gen circleci fiximports
gen: actors-code-gen type-gen cfgdoc-gen docsgen api-gen circleci
./scripts/fiximports
@echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO RUN 'make docsgen-cli'"
.PHONY: gen
jen: gen
snap: lotus lotus-miner lotus-worker
snap: lotus lotus-miner lotus-worker lotus-provider
snapcraft
# snapcraft upload ./lotus_*.snap
# separate from gen because it needs binaries
docsgen-cli: lotus lotus-miner lotus-worker
docsgen-cli: lotus lotus-miner lotus-worker lotus-provider
python3 ./scripts/generate-lotus-cli.py
./lotus config default > documentation/en/default-lotus-config.toml
./lotus-miner config default > documentation/en/default-lotus-miner-config.toml
./lotus-provider config default > documentation/en/default-lotus-provider-config.toml
.PHONY: docsgen-cli
print-%:

10
api/api_lp.go Normal file
View File

@ -0,0 +1,10 @@
package api
import "context"
type LotusProvider interface {
Version(context.Context) (Version, error) //perm:admin
// Trigger shutdown
Shutdown(context.Context) error //perm:admin
}

10
api/client/client.go
View File

@ -15,6 +15,16 @@ import (
"github.com/filecoin-project/lotus/lib/rpcenc"
)
// NewProviderRpc creates a new http jsonrpc client.
func NewProviderRpc(ctx context.Context, addr string, requestHeader http.Header) (api.LotusProvider, jsonrpc.ClientCloser, error) {
var res v1api.LotusProviderStruct
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
api.GetInternalStructs(&res), requestHeader, jsonrpc.WithErrors(api.RPCErrors))
return &res, closer, err
}
// NewCommonRPCV0 creates a new http jsonrpc client.
func NewCommonRPCV0(ctx context.Context, addr string, requestHeader http.Header) (api.CommonNet, jsonrpc.ClientCloser, error) {
var res v0api.CommonNetStruct

6
api/permissioned.go
View File

@ -41,6 +41,12 @@ func PermissionedWorkerAPI(a Worker) Worker {
return &out
}
func PermissionedAPI[T, P any](a T) *P {
var out P
permissionedProxies(a, &out)
return &out
}
func PermissionedWalletAPI(a Wallet) Wallet {
var out WalletStruct
permissionedProxies(a, &out)

36
api/proxy_gen.go
View File

@ -831,6 +831,19 @@ type GatewayMethods struct {
type GatewayStub struct {
}
type LotusProviderStruct struct {
Internal LotusProviderMethods
}
type LotusProviderMethods struct {
Shutdown func(p0 context.Context) error `perm:"admin"`
Version func(p0 context.Context) (Version, error) `perm:"admin"`
}
type LotusProviderStub struct {
}
type NetStruct struct {
Internal NetMethods
}
@ -5214,6 +5227,28 @@ func (s *GatewayStub) Web3ClientVersion(p0 context.Context) (string, error) {
return "", ErrNotSupported
}
func (s *LotusProviderStruct) Shutdown(p0 context.Context) error {
if s.Internal.Shutdown == nil {
return ErrNotSupported
}
return s.Internal.Shutdown(p0)
}
func (s *LotusProviderStub) Shutdown(p0 context.Context) error {
return ErrNotSupported
}
func (s *LotusProviderStruct) Version(p0 context.Context) (Version, error) {
if s.Internal.Version == nil {
return *new(Version), ErrNotSupported
}
return s.Internal.Version(p0)
}
func (s *LotusProviderStub) Version(p0 context.Context) (Version, error) {
return *new(Version), ErrNotSupported
}
func (s *NetStruct) ID(p0 context.Context) (peer.ID, error) {
if s.Internal.ID == nil {
return *new(peer.ID), ErrNotSupported
@ -7442,6 +7477,7 @@ var _ CommonNet = new(CommonNetStruct)
var _ EthSubscriber = new(EthSubscriberStruct)
var _ FullNode = new(FullNodeStruct)
var _ Gateway = new(GatewayStruct)
var _ LotusProvider = new(LotusProviderStruct)
var _ Net = new(NetStruct)
var _ Signable = new(SignableStruct)
var _ StorageMiner = new(StorageMinerStruct)

2
api/v1api/latest.go
View File

@ -12,3 +12,5 @@ type RawFullNodeAPI FullNode
func PermissionedFullAPI(a FullNode) FullNode {
return api.PermissionedFullAPI(a)
}
type LotusProviderStruct = api.LotusProviderStruct

2
api/version.go
View File

@ -59,6 +59,8 @@ var (
MinerAPIVersion0 = newVer(1, 5, 0)
WorkerAPIVersion0 = newVer(1, 7, 0)
ProviderAPIVersion0 = newVer(1, 0, 0)
)
//nolint:varcheck,deadcode

2
blockstore/splitstore/splitstore_test.go
View File

@ -3,9 +3,9 @@ package splitstore
import (
"context"
"crypto/rand"
"errors"
"fmt"
"math/rand"
"sync"
"sync/atomic"
"testing"

BIN
build/openrpc/full.json.gz
View File

Binary file not shown.

BIN
build/openrpc/gateway.json.gz
View File

Binary file not shown.

BIN
build/openrpc/miner.json.gz
View File

Binary file not shown.

BIN
build/openrpc/worker.json.gz
View File

Binary file not shown.

18
chain/actors/policy/policy.go
View File

@ -867,6 +867,24 @@ func AggregatePreCommitNetworkFee(nwVer network.Version, aggregateSize int, base
}
}
var PoStToSealMap map[abi.RegisteredPoStProof]abi.RegisteredSealProof
func init() {
PoStToSealMap = make(map[abi.RegisteredPoStProof]abi.RegisteredSealProof)
for sealProof, info := range abi.SealProofInfos {
PoStToSealMap[info.WinningPoStProof] = sealProof
PoStToSealMap[info.WindowPoStProof] = sealProof
}
}
func GetSealProofFromPoStProof(postProof abi.RegisteredPoStProof) (abi.RegisteredSealProof, error) {
sealProof, exists := PoStToSealMap[postProof]
if !exists {
return 0, xerrors.New("no corresponding RegisteredSealProof for the given RegisteredPoStProof")
}
return sealProof, nil
}
func min(a, b int) int {
if a < b {
return a

17
chain/actors/policy/policy.go.template
View File

@ -343,6 +343,23 @@ func AggregatePreCommitNetworkFee(nwVer network.Version, aggregateSize int, base
}
}
var PoStToSealMap map[abi.RegisteredPoStProof]abi.RegisteredSealProof
func init() {
PoStToSealMap = make(map[abi.RegisteredPoStProof]abi.RegisteredSealProof)
for sealProof, info := range abi.SealProofInfos {
PoStToSealMap[info.WinningPoStProof] = sealProof
PoStToSealMap[info.WindowPoStProof] = sealProof
}
}
func GetSealProofFromPoStProof(postProof abi.RegisteredPoStProof) (abi.RegisteredSealProof, error) {
sealProof, exists := PoStToSealMap[postProof]
if !exists {
return 0, xerrors.New("no corresponding RegisteredSealProof for the given RegisteredPoStProof")
}
return sealProof, nil
}
func min(a, b int) int {
if a < b {
return a

3
chain/consensus/common.go
View File

@ -362,7 +362,8 @@ func CreateBlockHeader(ctx context.Context, sm *stmgr.StateManager, pts *types.T
var blsMsgCids, secpkMsgCids []cid.Cid
var blsSigs []crypto.Signature
nv := sm.GetNetworkVersion(ctx, bt.Epoch)
for _, msg := range bt.Messages {
for _, msgTmp := range bt.Messages {
msg := msgTmp
if msg.Signature.Type == crypto.SigTypeBLS {
blsSigs = append(blsSigs, msg.Signature)
blsMessages = append(blsMessages, &msg.Message)

3
chain/gen/genesis/miners.go
View File

@ -251,7 +251,8 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sys vm.Syscal
}
params := &markettypes.PublishStorageDealsParams{}
for _, preseal := range m.Sectors {
for _, presealTmp := range m.Sectors {
preseal := presealTmp
preseal.Deal.VerifiedDeal = true
preseal.Deal.EndEpoch = minerInfos[i].presealExp
p := markettypes.ClientDealProposal{

2
chain/messagepool/block_proba_test.go
View File

@ -5,7 +5,6 @@ import (
"math"
"math/rand"
"testing"
"time"
)
func TestBlockProbability(t *testing.T) {
@ -23,7 +22,6 @@ func TestBlockProbability(t *testing.T) {
func TestWinnerProba(t *testing.T) {
//stm: @OTHER_IMPLEMENTATION_BLOCK_PROB_002
rand.Seed(time.Now().UnixNano())
const N = 1000000
winnerProba := noWinnersProb()
sum := 0

3
chain/state/statetree.go
View File

@ -438,7 +438,8 @@ func (st *StateTree) Flush(ctx context.Context) (cid.Cid, error) {
return cid.Undef, xerrors.Errorf("tried to flush state tree with snapshots on the stack")
}
for addr, sto := range st.snaps.layers[0].actors {
for addr, stoTmp := range st.snaps.layers[0].actors {
sto := stoTmp
if sto.Delete {
if err := st.root.Delete(abi.AddrKey(addr)); err != nil {
return cid.Undef, err

3
chain/types/fil.go
View File

@ -12,6 +12,9 @@ import (
type FIL BigInt
func (f FIL) String() string {
if f.Int == nil {
return "0 FIL"
}
return f.Unitless() + " FIL"
}

6
chain/vectors/gen/main.go
View File

@ -2,6 +2,7 @@ package main
import (
"context"
crand "crypto/rand"
"encoding/json"
"fmt"
"math/rand"
@ -145,7 +146,10 @@ func MakeUnsignedMessageVectors() []vectors.UnsignedMessageVector {
}
params := make([]byte, 32)
rand.Read(params)
_, err = crand.Read(params)
if err != nil {
panic(err)
}
msg := &types.Message{
To: to,

4
cli/helper.go
View File

@ -1,6 +1,7 @@
package cli
import (
"errors"
"fmt"
"io"
"os"
@ -8,7 +9,6 @@ import (
"syscall"
ufcli "github.com/urfave/cli/v2"
"golang.org/x/xerrors"
)
type PrintHelpErr struct {
@ -52,7 +52,7 @@ func RunApp(app *ufcli.App) {
fmt.Fprintf(os.Stderr, "ERROR: %s\n\n", err) // nolint:errcheck
}
var phe *PrintHelpErr
if xerrors.As(err, &phe) {
if errors.As(err, &phe) {
_ = ufcli.ShowCommandHelp(phe.Ctx, phe.Ctx.Command.Name)
}
os.Exit(1)

3
cli/net.go
View File

@ -847,7 +847,8 @@ var NetStatCmd = &cli.Command{
})
for _, stat := range stats {
printScope(&stat.stat, name+stat.name)
tmp := stat.stat
printScope(&tmp, name+stat.name)
}
}

86
cli/util/api.go
View File

@ -119,7 +119,7 @@ func GetAPIInfoMulti(ctx *cli.Context, t repo.RepoType) ([]APIInfo, error) {
}
}
return []APIInfo{}, fmt.Errorf("could not determine API endpoint for node type: %v", t.Type())
return []APIInfo{}, fmt.Errorf("could not determine API endpoint for node type: %v. Try setting environment variable: %s", t.Type(), primaryEnv)
}
func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) {
@ -164,6 +164,28 @@ func GetRawAPIMulti(ctx *cli.Context, t repo.RepoType, version string) ([]HttpHe
return httpHeads, nil
}
func GetRawAPIMultiV2(ctx *cli.Context, ainfoCfg []string, version string) ([]HttpHead, error) {
var httpHeads []HttpHead
if len(ainfoCfg) == 0 {
return httpHeads, xerrors.Errorf("could not get API info: none configured. \nConsider getting base.toml with './lotus-provider config get base >/tmp/base.toml' \nthen adding \n[APIs] \n ChainApiInfo = [\" result_from lotus auth api-info --perm=admin \"]\n and updating it with './lotus-provider config set /tmp/base.toml'")
}
for _, i := range ainfoCfg {
ainfo := ParseApiInfo(i)
addr, err := ainfo.DialArgs(version)
if err != nil {
return httpHeads, xerrors.Errorf("could not get DialArgs: %w", err)
}
httpHeads = append(httpHeads, HttpHead{addr: addr, header: ainfo.AuthHeader()})
}
if IsVeryVerbose {
_, _ = fmt.Fprintf(ctx.App.Writer, "using raw API %s endpoint: %s\n", version, httpHeads[0].addr)
}
return httpHeads, nil
}
func GetRawAPI(ctx *cli.Context, t repo.RepoType, version string) (string, http.Header, error) {
heads, err := GetRawAPIMulti(ctx, t, version)
if err != nil {
@ -393,6 +415,68 @@ func GetFullNodeAPIV1(ctx *cli.Context, opts ...GetFullNodeOption) (v1api.FullNo
return &v1API, finalCloser, nil
}
func GetFullNodeAPIV1LotusProvider(ctx *cli.Context, ainfoCfg []string, opts ...GetFullNodeOption) (v1api.FullNode, jsonrpc.ClientCloser, error) {
if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
return tn.(v1api.FullNode), func() {}, nil
}
var options GetFullNodeOptions
for _, opt := range opts {
opt(&options)
}
var rpcOpts []jsonrpc.Option
if options.ethSubHandler != nil {
rpcOpts = append(rpcOpts, jsonrpc.WithClientHandler("Filecoin", options.ethSubHandler), jsonrpc.WithClientHandlerAlias("eth_subscription", "Filecoin.EthSubscription"))
}
heads, err := GetRawAPIMultiV2(ctx, ainfoCfg, "v1")
if err != nil {
return nil, nil, err
}
if IsVeryVerbose {
_, _ = fmt.Fprintln(ctx.App.Writer, "using full node API v1 endpoint:", heads[0].addr)
}
var fullNodes []api.FullNode
var closers []jsonrpc.ClientCloser
for _, head := range heads {
v1api, closer, err := client.NewFullNodeRPCV1(ctx.Context, head.addr, head.header, rpcOpts...)
if err != nil {
log.Warnf("Not able to establish connection to node with addr: %s", head.addr)
continue
}
fullNodes = append(fullNodes, v1api)
closers = append(closers, closer)
}
// When running in cluster mode and trying to establish connections to multiple nodes, fail
// if less than 2 lotus nodes are actually running
if len(heads) > 1 && len(fullNodes) < 2 {
return nil, nil, xerrors.Errorf("Not able to establish connection to more than a single node")
}
finalCloser := func() {
for _, c := range closers {
c()
}
}
var v1API api.FullNodeStruct
FullNodeProxy(fullNodes, &v1API)
v, err := v1API.Version(ctx.Context)
if err != nil {
return nil, nil, err
}
if !v.APIVersion.EqMajorMinor(api.FullAPIVersion1) {
return nil, nil, xerrors.Errorf("Remote API version didn't match (expected %s, remote %s)", api.FullAPIVersion1, v.APIVersion)
}
return &v1API, finalCloser, nil
}
type GetStorageMinerOptions struct {
PreferHttp bool
}

11
cmd/lotus-bench/main.go
View File

@ -3,10 +3,10 @@ package main
import (
"bytes"
"context"
"crypto/rand"
"encoding/json"
"fmt"
"math/big"
"math/rand"
"os"
"path/filepath"
"sync"
@ -547,7 +547,10 @@ var sealBenchCmd = &cli.Command{
}
var challenge [32]byte
rand.Read(challenge[:])
_, err = rand.Read(challenge[:])
if err != nil {
return err
}
beforePost := time.Now()
@ -777,9 +780,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par
start := time.Now()
log.Infof("[%d] Writing piece into sector...", i)
r := rand.New(rand.NewSource(100 + int64(i)))
pi, err := sb.AddPiece(context.TODO(), sid, nil, abi.PaddedPieceSize(sectorSize).Unpadded(), r)
pi, err := sb.AddPiece(context.TODO(), sid, nil, abi.PaddedPieceSize(sectorSize).Unpadded(), rand.Reader)
if err != nil {
return nil, nil, err
}

2
cmd/lotus-miner/init.go
View File

@ -463,7 +463,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode
wsts := statestore.New(namespace.Wrap(mds, modules.WorkerCallsPrefix))
smsts := statestore.New(namespace.Wrap(mds, modules.ManagerWorkPrefix))
si := paths.NewIndex(nil)
si := paths.NewMemIndex(nil)
lstor, err := paths.NewLocal(ctx, lr, si, nil)
if err != nil {

3
cmd/lotus-miner/proving.go
View File

@ -559,7 +559,8 @@ var provingCheckProvableCmd = &cli.Command{
for parIdx, par := range partitions {
sectors := make(map[abi.SectorNumber]struct{})
sectorInfos, err := api.StateMinerSectors(ctx, addr, &par.LiveSectors, types.EmptyTSK)
tmp := par.LiveSectors
sectorInfos, err := api.StateMinerSectors(ctx, addr, &tmp, types.EmptyTSK)
if err != nil {
return err
}

259
cmd/lotus-provider/config.go Normal file
View File

@ -0,0 +1,259 @@
package main
import (
"context"
"database/sql"
"errors"
"fmt"
"io"
"os"
"path"
"strings"
"github.com/BurntSushi/toml"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/node/config"
)
var configCmd = &cli.Command{
Name: "config",
Usage: "Manage node config by layers. The layer 'base' will always be applied. ",
Subcommands: []*cli.Command{
configDefaultCmd,
configSetCmd,
configGetCmd,
configListCmd,
configViewCmd,
configRmCmd,
configMigrateCmd,
},
}
var configDefaultCmd = &cli.Command{
Name: "default",
Aliases: []string{"defaults"},
Usage: "Print default node config",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "no-comment",
Usage: "don't comment default values",
},
},
Action: func(cctx *cli.Context) error {
comment := !cctx.Bool("no-comment")
cfg, err := getDefaultConfig(comment)
if err != nil {
return err
}
fmt.Print(cfg)
return nil
},
}
func getDefaultConfig(comment bool) (string, error) {
c := config.DefaultLotusProvider()
cb, err := config.ConfigUpdate(c, nil, config.Commented(comment), config.DefaultKeepUncommented(), config.NoEnv())
if err != nil {
return "", err
}
return string(cb), nil
}
var configSetCmd = &cli.Command{
Name: "set",
Aliases: []string{"add", "update", "create"},
Usage: "Set a config layer or the base by providing a filename or stdin.",
ArgsUsage: "a layer's file name",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "title",
Usage: "title of the config layer (req'd for stdin)",
},
},
Action: func(cctx *cli.Context) error {
args := cctx.Args()
db, err := makeDB(cctx)
if err != nil {
return err
}
name := cctx.String("title")
var stream io.Reader = os.Stdin
if args.Len() != 1 {
if cctx.String("title") == "" {
return errors.New("must have a title for stdin, or a file name")
}
} else {
stream, err = os.Open(args.First())
if err != nil {
return fmt.Errorf("cannot open file %s: %w", args.First(), err)
}
if name == "" {
name = strings.Split(path.Base(args.First()), ".")[0]
}
}
bytes, err := io.ReadAll(stream)
if err != nil {
return fmt.Errorf("cannot read stream/file %w", err)
}
lp := config.DefaultLotusProvider() // ensure it's toml
_, err = toml.Decode(string(bytes), lp)
if err != nil {
return fmt.Errorf("cannot decode file: %w", err)
}
_ = lp
_, err = db.Exec(context.Background(),
`INSERT INTO harmony_config (title, config) VALUES ($1, $2)
ON CONFLICT (title) DO UPDATE SET config = excluded.config`, name, string(bytes))
if err != nil {
return fmt.Errorf("unable to save config layer: %w", err)
}
fmt.Println("Layer " + name + " created/updated")
return nil
},
}
var configGetCmd = &cli.Command{
Name: "get",
Aliases: []string{"cat", "show"},
Usage: "Get a config layer by name. You may want to pipe the output to a file, or use 'less'",
ArgsUsage: "layer name",
Action: func(cctx *cli.Context) error {
args := cctx.Args()
if args.Len() != 1 {
return fmt.Errorf("want 1 layer arg, got %d", args.Len())
}
db, err := makeDB(cctx)
if err != nil {
return err
}
var cfg string
err = db.QueryRow(context.Background(), `SELECT config FROM harmony_config WHERE title=$1`, args.First()).Scan(&cfg)
if err != nil {
return err
}
fmt.Println(cfg)
return nil
},
}
var configListCmd = &cli.Command{
Name: "list",
Aliases: []string{"ls"},
Usage: "List config layers you can get.",
Flags: []cli.Flag{},
Action: func(cctx *cli.Context) error {
db, err := makeDB(cctx)
if err != nil {
return err
}
var res []string
err = db.Select(context.Background(), &res, `SELECT title FROM harmony_config ORDER BY title`)
if err != nil {
return fmt.Errorf("unable to read from db: %w", err)
}
for _, r := range res {
fmt.Println(r)
}
return nil
},
}
var configRmCmd = &cli.Command{
Name: "remove",
Aliases: []string{"rm", "del", "delete"},
Usage: "Remove a named config layer.",
Flags: []cli.Flag{},
Action: func(cctx *cli.Context) error {
args := cctx.Args()
if args.Len() != 1 {
return errors.New("must have exactly 1 arg for the layer name")
}
db, err := makeDB(cctx)
if err != nil {
return err
}
ct, err := db.Exec(context.Background(), `DELETE FROM harmony_config WHERE title=$1`, args.First())
if err != nil {
return fmt.Errorf("unable to read from db: %w", err)
}
if ct == 0 {
return fmt.Errorf("no layer named %s", args.First())
}
return nil
},
}
var configViewCmd = &cli.Command{
Name: "interpret",
Aliases: []string{"view", "stacked", "stack"},
Usage: "Interpret stacked config layers by this version of lotus-provider, with system-generated comments.",
ArgsUsage: "a list of layers to be interpreted as the final config",
Flags: []cli.Flag{
&cli.StringSliceFlag{
Name: "layers",
Usage: "comma or space separated list of layers to be interpreted",
Value: cli.NewStringSlice("base"),
Required: true,
},
},
Action: func(cctx *cli.Context) error {
db, err := makeDB(cctx)
if err != nil {
return err
}
lp, err := getConfig(cctx, db)
if err != nil {
return err
}
cb, err := config.ConfigUpdate(lp, config.DefaultLotusProvider(), config.Commented(true), config.DefaultKeepUncommented(), config.NoEnv())
if err != nil {
return xerrors.Errorf("cannot interpret config: %w", err)
}
fmt.Println(string(cb))
return nil
},
}
func getConfig(cctx *cli.Context, db *harmonydb.DB) (*config.LotusProviderConfig, error) {
lp := config.DefaultLotusProvider()
have := []string{}
layers := cctx.StringSlice("layers")
for _, layer := range layers {
text := ""
err := db.QueryRow(cctx.Context, `SELECT config FROM harmony_config WHERE title=$1`, layer).Scan(&text)
if err != nil {
if strings.Contains(err.Error(), sql.ErrNoRows.Error()) {
return nil, fmt.Errorf("missing layer '%s' ", layer)
}
if layer == "base" {
return nil, errors.New(`lotus-provider defaults to a layer named 'base'.
Either use 'migrate' command or edit a base.toml and upload it with: lotus-provider config set base.toml`)
}
return nil, fmt.Errorf("could not read layer '%s': %w", layer, err)
}
meta, err := toml.Decode(text, &lp)
if err != nil {
return nil, fmt.Errorf("could not read layer, bad toml %s: %w", layer, err)
}
for _, k := range meta.Keys() {
have = append(have, strings.Join(k, " "))
}
}
_ = have // FUTURE: verify that required fields are here.
// If config includes 3rd-party config, consider JSONSchema as a way that
// 3rd-parties can dynamically include config requirements and we can
// validate the config. Because of layering, we must validate @ startup.
return lp, nil
}

160
cmd/lotus-provider/main.go Normal file
View File

@ -0,0 +1,160 @@
package main
import (
"context"
"fmt"
"os"
"os/signal"
"runtime/debug"
"syscall"
"github.com/fatih/color"
logging "github.com/ipfs/go-log/v2"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/lotus/build"
lcli "github.com/filecoin-project/lotus/cli"
cliutil "github.com/filecoin-project/lotus/cli/util"
"github.com/filecoin-project/lotus/lib/lotuslog"
"github.com/filecoin-project/lotus/lib/tracing"
"github.com/filecoin-project/lotus/node/repo"
)
var log = logging.Logger("main")
func SetupCloseHandler() {
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
go func() {
<-c
fmt.Println("\r- Ctrl+C pressed in Terminal")
debug.PrintStack()
os.Exit(1)
}()
}
func main() {
SetupCloseHandler()
lotuslog.SetupLogLevels()
local := []*cli.Command{
//initCmd,
runCmd,
stopCmd,
configCmd,
testCmd,
//backupCmd,
//lcli.WithCategory("chain", actorCmd),
//lcli.WithCategory("storage", sectorsCmd),
//lcli.WithCategory("storage", provingCmd),
//lcli.WithCategory("storage", storageCmd),
//lcli.WithCategory("storage", sealingCmd),
}
jaeger := tracing.SetupJaegerTracing("lotus")
defer func() {
if jaeger != nil {
_ = jaeger.ForceFlush(context.Background())
}
}()
for _, cmd := range local {
cmd := cmd
originBefore := cmd.Before
cmd.Before = func(cctx *cli.Context) error {
if jaeger != nil {
_ = jaeger.Shutdown(cctx.Context)
}
jaeger = tracing.SetupJaegerTracing("lotus/" + cmd.Name)
if cctx.IsSet("color") {
color.NoColor = !cctx.Bool("color")
}
if originBefore != nil {
return originBefore(cctx)
}
return nil
}
}
app := &cli.App{
Name: "lotus-provider",
Usage: "Filecoin decentralized storage network provider",
Version: build.UserVersion(),
EnableBashCompletion: true,
Flags: []cli.Flag{
&cli.BoolFlag{
// examined in the Before above
Name: "color",
Usage: "use color in display output",
DefaultText: "depends on output being a TTY",
},
&cli.StringFlag{
Name: "panic-reports",
EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"},
Hidden: true,
Value: "~/.lotusprovider", // should follow --repo default
},
&cli.StringFlag{
Name: "db-host",
EnvVars: []string{"LOTUS_DB_HOST"},
Usage: "Command separated list of hostnames for yugabyte cluster",
Value: "yugabyte",
},
&cli.StringFlag{
Name: "db-name",
EnvVars: []string{"LOTUS_DB_NAME", "LOTUS_HARMONYDB_HOSTS"},
Value: "yugabyte",
},
&cli.StringFlag{
Name: "db-user",
EnvVars: []string{"LOTUS_DB_USER", "LOTUS_HARMONYDB_USERNAME"},
Value: "yugabyte",
},
&cli.StringFlag{
Name: "db-password",
EnvVars: []string{"LOTUS_DB_PASSWORD", "LOTUS_HARMONYDB_PASSWORD"},
Value: "yugabyte",
},
&cli.StringFlag{
Name: "db-port",
EnvVars: []string{"LOTUS_DB_PORT", "LOTUS_HARMONYDB_PORT"},
Hidden: true,
Value: "5433",
},
&cli.StringFlag{
Name: "layers",
EnvVars: []string{"LOTUS_LAYERS", "LOTUS_CONFIG_LAYERS"},
Value: "base",
},
&cli.StringFlag{
Name: FlagRepoPath,
EnvVars: []string{"LOTUS_REPO_PATH"},
Value: "~/.lotusprovider",
},
cliutil.FlagVeryVerbose,
},
Commands: append(local, lcli.CommonCommands...),
Before: func(c *cli.Context) error {
return nil
},
After: func(c *cli.Context) error {
if r := recover(); r != nil {
// Generate report in LOTUS_PATH and re-raise panic
build.GeneratePanicReport(c.String("panic-reports"), c.String(FlagRepoPath), c.App.Name)
panic(r)
}
return nil
},
}
app.Setup()
app.Metadata["repoType"] = repo.Provider
lcli.RunApp(app)
}
const (
FlagRepoPath = "repo-path"
)

242
cmd/lotus-provider/migrate.go Normal file
View File

@ -0,0 +1,242 @@
package main
import (
"bytes"
"context"
"encoding/base64"
"errors"
"fmt"
"os"
"path"
"strings"
"github.com/BurntSushi/toml"
"github.com/fatih/color"
"github.com/ipfs/go-datastore"
"github.com/samber/lo"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
cliutil "github.com/filecoin-project/lotus/cli/util"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/modules"
"github.com/filecoin-project/lotus/node/repo"
)
var configMigrateCmd = &cli.Command{
Name: "from-miner",
Usage: "Express a database config (for lotus-provider) from an existing miner.",
Description: "Express a database config (for lotus-provider) from an existing miner.",
Flags: []cli.Flag{
&cli.StringFlag{
Name: FlagMinerRepo,
Aliases: []string{FlagMinerRepoDeprecation},
EnvVars: []string{"LOTUS_MINER_PATH", "LOTUS_STORAGE_PATH"},
Value: "~/.lotusminer",
Usage: fmt.Sprintf("Specify miner repo path. flag(%s) and env(LOTUS_STORAGE_PATH) are DEPRECATION, will REMOVE SOON", FlagMinerRepoDeprecation),
},
&cli.StringFlag{
Name: "repo",
EnvVars: []string{"LOTUS_PATH"},
Hidden: true,
Value: "~/.lotus",
},
&cli.StringFlag{
Name: "to-layer",
Aliases: []string{"t"},
Usage: "The layer name for this data push. 'base' is recommended for single-miner setup.",
},
&cli.BoolFlag{
Name: "overwrite",
Aliases: []string{"o"},
Usage: "Use this with --to-layer to replace an existing layer",
},
},
Action: fromMiner,
}
const (
FlagMinerRepo = "miner-repo"
)
const FlagMinerRepoDeprecation = "storagerepo"
func fromMiner(cctx *cli.Context) (err error) {
ctx := context.Background()
cliCommandColor := color.New(color.FgHiBlue).SprintFunc()
configColor := color.New(color.FgHiGreen).SprintFunc()
r, err := repo.NewFS(cctx.String(FlagMinerRepo))
if err != nil {
return err
}
ok, err := r.Exists()
if err != nil {
return err
}
if !ok {
return fmt.Errorf("repo not initialized")
}
lr, err := r.LockRO(repo.StorageMiner)
if err != nil {
return fmt.Errorf("locking repo: %w", err)
}
defer func() { _ = lr.Close() }()
cfgNode, err := lr.Config()
if err != nil {
return fmt.Errorf("getting node config: %w", err)
}
smCfg := cfgNode.(*config.StorageMiner)
db, err := harmonydb.NewFromConfig(smCfg.HarmonyDB)
if err != nil {
return fmt.Errorf("could not reach the database. Ensure the Miner config toml's HarmonyDB entry"+
" is setup to reach Yugabyte correctly: %w", err)
}
var titles []string
err = db.Select(ctx, &titles, `SELECT title FROM harmony_config WHERE LENGTH(config) > 0`)
if err != nil {
return fmt.Errorf("miner cannot reach the db. Ensure the config toml's HarmonyDB entry"+
" is setup to reach Yugabyte correctly: %s", err.Error())
}
name := cctx.String("to-layer")
if name == "" {
name = fmt.Sprintf("mig%d", len(titles))
} else {
if lo.Contains(titles, name) && !cctx.Bool("overwrite") {
return errors.New("the overwrite flag is needed to replace existing layer: " + name)
}
}
msg := "Layer " + configColor(name) + ` created. `
// Copy over identical settings:
buf, err := os.ReadFile(path.Join(lr.Path(), "config.toml"))
if err != nil {
return fmt.Errorf("could not read config.toml: %w", err)
}
var lpCfg config.LotusProviderConfig
_, err = toml.Decode(string(buf), &lpCfg)
if err != nil {
return fmt.Errorf("could not decode toml: %w", err)
}
// Populate Miner Address
mmeta, err := lr.Datastore(ctx, "/metadata")
if err != nil {
return xerrors.Errorf("opening miner metadata datastore: %w", err)
}
defer func() {
_ = mmeta.Close()
}()
maddrBytes, err := mmeta.Get(ctx, datastore.NewKey("miner-address"))
if err != nil {
return xerrors.Errorf("getting miner address datastore entry: %w", err)
}
addr, err := address.NewFromBytes(maddrBytes)
if err != nil {
return xerrors.Errorf("parsing miner actor address: %w", err)
}
lpCfg.Addresses.MinerAddresses = []string{addr.String()}
ks, err := lr.KeyStore()
if err != nil {
return xerrors.Errorf("keystore err: %w", err)
}
js, err := ks.Get(modules.JWTSecretName)
if err != nil {
return xerrors.Errorf("error getting JWTSecretName: %w", err)
}
lpCfg.Apis.StorageRPCSecret = base64.StdEncoding.EncodeToString(js.PrivateKey)
// Populate API Key
_, header, err := cliutil.GetRawAPI(cctx, repo.FullNode, "v0")
if err != nil {
return fmt.Errorf("cannot read API: %w", err)
}
ainfo, err := cliutil.GetAPIInfo(&cli.Context{}, repo.FullNode)
if err != nil {
return xerrors.Errorf(`could not get API info for FullNode: %w
Set the environment variable to the value of "lotus auth api-info --perm=admin"`, err)
}
lpCfg.Apis.ChainApiInfo = []string{header.Get("Authorization")[7:] + ":" + ainfo.Addr}
// Enable WindowPoSt
lpCfg.Subsystems.EnableWindowPost = true
msg += "\nBefore running lotus-provider, ensure any miner/worker answering of WindowPost is disabled by " +
"(on Miner) " + configColor("DisableBuiltinWindowPoSt=true") + " and (on Workers) not enabling windowpost on CLI or via " +
"environment variable " + configColor("LOTUS_WORKER_WINDOWPOST") + "."
// Express as configTOML
configTOML := &bytes.Buffer{}
if err = toml.NewEncoder(configTOML).Encode(lpCfg); err != nil {
return err
}
if !lo.Contains(titles, "base") {
cfg, err := getDefaultConfig(true)
if err != nil {
return xerrors.Errorf("Cannot get default config: %w", err)
}
_, err = db.Exec(ctx, "INSERT INTO harmony_config (title, config) VALUES ('base', $1)", cfg)
if err != nil {
return err
}
}
if cctx.Bool("overwrite") {
i, err := db.Exec(ctx, "DELETE FROM harmony_config WHERE title=$1", name)
if i != 0 {
fmt.Println("Overwriting existing layer")
}
if err != nil {
fmt.Println("Got error while deleting existing layer: " + err.Error())
}
}
_, err = db.Exec(ctx, "INSERT INTO harmony_config (title, config) VALUES ($1, $2)", name, configTOML.String())
if err != nil {
return err
}
dbSettings := ""
def := config.DefaultStorageMiner().HarmonyDB
if def.Hosts[0] != smCfg.HarmonyDB.Hosts[0] {
dbSettings += ` --db-host="` + strings.Join(smCfg.HarmonyDB.Hosts, ",") + `"`
}
if def.Port != smCfg.HarmonyDB.Port {
dbSettings += " --db-port=" + smCfg.HarmonyDB.Port
}
if def.Username != smCfg.HarmonyDB.Username {
dbSettings += ` --db-user="` + smCfg.HarmonyDB.Username + `"`
}
if def.Password != smCfg.HarmonyDB.Password {
dbSettings += ` --db-password="` + smCfg.HarmonyDB.Password + `"`
}
if def.Database != smCfg.HarmonyDB.Database {
dbSettings += ` --db-name="` + smCfg.HarmonyDB.Database + `"`
}
msg += `
To work with the config:
` + cliCommandColor(`lotus-provider `+dbSettings+` config help `)
msg += `
To run Lotus Provider: in its own machine or cgroup without other files, use the command:
` + cliCommandColor(`lotus-provider `+dbSettings+` run --layers="`+name+`"`)
fmt.Println(msg)
return nil
}

198
cmd/lotus-provider/proving.go Normal file
View File

@ -0,0 +1,198 @@
package main
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"os"
"time"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/dline"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/provider"
)
var testCmd = &cli.Command{
Name: "test",
Usage: "Utility functions for testing",
Subcommands: []*cli.Command{
//provingInfoCmd,
wdPostCmd,
},
}
var wdPostCmd = &cli.Command{
Name: "window-post",
Aliases: []string{"wd", "windowpost", "wdpost"},
Usage: "Compute a proof-of-spacetime for a sector (requires the sector to be pre-sealed). These will not send to the chain.",
Subcommands: []*cli.Command{
wdPostHereCmd,
wdPostTaskCmd,
},
}
// wdPostTaskCmd writes to harmony_task and wdpost_partition_tasks, then waits for the result.
// It is intended to be used to test the windowpost scheduler.
// The end of the compute task puts the task_id onto wdpost_proofs, which is read by the submit task.
// The submit task will not send test tasks to the chain, and instead will write the result to harmony_test.
// The result is read by this command, and printed to stdout.
var wdPostTaskCmd = &cli.Command{
Name: "task",
Aliases: []string{"scheduled", "schedule", "async", "asynchronous"},
Usage: "Test the windowpost scheduler by running it on the next available lotus-provider. ",
Flags: []cli.Flag{
&cli.Uint64Flag{
Name: "deadline",
Usage: "deadline to compute WindowPoSt for ",
Value: 0,
},
&cli.StringSliceFlag{
Name: "layers",
Usage: "list of layers to be interpreted (atop defaults). Default: base",
Value: cli.NewStringSlice("base"),
},
},
Action: func(cctx *cli.Context) error {
ctx := context.Background()
deps, err := getDeps(ctx, cctx)
if err != nil {
return err
}
ts, err := deps.full.ChainHead(ctx)
if err != nil {
return xerrors.Errorf("cannot get chainhead %w", err)
}
ht := ts.Height()
addr, err := address.NewFromString(deps.cfg.Addresses.MinerAddresses[0])
if err != nil {
return xerrors.Errorf("cannot get miner address %w", err)
}
maddr, err := address.IDFromAddress(addr)
if err != nil {
return xerrors.Errorf("cannot get miner id %w", err)
}
var id int64
_, err = deps.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
err = tx.QueryRow(`INSERT INTO harmony_task (name, posted_time, added_by) VALUES ('WdPost', CURRENT_TIMESTAMP, 123) RETURNING id`).Scan(&id)
if err != nil {
log.Error("inserting harmony_task: ", err)
return false, xerrors.Errorf("inserting harmony_task: %w", err)
}
_, err = tx.Exec(`INSERT INTO wdpost_partition_tasks
(task_id, sp_id, proving_period_start, deadline_index, partition_index) VALUES ($1, $2, $3, $4, $5)`,
id, maddr, ht, cctx.Uint64("deadline"), 0)
if err != nil {
log.Error("inserting wdpost_partition_tasks: ", err)
return false, xerrors.Errorf("inserting wdpost_partition_tasks: %w", err)
}
_, err = tx.Exec("INSERT INTO harmony_test (task_id) VALUES ($1)", id)
if err != nil {
return false, xerrors.Errorf("inserting into harmony_tests: %w", err)
}
return true, nil
})
if err != nil {
return xerrors.Errorf("writing SQL transaction: %w", err)
}
fmt.Printf("Inserted task %v. Waiting for success ", id)
var result sql.NullString
for {
time.Sleep(time.Second)
err = deps.db.QueryRow(ctx, `SELECT result FROM harmony_test WHERE task_id=$1`, id).Scan(&result)
if err != nil {
return xerrors.Errorf("reading result from harmony_test: %w", err)
}
if result.Valid {
break
}
fmt.Print(".")
}
log.Infof("Result:", result.String)
return nil
},
}
// This command is intended to be used to verify PoSt compute performance.
// It will not send any messages to the chain. Since it can compute any deadline, output may be incorrectly timed for the chain.
// The entire processing happens in this process while you wait. It does not use the scheduler.
var wdPostHereCmd = &cli.Command{
Name: "here",
Aliases: []string{"cli"},
Usage: "Compute WindowPoSt for performance and configuration testing.",
Description: `Note: This command is intended to be used to verify PoSt compute performance.
It will not send any messages to the chain. Since it can compute any deadline, output may be incorrectly timed for the chain.`,
ArgsUsage: "[deadline index]",
Flags: []cli.Flag{
&cli.Uint64Flag{
Name: "deadline",
Usage: "deadline to compute WindowPoSt for ",
Value: 0,
},
&cli.StringSliceFlag{
Name: "layers",
Usage: "list of layers to be interpreted (atop defaults). Default: base",
Value: cli.NewStringSlice("base"),
},
&cli.StringFlag{
Name: "storage-json",
Usage: "path to json file containing storage config",
Value: "~/.lotus-provider/storage.json",
},
&cli.Uint64Flag{
Name: "partition",
Usage: "partition to compute WindowPoSt for",
Value: 0,
},
},
Action: func(cctx *cli.Context) error {
ctx := context.Background()
deps, err := getDeps(ctx, cctx)
if err != nil {
return err
}
wdPostTask, wdPoStSubmitTask, derlareRecoverTask, err := provider.WindowPostScheduler(ctx, deps.cfg.Fees, deps.cfg.Proving, deps.full, deps.verif, deps.lw, nil,
deps.as, deps.maddrs, deps.db, deps.stor, deps.si, deps.cfg.Subsystems.WindowPostMaxTasks)
if err != nil {
return err
}
_, _ = wdPoStSubmitTask, derlareRecoverTask
if len(deps.maddrs) == 0 {
return errors.New("no miners to compute WindowPoSt for")
}
head, err := deps.full.ChainHead(ctx)
if err != nil {
return xerrors.Errorf("failed to get chain head: %w", err)
}
di := dline.NewInfo(head.Height(), cctx.Uint64("deadline"), 0, 0, 0, 10 /*challenge window*/, 0, 0)
for _, maddr := range deps.maddrs {
out, err := wdPostTask.DoPartition(ctx, head, address.Address(maddr), di, cctx.Uint64("partition"))
if err != nil {
fmt.Println("Error computing WindowPoSt for miner", maddr, err)
continue
}
fmt.Println("Computed WindowPoSt for miner", maddr, ":")
err = json.NewEncoder(os.Stdout).Encode(out)
if err != nil {
fmt.Println("Could not encode WindowPoSt output for miner", maddr, err)
continue
}
}
return nil
},
}

51
cmd/lotus-provider/rpc/rpc.go Normal file
View File

@ -0,0 +1,51 @@
package rpc
import (
"context"
"net/http"
"github.com/gorilla/mux"
// logging "github.com/ipfs/go-log/v2"
"github.com/filecoin-project/go-jsonrpc"
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/rpcenc"
"github.com/filecoin-project/lotus/metrics/proxy"
)
//var log = logging.Logger("lp/rpc")
func LotusProviderHandler(
authv func(ctx context.Context, token string) ([]auth.Permission, error),
remote http.HandlerFunc,
a api.LotusProvider,
permissioned bool) http.Handler {
mux := mux.NewRouter()
readerHandler, readerServerOpt := rpcenc.ReaderParamDecoder()
rpcServer := jsonrpc.NewServer(jsonrpc.WithServerErrors(api.RPCErrors), readerServerOpt)
wapi := proxy.MetricedAPI[api.LotusProvider, api.LotusProviderStruct](a)
if permissioned {
wapi = api.PermissionedAPI[api.LotusProvider, api.LotusProviderStruct](wapi)
}
rpcServer.Register("Filecoin", wapi)
rpcServer.AliasMethod("rpc.discover", "Filecoin.Discover")
mux.Handle("/rpc/v0", rpcServer)
mux.Handle("/rpc/streams/v0/push/{uuid}", readerHandler)
mux.PathPrefix("/remote").HandlerFunc(remote)
mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof
if !permissioned {
return mux
}
ah := &auth.Handler{
Verify: authv,
Next: mux.ServeHTTP,
}
return ah
}

459
cmd/lotus-provider/run.go Normal file
View File

@ -0,0 +1,459 @@
package main
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"net"
"net/http"
"os"
"strings"
"time"
"github.com/gbrlsnchs/jwt/v3"
"github.com/gorilla/mux"
ds "github.com/ipfs/go-datastore"
dssync "github.com/ipfs/go-datastore/sync"
"github.com/pkg/errors"
"github.com/samber/lo"
"github.com/urfave/cli/v2"
"go.opencensus.io/stats"
"go.opencensus.io/tag"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
lcli "github.com/filecoin-project/lotus/cli"
cliutil "github.com/filecoin-project/lotus/cli/util"
"github.com/filecoin-project/lotus/cmd/lotus-provider/rpc"
"github.com/filecoin-project/lotus/journal"
"github.com/filecoin-project/lotus/journal/alerting"
"github.com/filecoin-project/lotus/journal/fsjournal"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
"github.com/filecoin-project/lotus/lib/ulimit"
"github.com/filecoin-project/lotus/metrics"
"github.com/filecoin-project/lotus/node"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/lotus/provider"
"github.com/filecoin-project/lotus/provider/lpmessage"
"github.com/filecoin-project/lotus/provider/lpwinning"
"github.com/filecoin-project/lotus/storage/ctladdr"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer"
"github.com/filecoin-project/lotus/storage/sealer/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
type stackTracer interface {
StackTrace() errors.StackTrace
}
var runCmd = &cli.Command{
Name: "run",
Usage: "Start a lotus provider process",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "listen",
Usage: "host address and port the worker api will listen on",
Value: "0.0.0.0:12300",
EnvVars: []string{"LOTUS_WORKER_LISTEN"},
},
&cli.BoolFlag{
Name: "nosync",
Usage: "don't check full-node sync status",
},
&cli.BoolFlag{
Name: "halt-after-init",
Usage: "only run init, then return",
Hidden: true,
},
&cli.BoolFlag{
Name: "manage-fdlimit",
Usage: "manage open file limit",
Value: true,
},
&cli.StringSliceFlag{
Name: "layers",
Usage: "list of layers to be interpreted (atop defaults). Default: base",
Value: cli.NewStringSlice("base"),
},
&cli.StringFlag{
Name: "storage-json",
Usage: "path to json file containing storage config",
Value: "~/.lotus-provider/storage.json",
},
&cli.StringFlag{
Name: "journal",
Usage: "path to journal files",
Value: "~/.lotus-provider/",
},
},
Action: func(cctx *cli.Context) (err error) {
defer func() {
if err != nil {
if err, ok := err.(stackTracer); ok {
for _, f := range err.StackTrace() {
fmt.Printf("%+s:%d\n", f, f)
}
}
}
}()
if !cctx.Bool("enable-gpu-proving") {
err := os.Setenv("BELLMAN_NO_GPU", "true")
if err != nil {
return err
}
}
ctx, _ := tag.New(lcli.DaemonContext(cctx),
tag.Insert(metrics.Version, build.BuildVersion),
tag.Insert(metrics.Commit, build.CurrentCommit),
tag.Insert(metrics.NodeType, "provider"),
)
shutdownChan := make(chan struct{})
{
var ctxclose func()
ctx, ctxclose = context.WithCancel(ctx)
go func() {
<-shutdownChan
ctxclose()
}()
}
// Register all metric views
/*
if err := view.Register(
metrics.MinerNodeViews...,
); err != nil {
log.Fatalf("Cannot register the view: %v", err)
}
*/
// Set the metric to one so it is published to the exporter
stats.Record(ctx, metrics.LotusInfo.M(1))
if cctx.Bool("manage-fdlimit") {
if _, _, err := ulimit.ManageFdLimit(); err != nil {
log.Errorf("setting file descriptor limit: %s", err)
}
}
deps, err := getDeps(ctx, cctx)
if err != nil {
return err
}
cfg, db, full, verif, lw, as, maddrs, stor, si, localStore := deps.cfg, deps.db, deps.full, deps.verif, deps.lw, deps.as, deps.maddrs, deps.stor, deps.si, deps.localStore
var activeTasks []harmonytask.TaskInterface
sender, sendTask := lpmessage.NewSender(full, full, db)
activeTasks = append(activeTasks, sendTask)
///////////////////////////////////////////////////////////////////////
///// Task Selection
///////////////////////////////////////////////////////////////////////
{
if cfg.Subsystems.EnableWindowPost {
wdPostTask, wdPoStSubmitTask, derlareRecoverTask, err := provider.WindowPostScheduler(ctx, cfg.Fees, cfg.Proving, full, verif, lw, sender,
as, maddrs, db, stor, si, cfg.Subsystems.WindowPostMaxTasks)
if err != nil {
return err
}
activeTasks = append(activeTasks, wdPostTask, wdPoStSubmitTask, derlareRecoverTask)
}
if cfg.Subsystems.EnableWinningPost {
winPoStTask := lpwinning.NewWinPostTask(cfg.Subsystems.WinningPostMaxTasks, db, lw, verif, full, maddrs)
activeTasks = append(activeTasks, winPoStTask)
}
}
log.Infow("This lotus_provider instance handles",
"miner_addresses", maddrs,
"tasks", lo.Map(activeTasks, func(t harmonytask.TaskInterface, _ int) string { return t.TypeDetails().Name }))
taskEngine, err := harmonytask.New(db, activeTasks, deps.listenAddr)
if err != nil {
return err
}
defer taskEngine.GracefullyTerminate(time.Hour)
fh := &paths.FetchHandler{Local: localStore, PfHandler: &paths.DefaultPartialFileHandler{}}
remoteHandler := func(w http.ResponseWriter, r *http.Request) {
if !auth.HasPerm(r.Context(), nil, api.PermAdmin) {
w.WriteHeader(401)
_ = json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing admin permission"})
return
}
fh.ServeHTTP(w, r)
}
// local APIs
{
// debugging
mux := mux.NewRouter()
mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof
mux.PathPrefix("/remote").HandlerFunc(remoteHandler)
/*ah := &auth.Handler{
Verify: authv,
Next: mux.ServeHTTP,
}*/ // todo
}
var authVerify func(context.Context, string) ([]auth.Permission, error)
{
privateKey, err := base64.StdEncoding.DecodeString(deps.cfg.Apis.StorageRPCSecret)
if err != nil {
return xerrors.Errorf("decoding storage rpc secret: %w", err)
}
authVerify = func(ctx context.Context, token string) ([]auth.Permission, error) {
var payload jwtPayload
if _, err := jwt.Verify([]byte(token), jwt.NewHS256(privateKey), &payload); err != nil {
return nil, xerrors.Errorf("JWT Verification failed: %w", err)
}
return payload.Allow, nil
}
}
// Serve the RPC.
srv := &http.Server{
Handler: rpc.LotusProviderHandler(
authVerify,
remoteHandler,
&ProviderAPI{deps, shutdownChan},
true),
ReadHeaderTimeout: time.Minute * 3,
BaseContext: func(listener net.Listener) context.Context {
ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, "lotus-worker"))
return ctx
},
}
go func() {
<-ctx.Done()
log.Warn("Shutting down...")
if err := srv.Shutdown(context.TODO()); err != nil {
log.Errorf("shutting down RPC server failed: %s", err)
}
log.Warn("Graceful shutdown successful")
}()
// Monitor for shutdown.
// TODO provide a graceful shutdown API on shutdownChan
finishCh := node.MonitorShutdown(shutdownChan) //node.ShutdownHandler{Component: "rpc server", StopFunc: rpcStopper},
//node.ShutdownHandler{Component: "provider", StopFunc: stop},
<-finishCh
return nil
},
}
func makeDB(cctx *cli.Context) (*harmonydb.DB, error) {
dbConfig := config.HarmonyDB{
Username: cctx.String("db-user"),
Password: cctx.String("db-password"),
Hosts: strings.Split(cctx.String("db-host"), ","),
Database: cctx.String("db-name"),
Port: cctx.String("db-port"),
}
return harmonydb.NewFromConfig(dbConfig)
}
type jwtPayload struct {
Allow []auth.Permission
}
func StorageAuth(apiKey string) (sealer.StorageAuth, error) {
if apiKey == "" {
return nil, xerrors.Errorf("no api key provided")
}
rawKey, err := base64.StdEncoding.DecodeString(apiKey)
if err != nil {
return nil, xerrors.Errorf("decoding api key: %w", err)
}
key := jwt.NewHS256(rawKey)
p := jwtPayload{
Allow: []auth.Permission{"admin"},
}
token, err := jwt.Sign(&p, key)
if err != nil {
return nil, err
}
headers := http.Header{}
headers.Add("Authorization", "Bearer "+string(token))
return sealer.StorageAuth(headers), nil
}
type Deps struct {
cfg *config.LotusProviderConfig
db *harmonydb.DB
full api.FullNode
verif storiface.Verifier
lw *sealer.LocalWorker
as *ctladdr.AddressSelector
maddrs []dtypes.MinerAddress
stor *paths.Remote
si *paths.DBIndex
localStore *paths.Local
listenAddr string
}
func getDeps(ctx context.Context, cctx *cli.Context) (*Deps, error) {
// Open repo
repoPath := cctx.String(FlagRepoPath)
fmt.Println("repopath", repoPath)
r, err := repo.NewFS(repoPath)
if err != nil {
return nil, err
}
ok, err := r.Exists()
if err != nil {
return nil, err
}
if !ok {
if err := r.Init(repo.Provider); err != nil {
return nil, err
}
}
db, err := makeDB(cctx)
if err != nil {
return nil, err
}
///////////////////////////////////////////////////////////////////////
///// Dependency Setup
///////////////////////////////////////////////////////////////////////
// The config feeds into task runners & their helpers
cfg, err := getConfig(cctx, db)
if err != nil {
return nil, err
}
log.Debugw("config", "config", cfg)
var verif storiface.Verifier = ffiwrapper.ProofVerifier
as, err := provider.AddressSelector(&cfg.Addresses)()
if err != nil {
return nil, err
}
de, err := journal.ParseDisabledEvents(cfg.Journal.DisabledEvents)
if err != nil {
return nil, err
}
j, err := fsjournal.OpenFSJournalPath(cctx.String("journal"), de)
if err != nil {
return nil, err
}
full, fullCloser, err := cliutil.GetFullNodeAPIV1LotusProvider(cctx, cfg.Apis.ChainApiInfo)
if err != nil {
return nil, err
}
go func() {
select {
case <-ctx.Done():
fullCloser()
_ = j.Close()
}
}()
sa, err := StorageAuth(cfg.Apis.StorageRPCSecret)
if err != nil {
return nil, xerrors.Errorf(`'%w' while parsing the config toml's
[Apis]
StorageRPCSecret=%v
Get it with: jq .PrivateKey ~/.lotus-miner/keystore/MF2XI2BNNJ3XILLQOJUXMYLUMU`, err, cfg.Apis.StorageRPCSecret)
}
al := alerting.NewAlertingSystem(j)
si := paths.NewDBIndex(al, db)
bls := &paths.BasicLocalStorage{
PathToJSON: cctx.String("storage-json"),
}
listenAddr := cctx.String("listen")
const unspecifiedAddress = "0.0.0.0"
addressSlice := strings.Split(listenAddr, ":")
if ip := net.ParseIP(addressSlice[0]); ip != nil {
if ip.String() == unspecifiedAddress {
rip, err := db.GetRoutableIP()
if err != nil {
return nil, err
}
listenAddr = rip + ":" + addressSlice[1]
}
}
localStore, err := paths.NewLocal(ctx, bls, si, []string{"http://" + listenAddr + "/remote"})
if err != nil {
return nil, err
}
stor := paths.NewRemote(localStore, si, http.Header(sa), 10, &paths.DefaultPartialFileHandler{})
wstates := statestore.New(dssync.MutexWrap(ds.NewMapDatastore()))
// todo localWorker isn't the abstraction layer we want to use here, we probably want to go straight to ffiwrapper
// maybe with a lotus-provider specific abstraction. LocalWorker does persistent call tracking which we probably
// don't need (ehh.. maybe we do, the async callback system may actually work decently well with harmonytask)
lw := sealer.NewLocalWorker(sealer.WorkerConfig{}, stor, localStore, si, nil, wstates)
var maddrs []dtypes.MinerAddress
for _, s := range cfg.Addresses.MinerAddresses {
addr, err := address.NewFromString(s)
if err != nil {
return nil, err
}
maddrs = append(maddrs, dtypes.MinerAddress(addr))
}
return &Deps{ // lint: intentionally not-named so it will fail if one is forgotten
cfg,
db,
full,
verif,
lw,
as,
maddrs,
stor,
si,
localStore,
listenAddr,
}, nil
}
type ProviderAPI struct {
*Deps
ShutdownChan chan struct{}
}
func (p *ProviderAPI) Version(context.Context) (api.Version, error) {
return api.ProviderAPIVersion0, nil
}
// Trigger shutdown
func (p *ProviderAPI) Shutdown(context.Context) error {
close(p.ShutdownChan)
return nil
}

29
cmd/lotus-provider/stop.go Normal file
View File

@ -0,0 +1,29 @@
package main
import (
_ "net/http/pprof"
"github.com/urfave/cli/v2"
lcli "github.com/filecoin-project/lotus/cli"
)
var stopCmd = &cli.Command{
Name: "stop",
Usage: "Stop a running lotus provider",
Flags: []cli.Flag{},
Action: func(cctx *cli.Context) error {
api, closer, err := lcli.GetAPI(cctx)
if err != nil {
return err
}
defer closer()
err = api.Shutdown(lcli.ReqContext(cctx))
if err != nil {
return err
}
return nil
},
}

3
cmd/lotus-shed/terminations.go
View File

@ -157,7 +157,8 @@ var terminationsCmd = &cli.Command{
}
for _, t := range termParams.Terminations {
sectors, err := minerSt.LoadSectors(&t.Sectors)
tmp := t.Sectors
sectors, err := minerSt.LoadSectors(&tmp)
if err != nil {
return err
}

3
cmd/lotus-sim/simulation/stages/funding_stage.go
View File

@ -166,7 +166,8 @@ func (fs *FundingStage) PackMessages(ctx context.Context, bb *blockbuilder.Block
)
}()
for _, actor := range targets {
for _, actorTmp := range targets {
actor := actorTmp
switch {
case builtin.IsAccountActor(actor.Code):
if _, err := bb.PushMessage(&types.Message{

6
cmd/lotus-worker/sealworker/rpc.go
View File

@ -26,7 +26,11 @@ import (
var log = logging.Logger("sealworker")
func WorkerHandler(authv func(ctx context.Context, token string) ([]auth.Permission, error), remote http.HandlerFunc, a api.Worker, permissioned bool) http.Handler {
func WorkerHandler(
authv func(ctx context.Context, token string) ([]auth.Permission, error),
remote http.HandlerFunc,
a api.Worker,
permissioned bool) http.Handler {
mux := mux.NewRouter()
readerHandler, readerServerOpt := rpcenc.ReaderParamDecoder()
rpcServer := jsonrpc.NewServer(jsonrpc.WithServerErrors(api.RPCErrors), readerServerOpt)

20
cmd/lotus/daemon.go
View File

@ -269,6 +269,26 @@ var DaemonCmd = &cli.Command{
}
}
if cctx.Bool("remove-existing-chain") {
lr, err := repo.NewFS(cctx.String("repo"))
if err != nil {
return xerrors.Errorf("error opening fs repo: %w", err)
}
exists, err := lr.Exists()
if err != nil {
return err
}
if !exists {
return xerrors.Errorf("lotus repo doesn't exist")
}
err = removeExistingChain(cctx, lr)
if err != nil {
return err
}
}
chainfile := cctx.String("import-chain")
snapshot := cctx.String("import-snapshot")
willImportChain := false

66
documentation/en/default-lotus-miner-config.toml
View File

@ -145,6 +145,14 @@
# env var: LOTUS_SUBSYSTEMS_ENABLEMARKETS
#EnableMarkets = false
# When enabled, the sector index will reside in an external database
# as opposed to the local KV store in the miner process
# This is useful to allow workers to bypass the lotus miner to access sector information
#
# type: bool
# env var: LOTUS_SUBSYSTEMS_ENABLESECTORINDEXDB
#EnableSectorIndexDB = false
# type: string
# env var: LOTUS_SUBSYSTEMS_SEALERAPIINFO
#SealerApiInfo = ""
@ -153,6 +161,31 @@
# env var: LOTUS_SUBSYSTEMS_SECTORINDEXAPIINFO
#SectorIndexApiInfo = ""
# When window post is enabled, the miner will automatically submit window post proofs
# for all sectors that are eligible for window post
# IF WINDOW POST IS DISABLED, THE MINER WILL NOT SUBMIT WINDOW POST PROOFS
# THIS WILL RESULT IN FAULTS AND PENALTIES IF NO OTHER MECHANISM IS RUNNING
# TO SUBMIT WINDOW POST PROOFS.
# Note: This option is entirely disabling the window post scheduler,
# not just the builtin PoSt computation like Proving.DisableBuiltinWindowPoSt.
# This option will stop lotus-miner from performing any actions related
# to window post, including scheduling, submitting proofs, and recovering
# sectors.
#
# type: bool
# env var: LOTUS_SUBSYSTEMS_DISABLEWINDOWPOST
#DisableWindowPoSt = false
# When winning post is disabled, the miner process will NOT attempt to mine
# blocks. This should only be set when there's an external process mining
# blocks on behalf of the miner.
# When disabled and no external block producers are configured, all potential
# block rewards will be missed!
#
# type: bool
# env var: LOTUS_SUBSYSTEMS_DISABLEWINNINGPOST
#DisableWinningPoSt = false
[Dealmaking]
# When enabled, the miner can accept online deals
@ -896,3 +929,36 @@
#GCInterval = "1m0s"
[HarmonyDB]
# HOSTS is a list of hostnames to nodes running YugabyteDB
# in a cluster. Only 1 is required
#
# type: []string
# env var: LOTUS_HARMONYDB_HOSTS
#Hosts = ["127.0.0.1"]
# The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default.
#
# type: string
# env var: LOTUS_HARMONYDB_USERNAME
#Username = "yugabyte"
# The password for the related username. Blank for default.
#
# type: string
# env var: LOTUS_HARMONYDB_PASSWORD
#Password = "yugabyte"
# The database (logical partition) within Yugabyte. Blank for default.
#
# type: string
# env var: LOTUS_HARMONYDB_DATABASE
#Database = "yugabyte"
# The port to find Yugabyte. Blank for default.
#
# type: string
# env var: LOTUS_HARMONYDB_PORT
#Port = "5433"

209
documentation/en/default-lotus-provider-config.toml Normal file
View File

@ -0,0 +1,209 @@
[Subsystems]
# type: bool
#EnableWindowPost = false
# type: int
#WindowPostMaxTasks = 0
# type: bool
#EnableWinningPost = false
# type: int
#WinningPostMaxTasks = 0
[Fees]
# type: types.FIL
#DefaultMaxFee = "0.07 FIL"
# type: types.FIL
#MaxPreCommitGasFee = "0.025 FIL"
# type: types.FIL
#MaxCommitGasFee = "0.05 FIL"
# type: types.FIL
#MaxTerminateGasFee = "0.5 FIL"
# WindowPoSt is a high-value operation, so the default fee should be high.
#
# type: types.FIL
#MaxWindowPoStGasFee = "5 FIL"
# type: types.FIL
#MaxPublishDealsFee = "0.05 FIL"
[Fees.MaxPreCommitBatchGasFee]
# type: types.FIL
#Base = "0 FIL"
# type: types.FIL
#PerSector = "0.02 FIL"
[Fees.MaxCommitBatchGasFee]
# type: types.FIL
#Base = "0 FIL"
# type: types.FIL
#PerSector = "0.03 FIL"
[Addresses]
# Addresses to send PreCommit messages from
#
# type: []string
#PreCommitControl = []
# Addresses to send Commit messages from
#
# type: []string
#CommitControl = []
# type: []string
#TerminateControl = []
# DisableOwnerFallback disables usage of the owner address for messages
# sent automatically
#
# type: bool
#DisableOwnerFallback = false
# DisableWorkerFallback disables usage of the worker address for messages
# sent automatically, if control addresses are configured.
# A control address that doesn't have enough funds will still be chosen
# over the worker address if this flag is set.
#
# type: bool
#DisableWorkerFallback = false
[Proving]
# Maximum number of sector checks to run in parallel. (0 = unlimited)
#
# WARNING: Setting this value too high may make the node crash by running out of stack
# WARNING: Setting this value too low may make sector challenge reading much slower, resulting in failed PoSt due
# to late submission.
#
# After changing this option, confirm that the new value works in your setup by invoking
# 'lotus-miner proving compute window-post 0'
#
# type: int
#ParallelCheckLimit = 32
# Maximum amount of time a proving pre-check can take for a sector. If the check times out the sector will be skipped
#
# WARNING: Setting this value too low risks in sectors being skipped even though they are accessible, just reading the
# test challenge took longer than this timeout
# WARNING: Setting this value too high risks missing PoSt deadline in case IO operations related to this sector are
# blocked (e.g. in case of disconnected NFS mount)
#
# type: Duration
#SingleCheckTimeout = "10m0s"
# Maximum amount of time a proving pre-check can take for an entire partition. If the check times out, sectors in
# the partition which didn't get checked on time will be skipped
#
# WARNING: Setting this value too low risks in sectors being skipped even though they are accessible, just reading the
# test challenge took longer than this timeout
# WARNING: Setting this value too high risks missing PoSt deadline in case IO operations related to this partition are
# blocked or slow
#
# type: Duration
#PartitionCheckTimeout = "20m0s"
# Disable Window PoSt computation on the lotus-miner process even if no window PoSt workers are present.
#
# WARNING: If no windowPoSt workers are connected, window PoSt WILL FAIL resulting in faulty sectors which will need
# to be recovered. Before enabling this option, make sure your PoSt workers work correctly.
#
# After changing this option, confirm that the new value works in your setup by invoking
# 'lotus-miner proving compute window-post 0'
#
# type: bool
#DisableBuiltinWindowPoSt = false
# Disable Winning PoSt computation on the lotus-miner process even if no winning PoSt workers are present.
#
# WARNING: If no WinningPoSt workers are connected, Winning PoSt WILL FAIL resulting in lost block rewards.
# Before enabling this option, make sure your PoSt workers work correctly.
#
# type: bool
#DisableBuiltinWinningPoSt = false
# Disable WindowPoSt provable sector readability checks.
#
# In normal operation, when preparing to compute WindowPoSt, lotus-miner will perform a round of reading challenges
# from all sectors to confirm that those sectors can be proven. Challenges read in this process are discarded, as
# we're only interested in checking that sector data can be read.
#
# When using builtin proof computation (no PoSt workers, and DisableBuiltinWindowPoSt is set to false), this process
# can save a lot of time and compute resources in the case that some sectors are not readable - this is caused by
# the builtin logic not skipping snark computation when some sectors need to be skipped.
#
# When using PoSt workers, this process is mostly redundant, with PoSt workers challenges will be read once, and
# if challenges for some sectors aren't readable, those sectors will just get skipped.
#
# Disabling sector pre-checks will slightly reduce IO load when proving sectors, possibly resulting in shorter
# time to produce window PoSt. In setups with good IO capabilities the effect of this option on proving time should
# be negligible.
#
# NOTE: It likely is a bad idea to disable sector pre-checks in setups with no PoSt workers.
#
# NOTE: Even when this option is enabled, recovering sectors will be checked before recovery declaration message is
# sent to the chain
#
# After changing this option, confirm that the new value works in your setup by invoking
# 'lotus-miner proving compute window-post 0'
#
# type: bool
#DisableWDPoStPreChecks = false
# Maximum number of partitions to prove in a single SubmitWindowPoSt messace. 0 = network limit (3 in nv21)
#
# A single partition may contain up to 2349 32GiB sectors, or 2300 64GiB sectors.
# //
# Note that setting this value lower may result in less efficient gas use - more messages will be sent,
# to prove each deadline, resulting in more total gas use (but each message will have lower gas limit)
#
# Setting this value above the network limit has no effect
#
# type: int
#MaxPartitionsPerPoStMessage = 0
# In some cases when submitting DeclareFaultsRecovered messages,
# there may be too many recoveries to fit in a BlockGasLimit.
# In those cases it may be necessary to set this value to something low (eg 1);
# Note that setting this value lower may result in less efficient gas use - more messages will be sent than needed,
# resulting in more total gas use (but each message will have lower gas limit)
#
# type: int
#MaxPartitionsPerRecoveryMessage = 0
# Enable single partition per PoSt Message for partitions containing recovery sectors
#
# In cases when submitting PoSt messages which contain recovering sectors, the default network limit may still be
# too high to fit in the block gas limit. In those cases, it becomes useful to only house the single partition
# with recovering sectors in the post message
#
# Note that setting this value lower may result in less efficient gas use - more messages will be sent,
# to prove each deadline, resulting in more total gas use (but each message will have lower gas limit)
#
# type: bool
#SingleRecoveringPartitionPerPostMessage = false
[Journal]
# Events of the form: "system1:event1,system1:event2[,...]"
#
# type: string
#DisabledEvents = ""
[Apis]
# RPC Secret for the storage subsystem.
# If integrating with lotus-miner this must match the value from
# cat ~/.lotusminer/keystore/MF2XI2BNNJ3XILLQOJUXMYLUMU | jq -r .PrivateKey
#
# type: string
#StorageRPCSecret = ""

19
go.mod
View File

@ -1,6 +1,6 @@
module github.com/filecoin-project/lotus
go 1.19
go 1.20
retract v1.14.0 // Accidentally force-pushed tag, use v1.14.1+ instead.
@ -62,6 +62,7 @@ require (
github.com/filecoin-project/test-vectors/schema v0.0.7
github.com/gbrlsnchs/jwt/v3 v3.0.1
github.com/gdamore/tcell/v2 v2.2.0
github.com/georgysavva/scany/v2 v2.0.0
github.com/go-openapi/spec v0.19.11
github.com/golang/mock v1.6.0
github.com/google/uuid v1.3.0
@ -104,6 +105,8 @@ require (
github.com/ipld/go-ipld-selector-text-lite v0.0.1
github.com/ipni/go-libipni v0.0.8
github.com/ipni/index-provider v0.12.0
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa
github.com/jackc/pgx/v5 v5.4.1
github.com/kelseyhightower/envconfig v1.4.0
github.com/koalacxr/quantile v0.0.1
github.com/libp2p/go-buffer-pool v0.1.0
@ -129,11 +132,14 @@ require (
github.com/multiformats/go-multihash v0.2.3
github.com/multiformats/go-varint v0.0.7
github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
github.com/pkg/errors v0.9.1
github.com/polydawn/refmt v0.89.0
github.com/prometheus/client_golang v1.14.0
github.com/prometheus/client_golang v1.16.0
github.com/puzpuzpuz/xsync/v2 v2.4.0
github.com/raulk/clock v1.1.0
github.com/raulk/go-watchdog v1.3.0
github.com/samber/lo v1.38.1
github.com/stretchr/testify v1.8.4
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
github.com/triplewz/poseidon v0.0.0-20220525065023-a7cdb0e183e7
@ -247,6 +253,9 @@ require (
github.com/ipfs/go-verifcid v0.0.2 // indirect
github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 // indirect
github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/puddle/v2 v2.2.0 // indirect
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
@ -288,13 +297,11 @@ require (
github.com/onsi/ginkgo/v2 v2.11.0 // indirect
github.com/opencontainers/runtime-spec v1.1.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/prometheus/procfs v0.10.1 // indirect
github.com/prometheus/statsd_exporter v0.22.7 // indirect
github.com/quic-go/qpack v0.4.0 // indirect
github.com/quic-go/qtls-go1-20 v0.3.3 // indirect
@ -308,7 +315,7 @@ require (
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/tidwall/gjson v1.14.4 // indirect
github.com/twmb/murmur3 v1.1.6 // indirect
github.com/ugorji/go/codec v1.2.6 // indirect
github.com/ugorji/go/codec v1.2.11 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.0.1 // indirect
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect

30
go.sum
View File

@ -169,6 +169,7 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp
github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cockroachdb/cockroach-go/v2 v2.2.0 h1:/5znzg5n373N/3ESjHF5SMLxiW4RKB05Ql//KWfeTFs=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
@ -393,6 +394,8 @@ github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdk
github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
github.com/gdamore/tcell/v2 v2.2.0 h1:vSyEgKwraXPSOkvCk7IwOSyX+Pv3V2cV9CikJMXg4U4=
github.com/gdamore/tcell/v2 v2.2.0/go.mod h1:cTTuF84Dlj/RqmaCIV5p4w8uG1zWdk0SF6oBpwHp4fU=
github.com/georgysavva/scany/v2 v2.0.0 h1:RGXqxDv4row7/FYoK8MRXAZXqoWF/NM+NP0q50k3DKU=
github.com/georgysavva/scany/v2 v2.0.0/go.mod h1:sigOdh+0qb/+aOs3TVhehVT10p8qJL7K/Zhyz8vWo38=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
@ -450,6 +453,7 @@ github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@ -855,6 +859,16 @@ github.com/ipni/index-provider v0.12.0 h1:R3F6dxxKNv4XkE4GJZNLOG0bDEbBQ/S5iztXwS
github.com/ipni/index-provider v0.12.0/go.mod h1:GhyrADJp7n06fqoc1djzkvL4buZYHzV8SoWrlxEo5F4=
github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=
github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4=
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa h1:s+4MhCQ6YrzisK6hFJUX53drDT4UsSW3DEhKn0ifuHw=
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.4.1 h1:oKfB/FhuVtit1bBM3zNRRsZ925ZkMN3HXL+LgLUM9lE=
github.com/jackc/pgx/v5 v5.4.1/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY=
github.com/jackc/puddle/v2 v2.2.0 h1:RdcDk92EJBuBS55nQMMYFXTxwstHug4jkhT5pq8VxPk=
github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
@ -945,6 +959,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E=
github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ=
github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E=
github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ=
@ -1415,8 +1430,8 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr
github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ=
github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@ -1450,8 +1465,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0=
github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI=
github.com/puzpuzpuz/xsync/v2 v2.4.0 h1:5sXAMHrtx1bg9nbRZTOn8T4MkWe5V+o8yKRH02Eznag=
@ -1486,6 +1501,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM=
github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sercand/kuberesolver v2.4.0+incompatible h1:WE2OlRf6wjLxHwNkkFLQGaZcVLEXjMjBPjjEU5vksH8=
@ -1596,10 +1613,9 @@ github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg=
github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg=
github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn0=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=
github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=
github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=

174
itests/harmonydb_test.go Normal file
View File

@ -0,0 +1,174 @@
package itests
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"testing"
"github.com/filecoin-project/lotus/itests/kit"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/node/impl"
)
func withSetup(t *testing.T, f func(*kit.TestMiner)) {
_, miner, _ := kit.EnsembleMinimal(t,
kit.LatestActorsAt(-1),
kit.MockProofs(),
kit.WithSectorIndexDB(),
)
f(miner)
}
func TestCrud(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
withSetup(t, func(miner *kit.TestMiner) {
cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB
_, err := cdb.Exec(ctx, `
INSERT INTO
itest_scratch (some_int, content)
VALUES
(11, 'cows'),
(5, 'cats')
`)
if err != nil {
t.Fatal("Could not insert: ", err)
}
var ints []struct {
Count int `db:"some_int"`
Animal string `db:"content"`
Unpopulated int
}
err = cdb.Select(ctx, &ints, "SELECT content, some_int FROM itest_scratch")
if err != nil {
t.Fatal("Could not select: ", err)
}
if len(ints) != 2 {
t.Fatal("unexpected count of returns. Want 2, Got ", len(ints))
}
if ints[0].Count != 11 || ints[1].Count != 5 {
t.Fatal("expected [11,5] got ", ints)
}
if ints[0].Animal != "cows" || ints[1].Animal != "cats" {
t.Fatal("expected, [cows, cats] ", ints)
}
fmt.Println("test completed")
})
}
func TestTransaction(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
withSetup(t, func(miner *kit.TestMiner) {
cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB
if _, err := cdb.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (4), (5), (6)"); err != nil {
t.Fatal("E0", err)
}
_, err := cdb.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
if _, err := tx.Exec("INSERT INTO itest_scratch (some_int) VALUES (7), (8), (9)"); err != nil {
t.Fatal("E1", err)
}
// sum1 is read from OUTSIDE the transaction so it's the old value
var sum1 int
if err := cdb.QueryRow(ctx, "SELECT SUM(some_int) FROM itest_scratch").Scan(&sum1); err != nil {
t.Fatal("E2", err)
}
if sum1 != 4+5+6 {
t.Fatal("Expected 15, got ", sum1)
}
// sum2 is from INSIDE the transaction, so the updated value.
var sum2 int
if err := tx.QueryRow("SELECT SUM(some_int) FROM itest_scratch").Scan(&sum2); err != nil {
t.Fatal("E3", err)
}
if sum2 != 4+5+6+7+8+9 {
t.Fatal("Expected 39, got ", sum2)
}
return false, nil // rollback
})
if err != nil {
t.Fatal("ET", err)
}
var sum2 int
// Query() example (yes, QueryRow would be preferred here)
q, err := cdb.Query(ctx, "SELECT SUM(some_int) FROM itest_scratch")
if err != nil {
t.Fatal("E4", err)
}
defer q.Close()
var rowCt int
for q.Next() {
err := q.Scan(&sum2)
if err != nil {
t.Fatal("error scanning ", err)
}
rowCt++
}
if sum2 != 4+5+6 {
t.Fatal("Expected 15, got ", sum2)
}
if rowCt != 1 {
t.Fatal("unexpected count of rows")
}
})
}
func TestPartialWalk(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
withSetup(t, func(miner *kit.TestMiner) {
cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB
if _, err := cdb.Exec(ctx, `
INSERT INTO
itest_scratch (content, some_int)
VALUES
('andy was here', 5),
('lotus is awesome', 6),
('hello world', 7),
('3rd integration test', 8),
('fiddlesticks', 9)
`); err != nil {
t.Fatal("e1", err)
}
// TASK: FIND THE ID of the string with a specific SHA256
needle := "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"
q, err := cdb.Query(ctx, `SELECT id, content FROM itest_scratch`)
if err != nil {
t.Fatal("e2", err)
}
defer q.Close()
var tmp struct {
Src string `db:"content"`
ID int
}
var done bool
for q.Next() {
if err := q.StructScan(&tmp); err != nil {
t.Fatal("structscan err " + err.Error())
}
bSha := sha256.Sum256([]byte(tmp.Src))
if hex.EncodeToString(bSha[:]) == needle {
done = true
break
}
}
if !done {
t.Fatal("We didn't find it.")
}
// Answer: tmp.ID
})
}

266
itests/harmonytask_test.go Normal file
View File

@ -0,0 +1,266 @@
package itests
import (
"context"
"errors"
"fmt"
"sort"
"sync"
"testing"
"time"
logging "github.com/ipfs/go-log/v2"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/lotus/itests/kit"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
"github.com/filecoin-project/lotus/lib/harmony/resources"
"github.com/filecoin-project/lotus/node/impl"
)
type task1 struct {
toAdd []int
myPersonalTableLock sync.Mutex
myPersonalTable map[harmonytask.TaskID]int // This would typically be a DB table
WorkCompleted []string
}
func withDbSetup(t *testing.T, f func(*kit.TestMiner)) {
_, miner, _ := kit.EnsembleMinimal(t,
kit.LatestActorsAt(-1),
kit.MockProofs(),
kit.WithSectorIndexDB(),
)
logging.SetLogLevel("harmonytask", "debug")
f(miner)
}
func (t *task1) Do(tID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) {
if !stillOwned() {
return false, errors.New("Why not still owned?")
}
t.myPersonalTableLock.Lock()
defer t.myPersonalTableLock.Unlock()
t.WorkCompleted = append(t.WorkCompleted, fmt.Sprintf("taskResult%d", t.myPersonalTable[tID]))
return true, nil
}
func (t *task1) CanAccept(list []harmonytask.TaskID, e *harmonytask.TaskEngine) (*harmonytask.TaskID, error) {
return &list[0], nil
}
func (t *task1) TypeDetails() harmonytask.TaskTypeDetails {
return harmonytask.TaskTypeDetails{
Max: 100,
Name: "ThingOne",
MaxFailures: 1,
Cost: resources.Resources{
Cpu: 1,
Ram: 100 << 10, // at 100kb, it's tiny
},
}
}
func (t *task1) Adder(add harmonytask.AddTaskFunc) {
for _, vTmp := range t.toAdd {
v := vTmp
add(func(tID harmonytask.TaskID, tx *harmonydb.Tx) (bool, error) {
t.myPersonalTableLock.Lock()
defer t.myPersonalTableLock.Unlock()
t.myPersonalTable[tID] = v
return true, nil
})
}
}
func init() {
//logging.SetLogLevel("harmonydb", "debug")
//logging.SetLogLevel("harmonytask", "debug")
}
func TestHarmonyTasks(t *testing.T) {
//t.Parallel()
withDbSetup(t, func(m *kit.TestMiner) {
cdb := m.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB
t1 := &task1{
toAdd: []int{56, 73},
myPersonalTable: map[harmonytask.TaskID]int{},
}
harmonytask.POLL_DURATION = time.Millisecond * 100
e, err := harmonytask.New(cdb, []harmonytask.TaskInterface{t1}, "test:1")
require.NoError(t, err)
time.Sleep(time.Second) // do the work. FLAKYNESS RISK HERE.
e.GracefullyTerminate(time.Minute)
expected := []string{"taskResult56", "taskResult73"}
sort.Strings(t1.WorkCompleted)
require.Equal(t, expected, t1.WorkCompleted, "unexpected results")
})
}
type passthru struct {
dtl harmonytask.TaskTypeDetails
do func(tID harmonytask.TaskID, stillOwned func() bool) (done bool, err error)
canAccept func(list []harmonytask.TaskID, e *harmonytask.TaskEngine) (*harmonytask.TaskID, error)
adder func(add harmonytask.AddTaskFunc)
}
func (t *passthru) Do(tID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) {
return t.do(tID, stillOwned)
}
func (t *passthru) CanAccept(list []harmonytask.TaskID, e *harmonytask.TaskEngine) (*harmonytask.TaskID, error) {
return t.canAccept(list, e)
}
func (t *passthru) TypeDetails() harmonytask.TaskTypeDetails {
return t.dtl
}
func (t *passthru) Adder(add harmonytask.AddTaskFunc) {
if t.adder != nil {
t.adder(add)
}
}
// Common stuff
var dtl = harmonytask.TaskTypeDetails{Name: "foo", Max: -1, Cost: resources.Resources{}}
var lettersMutex sync.Mutex
func fooLetterAdder(t *testing.T, cdb *harmonydb.DB) *passthru {
return &passthru{
dtl: dtl,
canAccept: func(list []harmonytask.TaskID, e *harmonytask.TaskEngine) (*harmonytask.TaskID, error) {
return nil, nil
},
adder: func(add harmonytask.AddTaskFunc) {
for _, vTmp := range []string{"A", "B"} {
v := vTmp
add(func(tID harmonytask.TaskID, tx *harmonydb.Tx) (bool, error) {
_, err := tx.Exec("INSERT INTO itest_scratch (some_int, content) VALUES ($1,$2)", tID, v)
require.NoError(t, err)
return true, nil
})
}
},
}
}
func fooLetterSaver(t *testing.T, cdb *harmonydb.DB, dest *[]string) *passthru {
return &passthru{
dtl: dtl,
canAccept: func(list []harmonytask.TaskID, e *harmonytask.TaskEngine) (*harmonytask.TaskID, error) {
return &list[0], nil
},
do: func(tID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) {
var content string
err = cdb.QueryRow(context.Background(),
"SELECT content FROM itest_scratch WHERE some_int=$1", tID).Scan(&content)
require.NoError(t, err)
lettersMutex.Lock()
defer lettersMutex.Unlock()
*dest = append(*dest, content)
return true, nil
},
}
}
func TestHarmonyTasksWith2PartiesPolling(t *testing.T) {
//t.Parallel()
withDbSetup(t, func(m *kit.TestMiner) {
cdb := m.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB
senderParty := fooLetterAdder(t, cdb)
var dest []string
workerParty := fooLetterSaver(t, cdb, &dest)
harmonytask.POLL_DURATION = time.Millisecond * 100
sender, err := harmonytask.New(cdb, []harmonytask.TaskInterface{senderParty}, "test:1")
require.NoError(t, err)
worker, err := harmonytask.New(cdb, []harmonytask.TaskInterface{workerParty}, "test:2")
require.NoError(t, err)
time.Sleep(time.Second) // do the work. FLAKYNESS RISK HERE.
sender.GracefullyTerminate(time.Second * 5)
worker.GracefullyTerminate(time.Second * 5)
sort.Strings(dest)
require.Equal(t, []string{"A", "B"}, dest)
})
}
func TestWorkStealing(t *testing.T) {
//t.Parallel()
withDbSetup(t, func(m *kit.TestMiner) {
cdb := m.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB
ctx := context.Background()
// The dead worker will be played by a few SQL INSERTS.
_, err := cdb.Exec(ctx, `INSERT INTO harmony_machines
(id, last_contact,host_and_port, cpu, ram, gpu)
VALUES (300, DATE '2000-01-01', 'test:1', 4, 400000, 1)`)
require.ErrorIs(t, err, nil)
_, err = cdb.Exec(ctx, `INSERT INTO harmony_task
(id, name, owner_id, posted_time, added_by)
VALUES (1234, 'foo', 300, DATE '2000-01-01', 300)`)
require.ErrorIs(t, err, nil)
_, err = cdb.Exec(ctx, "INSERT INTO itest_scratch (some_int, content) VALUES (1234, 'M')")
require.ErrorIs(t, err, nil)
harmonytask.POLL_DURATION = time.Millisecond * 100
harmonytask.CLEANUP_FREQUENCY = time.Millisecond * 100
var dest []string
worker, err := harmonytask.New(cdb, []harmonytask.TaskInterface{fooLetterSaver(t, cdb, &dest)}, "test:2")
require.ErrorIs(t, err, nil)
time.Sleep(time.Second) // do the work. FLAKYNESS RISK HERE.
worker.GracefullyTerminate(time.Second * 5)
require.Equal(t, []string{"M"}, dest)
})
}
func TestTaskRetry(t *testing.T) {
//t.Parallel()
withDbSetup(t, func(m *kit.TestMiner) {
cdb := m.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB
senderParty := fooLetterAdder(t, cdb)
harmonytask.POLL_DURATION = time.Millisecond * 100
sender, err := harmonytask.New(cdb, []harmonytask.TaskInterface{senderParty}, "test:1")
require.NoError(t, err)
alreadyFailed := map[string]bool{}
var dest []string
fails2xPerMsg := &passthru{
dtl: dtl,
canAccept: func(list []harmonytask.TaskID, e *harmonytask.TaskEngine) (*harmonytask.TaskID, error) {
return &list[0], nil
},
do: func(tID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) {
var content string
err = cdb.QueryRow(context.Background(),
"SELECT content FROM itest_scratch WHERE some_int=$1", tID).Scan(&content)
require.NoError(t, err)
lettersMutex.Lock()
defer lettersMutex.Unlock()
if !alreadyFailed[content] {
alreadyFailed[content] = true
return false, errors.New("intentional 'error'")
}
dest = append(dest, content)
return true, nil
},
}
rcv, err := harmonytask.New(cdb, []harmonytask.TaskInterface{fails2xPerMsg}, "test:2")
require.NoError(t, err)
time.Sleep(time.Second)
sender.GracefullyTerminate(time.Hour)
rcv.GracefullyTerminate(time.Hour)
sort.Strings(dest)
require.Equal(t, []string{"A", "B"}, dest)
type hist struct {
TaskID int
Result bool
Err string
}
var res []hist
require.NoError(t, cdb.Select(context.Background(), &res,
`SELECT task_id, result, err FROM harmony_task_history
ORDER BY result DESC, task_id`))
require.Equal(t, []hist{
{1, true, ""},
{2, true, ""},
{1, false, "error: intentional 'error'"},
{2, false, "error: intentional 'error'"}}, res)
})
}

32
itests/kit/ensemble.go
View File

@ -49,11 +49,13 @@ import (
"github.com/filecoin-project/lotus/cmd/lotus-worker/sealworker"
"github.com/filecoin-project/lotus/gateway"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/markets/idxprov"
"github.com/filecoin-project/lotus/markets/idxprov/idxprov_test"
lotusminer "github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/impl"
"github.com/filecoin-project/lotus/node/modules"
"github.com/filecoin-project/lotus/node/modules/dtypes"
testing2 "github.com/filecoin-project/lotus/node/modules/testing"
@ -359,6 +361,8 @@ func (n *Ensemble) Start() *Ensemble {
n.mn = mocknet.New()
}
sharedITestID := harmonydb.ITestNewID()
// ---------------------
// FULL NODES
// ---------------------
@ -603,6 +607,7 @@ func (n *Ensemble) Start() *Ensemble {
cfg.Subsystems.EnableMining = m.options.subsystems.Has(SMining)
cfg.Subsystems.EnableSealing = m.options.subsystems.Has(SSealing)
cfg.Subsystems.EnableSectorStorage = m.options.subsystems.Has(SSectorStorage)
cfg.Subsystems.EnableSectorIndexDB = m.options.subsystems.Has(SHarmony)
cfg.Dealmaking.MaxStagingDealsBytes = m.options.maxStagingDealsBytes
if m.options.mainMiner != nil {
@ -724,6 +729,17 @@ func (n *Ensemble) Start() *Ensemble {
// upgrades
node.Override(new(stmgr.UpgradeSchedule), n.options.upgradeSchedule),
node.Override(new(harmonydb.ITestID), sharedITestID),
node.Override(new(config.HarmonyDB), func() config.HarmonyDB {
return config.HarmonyDB{
Hosts: []string{envElse("LOTUS_HARMONYDB_HOSTS", "127.0.0.1")},
Database: "yugabyte",
Username: "yugabyte",
Password: "yugabyte",
Port: "5433",
}
}),
}
if m.options.subsystems.Has(SMarkets) {
@ -770,6 +786,12 @@ func (n *Ensemble) Start() *Ensemble {
require.NoError(n.t, err)
n.t.Cleanup(func() { _ = stop(context.Background()) })
mCopy := m
n.t.Cleanup(func() {
if mCopy.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB != nil {
mCopy.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB.ITestDeleteAll()
}
})
m.BaseAPI = m.StorageMiner
@ -826,6 +848,8 @@ func (n *Ensemble) Start() *Ensemble {
auth := http.Header(nil)
// FUTURE: Use m.MinerNode.(BaseAPI).(impl.StorageMinerAPI).HarmonyDB to setup.
remote := paths.NewRemote(localStore, m.MinerNode, auth, 20, &paths.DefaultPartialFileHandler{})
store := m.options.workerStorageOpt(remote)
@ -855,6 +879,7 @@ func (n *Ensemble) Start() *Ensemble {
require.NoError(n.t, err)
n.active.workers = append(n.active.workers, m)
}
// If we are here, we have processed all inactive workers and moved them
@ -1067,3 +1092,10 @@ func importPreSealMeta(ctx context.Context, meta genesis.Miner, mds dtypes.Metad
size := binary.PutUvarint(buf, uint64(maxSectorID))
return mds.Put(ctx, datastore.NewKey(pipeline.StorageCounterDSPrefix), buf[:size])
}
func envElse(env, els string) string {
if v := os.Getenv(env); v != "" {
return v
}
return els
}

2
itests/kit/node_miner.go
View File

@ -37,6 +37,8 @@ const (
SSealing
SSectorStorage
SHarmony
MinerSubsystems = iota
)

7
itests/kit/node_opts.go
View File

@ -89,6 +89,13 @@ func WithAllSubsystems() NodeOpt {
}
}
func WithSectorIndexDB() NodeOpt {
return func(opts *nodeOpts) error {
opts.subsystems = opts.subsystems.Add(SHarmony)
return nil
}
}
func WithSubsystems(systems ...MinerSubsystem) NodeOpt {
return func(opts *nodeOpts) error {
for _, s := range systems {

1
itests/path_type_filters_test.go
View File

@ -15,6 +15,7 @@ import (
)
func TestPathTypeFilters(t *testing.T) {
runTest := func(t *testing.T, name string, asserts func(t *testing.T, ctx context.Context, miner *kit.TestMiner, run func())) {
t.Run(name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())

2
itests/wdpost_worker_config_test.go
View File

@ -139,7 +139,7 @@ func TestWindowPostNoBuiltinWindowWithWorker(t *testing.T) {
t.Log("post message landed")
bm.MineBlocks(ctx, 2*time.Millisecond)
bm.MineBlocksMustPost(ctx, 2*time.Millisecond)
waitUntil = di.Open + di.WPoStChallengeWindow*3
t.Logf("End for head.Height > %d", waitUntil)

163
itests/worker_test.go
View File

@ -146,7 +146,7 @@ func TestWindowPostWorker(t *testing.T) {
di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
require.NoError(t, err)
bm := ens.InterconnectAll().BeginMining(2 * time.Millisecond)[0]
bm := ens.InterconnectAll().BeginMiningMustPost(2 * time.Millisecond)[0]
di = di.NextNotElapsed()
@ -172,7 +172,7 @@ func TestWindowPostWorker(t *testing.T) {
t.Log("post message landed")
bm.MineBlocks(ctx, 2*time.Millisecond)
bm.MineBlocksMustPost(ctx, 2*time.Millisecond)
waitUntil = di.Open + di.WPoStChallengeWindow*3
t.Logf("End for head.Height > %d", waitUntil)
@ -235,6 +235,8 @@ func TestWindowPostWorker(t *testing.T) {
type badWorkerStorage struct {
paths.Store
t *testing.T
badsector *uint64
notBadCount int
}
@ -242,10 +244,12 @@ type badWorkerStorage struct {
func (bs *badWorkerStorage) GenerateSingleVanillaProof(ctx context.Context, minerID abi.ActorID, si storiface.PostSectorChallenge, ppt abi.RegisteredPoStProof) ([]byte, error) {
if atomic.LoadUint64(bs.badsector) == uint64(si.SectorNumber) {
bs.notBadCount--
bs.t.Logf("Generating proof for sector %d maybe bad nbc=%d", si.SectorNumber, bs.notBadCount)
if bs.notBadCount < 0 {
return nil, xerrors.New("no proof for you")
}
}
bs.t.Logf("Generating proof for sector %d", si.SectorNumber)
return bs.Store.GenerateSingleVanillaProof(ctx, minerID, si, ppt)
}
@ -268,6 +272,7 @@ func TestWindowPostWorkerSkipBadSector(t *testing.T) {
return &badWorkerStorage{
Store: store,
badsector: &badsector,
t: t,
}
}),
kit.ConstructorOpts(node.ApplyIf(node.IsType(repo.StorageMiner),
@ -275,6 +280,7 @@ func TestWindowPostWorkerSkipBadSector(t *testing.T) {
return &badWorkerStorage{
Store: store,
badsector: &badsector,
t: t,
notBadCount: 1,
}
}))))
@ -506,157 +512,6 @@ func TestWorkerName(t *testing.T) {
require.True(t, found)
}
// Tests that V1_1 proofs on post workers with faults
func TestWindowPostV1P1NV20WorkerFault(t *testing.T) {
kit.QuietMiningLogs()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
blocktime := 2 * time.Millisecond
sectors := 2 * 48 * 2
var badsector uint64 = 100000
client, miner, _, ens := kit.EnsembleWorker(t,
kit.PresealSectors(sectors), // 2 sectors per partition, 2 partitions in all 48 deadlines
kit.GenesisNetworkVersion(network.Version20),
kit.ConstructorOpts(
node.Override(new(config.ProvingConfig), func() config.ProvingConfig {
c := config.DefaultStorageMiner()
c.Proving.DisableBuiltinWindowPoSt = true
return c.Proving
}),
node.Override(new(*wdpost.WindowPoStScheduler), modules.WindowPostScheduler(
config.DefaultStorageMiner().Fees,
config.ProvingConfig{
DisableBuiltinWindowPoSt: true,
DisableBuiltinWinningPoSt: false,
DisableWDPoStPreChecks: false,
},
)),
node.Override(new(paths.Store), func(store *paths.Remote) paths.Store {
return &badWorkerStorage{
Store: store,
badsector: &badsector,
notBadCount: 1,
}
})),
kit.ThroughRPC(),
kit.WithTaskTypes([]sealtasks.TaskType{sealtasks.TTGenerateWindowPoSt}),
kit.WithWorkerStorage(func(store paths.Store) paths.Store {
return &badWorkerStorage{
Store: store,
badsector: &badsector,
}
}))
bm := ens.InterconnectAll().BeginMining(blocktime)[0]
maddr, err := miner.ActorAddress(ctx)
require.NoError(t, err)
// wait for sectors to be committed
require.Eventually(t, func() bool {
di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
require.NoError(t, err)
parts, err := client.StateMinerPartitions(ctx, maddr, di.Index, types.EmptyTSK)
require.NoError(t, err)
return len(parts) > 1
}, 30*time.Second, 100*time.Millisecond)
// Wait until just before a deadline opens
{
di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
require.NoError(t, err)
di = di.NextNotElapsed()
t.Log("Running one proving period")
waitUntil := di.Open + di.WPoStChallengeWindow - di.WPoStChallengeLookback - 1
client.WaitTillChain(ctx, kit.HeightAtLeast(waitUntil))
t.Log("Waiting for post message")
bm.Stop()
}
// Remove one sector in the next deadline (so it's skipped)
{
di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
require.NoError(t, err)
parts, err := client.StateMinerPartitions(ctx, maddr, di.Index+1, types.EmptyTSK)
require.NoError(t, err)
require.Greater(t, len(parts), 0)
secs := parts[0].AllSectors
n, err := secs.Count()
require.NoError(t, err)
require.Equal(t, uint64(2), n)
// Drop the sector in first partition
sid, err := secs.First()
require.NoError(t, err)
t.Logf("Drop sector %d; dl %d part %d", sid, di.Index, 0)
atomic.StoreUint64(&badsector, sid)
require.NoError(t, err)
}
bm.MineBlocksMustPost(ctx, 2*time.Millisecond)
mi, err := client.StateMinerInfo(ctx, maddr, types.EmptyTSK)
require.NoError(t, err)
wact, err := client.StateGetActor(ctx, mi.Worker, types.EmptyTSK)
require.NoError(t, err)
en := wact.Nonce
// wait for a new message to be sent from worker address, it will be a PoSt
waitForProof:
for {
//stm: @CHAIN_STATE_GET_ACTOR_001
wact, err := client.StateGetActor(ctx, mi.Worker, types.EmptyTSK)
require.NoError(t, err)
if wact.Nonce > en {
break waitForProof
}
build.Clock.Sleep(blocktime)
}
slm, err := client.StateListMessages(ctx, &api.MessageMatch{To: maddr}, types.EmptyTSK, 0)
require.NoError(t, err)
pmr, err := client.StateSearchMsg(ctx, types.EmptyTSK, slm[0], -1, false)
require.NoError(t, err)
nv, err := client.StateNetworkVersion(ctx, pmr.TipSet)
require.NoError(t, err)
require.Equal(t, network.Version20, nv)
require.True(t, pmr.Receipt.ExitCode.IsSuccess())
slmsg, err := client.ChainGetMessage(ctx, slm[0])
require.NoError(t, err)
var params miner11.SubmitWindowedPoStParams
require.NoError(t, params.UnmarshalCBOR(bytes.NewBuffer(slmsg.Params)))
require.Equal(t, abi.RegisteredPoStProof_StackedDrgWindow2KiBV1_1, params.Proofs[0].PoStProof)
require.Len(t, params.Partitions, 2)
sc0, err := params.Partitions[0].Skipped.Count()
require.NoError(t, err)
require.Equal(t, uint64(1), sc0)
sc1, err := params.Partitions[1].Skipped.Count()
require.NoError(t, err)
require.Equal(t, uint64(0), sc1)
}
// Tests that V1_1 proofs on post worker
func TestWindowPostV1P1NV20Worker(t *testing.T) {
kit.QuietMiningLogs()
@ -685,7 +540,7 @@ func TestWindowPostV1P1NV20Worker(t *testing.T) {
kit.ThroughRPC(),
kit.WithTaskTypes([]sealtasks.TaskType{sealtasks.TTGenerateWindowPoSt}))
ens.InterconnectAll().BeginMining(blocktime)
ens.InterconnectAll().BeginMiningMustPost(blocktime)
maddr, err := miner.ActorAddress(ctx)
require.NoError(t, err)

6
journal/fsjournal/fs.go
View File

@ -37,7 +37,11 @@ type fsJournal struct {
// OpenFSJournal constructs a rolling filesystem journal, with a default
// per-file size limit of 1GiB.
func OpenFSJournal(lr repo.LockedRepo, disabled journal.DisabledEvents) (journal.Journal, error) {
dir := filepath.Join(lr.Path(), "journal")
return OpenFSJournalPath(lr.Path(), disabled)
}
func OpenFSJournalPath(path string, disabled journal.DisabledEvents) (journal.Journal, error) {
dir := filepath.Join(path, "journal")
if err := os.MkdirAll(dir, 0755); err != nil {
return nil, fmt.Errorf("failed to mk directory %s for file journal: %w", dir, err)
}

3
journal/types.go
View File

@ -26,6 +26,9 @@ func ParseDisabledEvents(s string) (DisabledEvents, error) {
s = strings.TrimSpace(s) // sanitize
evts := strings.Split(s, ",")
ret := make(DisabledEvents, 0, len(evts))
if len(s) == 0 {
return ret, nil
}
for _, evt := range evts {
evt = strings.TrimSpace(evt) // sanitize
s := strings.Split(evt, ":")

35
lib/harmony/harmonydb/doc.go Normal file
View File

@ -0,0 +1,35 @@
/*
# HarmonyDB provides database abstractions over SP-wide Postgres-compatible instance(s).
# Features
Rolling to secondary database servers on connection failure
Convenience features for Go + SQL
Prevention of SQL injection vulnerabilities
Monitors behavior via Prometheus stats and logging of errors.
# Usage
Processes should use New() to instantiate a *DB and keep it.
Consumers can use this *DB concurrently.
Creating and changing tables & views should happen in ./sql/ folder.
Name the file "today's date" in the format: YYYYMMDD.sql (ex: 20231231.sql for the year's last day)
a. CREATE TABLE should NOT have a schema:
GOOD: CREATE TABLE foo ();
BAD: CREATE TABLE me.foo ();
b. Schema is managed for you. It provides isolation for integraton tests & multi-use.
c. Git Merges: All run once, so old-after-new is OK when there are no deps.
d. NEVER change shipped sql files. Have later files make corrections.
e. Anything not ran will be ran, so an older date making it to master is OK.
Write SQL with context, raw strings, and args:
name := "Alice"
var ID int
err := QueryRow(ctx, "SELECT id from people where first_name=?", name).Scan(&ID)
fmt.Println(ID)
Note: Scan() is column-oriented, while Select() & StructScan() is field name/tag oriented.
*/
package harmonydb

297
lib/harmony/harmonydb/harmonydb.go Normal file
View File

@ -0,0 +1,297 @@
package harmonydb
import (
"context"
"embed"
"fmt"
"math/rand"
"net"
"regexp"
"sort"
"strconv"
"strings"
"time"
logging "github.com/ipfs/go-log/v2"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/node/config"
)
type ITestID string
// ItestNewID see ITestWithID doc
func ITestNewID() ITestID {
return ITestID(strconv.Itoa(rand.Intn(99999)))
}
type DB struct {
pgx *pgxpool.Pool
cfg *pgxpool.Config
schema string
hostnames []string
}
var logger = logging.Logger("harmonydb")
// NewFromConfig is a convenience function.
// In usage:
//
// db, err := NewFromConfig(config.HarmonyDB) // in binary init
func NewFromConfig(cfg config.HarmonyDB) (*DB, error) {
return New(
cfg.Hosts,
cfg.Username,
cfg.Password,
cfg.Database,
cfg.Port,
"",
)
}
func NewFromConfigWithITestID(cfg config.HarmonyDB) func(id ITestID) (*DB, error) {
return func(id ITestID) (*DB, error) {
return New(
cfg.Hosts,
cfg.Username,
cfg.Password,
cfg.Database,
cfg.Port,
id,
)
}
}
// New is to be called once per binary to establish the pool.
// log() is for errors. It returns an upgraded database's connection.
// This entry point serves both production and integration tests, so it's more DI.
func New(hosts []string, username, password, database, port string, itestID ITestID) (*DB, error) {
itest := string(itestID)
connString := ""
if len(hosts) > 0 {
connString = "host=" + hosts[0] + " "
}
for k, v := range map[string]string{"user": username, "password": password, "dbname": database, "port": port} {
if strings.TrimSpace(v) != "" {
connString += k + "=" + v + " "
}
}
schema := "lotus"
if itest != "" {
schema = "itest_" + itest
}
if err := ensureSchemaExists(connString, schema); err != nil {
return nil, err
}
cfg, err := pgxpool.ParseConfig(connString + "search_path=" + schema)
if err != nil {
return nil, err
}
// enable multiple fallback hosts.
for _, h := range hosts[1:] {
cfg.ConnConfig.Fallbacks = append(cfg.ConnConfig.Fallbacks, &pgconn.FallbackConfig{Host: h})
}
cfg.ConnConfig.OnNotice = func(conn *pgconn.PgConn, n *pgconn.Notice) {
logger.Debug("database notice: " + n.Message + ": " + n.Detail)
DBMeasures.Errors.M(1)
}
db := DB{cfg: cfg, schema: schema, hostnames: hosts} // pgx populated in AddStatsAndConnect
if err := db.addStatsAndConnect(); err != nil {
return nil, err
}
return &db, db.upgrade()
}
type tracer struct {
}
type ctxkey string
const SQL_START = ctxkey("sqlStart")
const SQL_STRING = ctxkey("sqlString")
func (t tracer) TraceQueryStart(ctx context.Context, conn *pgx.Conn, data pgx.TraceQueryStartData) context.Context {
return context.WithValue(context.WithValue(ctx, SQL_START, time.Now()), SQL_STRING, data.SQL)
}
func (t tracer) TraceQueryEnd(ctx context.Context, conn *pgx.Conn, data pgx.TraceQueryEndData) {
DBMeasures.Hits.M(1)
ms := time.Since(ctx.Value(SQL_START).(time.Time)).Milliseconds()
DBMeasures.TotalWait.M(ms)
DBMeasures.Waits.Observe(float64(ms))
if data.Err != nil {
DBMeasures.Errors.M(1)
}
logger.Debugw("SQL run",
"query", ctx.Value(SQL_STRING).(string),
"err", data.Err,
"rowCt", data.CommandTag.RowsAffected(),
"milliseconds", ms)
}
func (db *DB) GetRoutableIP() (string, error) {
tx, err := db.pgx.Begin(context.Background())
if err != nil {
return "", err
}
defer func() { _ = tx.Rollback(context.Background()) }()
local := tx.Conn().PgConn().Conn().LocalAddr()
addr, ok := local.(*net.TCPAddr)
if !ok {
return "", fmt.Errorf("could not get local addr from %v", addr)
}
return addr.IP.String(), nil
}
// addStatsAndConnect connects a prometheus logger. Be sure to run this before using the DB.
func (db *DB) addStatsAndConnect() error {
db.cfg.ConnConfig.Tracer = tracer{}
hostnameToIndex := map[string]float64{}
for i, h := range db.hostnames {
hostnameToIndex[h] = float64(i)
}
db.cfg.AfterConnect = func(ctx context.Context, c *pgx.Conn) error {
s := db.pgx.Stat()
DBMeasures.OpenConnections.M(int64(s.TotalConns()))
DBMeasures.WhichHost.Observe(hostnameToIndex[c.Config().Host])
//FUTURE place for any connection seasoning
return nil
}
// Timeout the first connection so we know if the DB is down.
ctx, ctxClose := context.WithDeadline(context.Background(), time.Now().Add(5*time.Second))
defer ctxClose()
var err error
db.pgx, err = pgxpool.NewWithConfig(ctx, db.cfg)
if err != nil {
logger.Error(fmt.Sprintf("Unable to connect to database: %v\n", err))
return err
}
return nil
}
// ITestDeleteAll will delete everything created for "this" integration test.
// This must be called at the end of each integration test.
func (db *DB) ITestDeleteAll() {
if !strings.HasPrefix(db.schema, "itest_") {
fmt.Println("Warning: this should never be called on anything but an itest schema.")
return
}
defer db.pgx.Close()
_, err := db.pgx.Exec(context.Background(), "DROP SCHEMA "+db.schema+" CASCADE")
if err != nil {
fmt.Println("warning: unclean itest shutdown: cannot delete schema: " + err.Error())
return
}
}
var schemaREString = "^[A-Za-z0-9_]+$"
var schemaRE = regexp.MustCompile(schemaREString)
func ensureSchemaExists(connString, schema string) error {
// FUTURE allow using fallback DBs for start-up.
ctx, cncl := context.WithDeadline(context.Background(), time.Now().Add(3*time.Second))
p, err := pgx.Connect(ctx, connString)
defer cncl()
if err != nil {
return xerrors.Errorf("unable to connect to db: %s, err: %v", connString, err)
}
defer func() { _ = p.Close(context.Background()) }()
if len(schema) < 5 || !schemaRE.MatchString(schema) {
return xerrors.New("schema must be of the form " + schemaREString + "\n Got: " + schema)
}
_, err = p.Exec(context.Background(), "CREATE SCHEMA IF NOT EXISTS "+schema)
if err != nil {
return xerrors.Errorf("cannot create schema: %w", err)
}
return nil
}
//go:embed sql
var fs embed.FS
func (db *DB) upgrade() error {
// Does the version table exist? if not, make it.
// NOTE: This cannot change except via the next sql file.
_, err := db.Exec(context.Background(), `CREATE TABLE IF NOT EXISTS base (
id SERIAL PRIMARY KEY,
entry CHAR(12),
applied TIMESTAMP DEFAULT current_timestamp
)`)
if err != nil {
logger.Error("Upgrade failed.")
return xerrors.Errorf("Cannot create base table %w", err)
}
// __Run scripts in order.__
landed := map[string]bool{}
{
var landedEntries []struct{ Entry string }
err = db.Select(context.Background(), &landedEntries, "SELECT entry FROM base")
if err != nil {
logger.Error("Cannot read entries: " + err.Error())
return xerrors.Errorf("cannot read entries: %w", err)
}
for _, l := range landedEntries {
landed[l.Entry[:8]] = true
}
}
dir, err := fs.ReadDir("sql")
if err != nil {
logger.Error("Cannot read fs entries: " + err.Error())
return err
}
sort.Slice(dir, func(i, j int) bool { return dir[i].Name() < dir[j].Name() })
if len(dir) == 0 {
logger.Error("No sql files found.")
}
for _, e := range dir {
name := e.Name()
if !strings.HasSuffix(name, ".sql") {
logger.Debug("Must have only SQL files here, found: " + name)
continue
}
if landed[name[:8]] {
logger.Debug("DB Schema " + name + " already applied.")
continue
}
file, err := fs.ReadFile("sql/" + name)
if err != nil {
logger.Error("weird embed file read err")
return err
}
for _, s := range strings.Split(string(file), ";") { // Implement the changes.
if len(strings.TrimSpace(s)) == 0 {
continue
}
_, err = db.pgx.Exec(context.Background(), s)
if err != nil {
msg := fmt.Sprintf("Could not upgrade! File %s, Query: %s, Returned: %s", name, s, err.Error())
logger.Error(msg)
return xerrors.New(msg) // makes devs lives easier by placing message at the end.
}
}
// Mark Completed.
_, err = db.Exec(context.Background(), "INSERT INTO base (entry) VALUES ($1)", name[:8])
if err != nil {
logger.Error("Cannot update base: " + err.Error())
return xerrors.Errorf("cannot insert into base: %w", err)
}
}
return nil
}

77
lib/harmony/harmonydb/metrics.go Normal file
View File

@ -0,0 +1,77 @@
package harmonydb
import (
"github.com/prometheus/client_golang/prometheus"
"go.opencensus.io/stats"
"go.opencensus.io/stats/view"
"go.opencensus.io/tag"
"github.com/filecoin-project/lotus/metrics"
)
var (
dbTag, _ = tag.NewKey("db_name")
pre = "harmonydb_base_"
waitsBuckets = []float64{0, 10, 20, 30, 50, 80, 130, 210, 340, 550, 890}
whichHostBuckets = []float64{0, 1, 2, 3, 4, 5}
)
// DBMeasures groups all db metrics.
var DBMeasures = struct {
Hits *stats.Int64Measure
TotalWait *stats.Int64Measure
Waits prometheus.Histogram
OpenConnections *stats.Int64Measure
Errors *stats.Int64Measure
WhichHost prometheus.Histogram
}{
Hits: stats.Int64(pre+"hits", "Total number of uses.", stats.UnitDimensionless),
TotalWait: stats.Int64(pre+"total_wait", "Total delay. A numerator over hits to get average wait.", stats.UnitMilliseconds),
Waits: prometheus.NewHistogram(prometheus.HistogramOpts{
Name: pre + "waits",
Buckets: waitsBuckets,
Help: "The histogram of waits for query completions.",
}),
OpenConnections: stats.Int64(pre+"open_connections", "Total connection count.", stats.UnitDimensionless),
Errors: stats.Int64(pre+"errors", "Total error count.", stats.UnitDimensionless),
WhichHost: prometheus.NewHistogram(prometheus.HistogramOpts{
Name: pre + "which_host",
Buckets: whichHostBuckets,
Help: "The index of the hostname being used",
}),
}
// CacheViews groups all cache-related default views.
func init() {
metrics.RegisterViews(
&view.View{
Measure: DBMeasures.Hits,
Aggregation: view.Sum(),
TagKeys: []tag.Key{dbTag},
},
&view.View{
Measure: DBMeasures.TotalWait,
Aggregation: view.Sum(),
TagKeys: []tag.Key{dbTag},
},
&view.View{
Measure: DBMeasures.OpenConnections,
Aggregation: view.LastValue(),
TagKeys: []tag.Key{dbTag},
},
&view.View{
Measure: DBMeasures.Errors,
Aggregation: view.Sum(),
TagKeys: []tag.Key{dbTag},
},
)
err := prometheus.Register(DBMeasures.Waits)
if err != nil {
panic(err)
}
err = prometheus.Register(DBMeasures.WhichHost)
if err != nil {
panic(err)
}
}

7
lib/harmony/harmonydb/sql/20230706.sql Normal file
View File

@ -0,0 +1,7 @@
CREATE TABLE itest_scratch (
id SERIAL PRIMARY KEY,
content TEXT,
some_int INTEGER,
second_int INTEGER,
update_time TIMESTAMP DEFAULT current_timestamp
)

45
lib/harmony/harmonydb/sql/20230712.sql Normal file
View File

@ -0,0 +1,45 @@
create table sector_location
(
miner_id bigint not null,
sector_num bigint not null,
sector_filetype int not null,
storage_id varchar not null,
is_primary bool,
read_ts timestamp(6),
read_refs int,
write_ts timestamp(6),
write_lock_owner varchar,
constraint sectorlocation_pk
primary key (miner_id, sector_num, sector_filetype, storage_id)
);
alter table sector_location
alter column read_refs set not null;
alter table sector_location
alter column read_refs set default 0;
create table storage_path
(
"storage_id" varchar not null
constraint "storage_path_pkey"
primary key,
"urls" varchar, -- comma separated list of urls
"weight" bigint,
"max_storage" bigint,
"can_seal" bool,
"can_store" bool,
"groups" varchar, -- comma separated list of group names
"allow_to" varchar, -- comma separated list of allowed groups
"allow_types" varchar, -- comma separated list of allowed file types
"deny_types" varchar, -- comma separated list of denied file types
"capacity" bigint,
"available" bigint,
"fs_available" bigint,
"reserved" bigint,
"used" bigint,
"last_heartbeat" timestamp(6),
"heartbeat_err" varchar
);

51
lib/harmony/harmonydb/sql/20230719.sql Normal file
View File

@ -0,0 +1,51 @@
/* For HarmonyTask base implementation. */
CREATE TABLE harmony_machines (
id SERIAL PRIMARY KEY NOT NULL,
last_contact TIMESTAMP NOT NULL DEFAULT current_timestamp,
host_and_port varchar(300) NOT NULL,
cpu INTEGER NOT NULL,
ram BIGINT NOT NULL,
gpu FLOAT NOT NULL
);
CREATE TABLE harmony_task (
id SERIAL PRIMARY KEY NOT NULL,
initiated_by INTEGER,
update_time TIMESTAMP NOT NULL DEFAULT current_timestamp,
posted_time TIMESTAMP NOT NULL,
owner_id INTEGER REFERENCES harmony_machines (id) ON DELETE SET NULL,
added_by INTEGER NOT NULL,
previous_task INTEGER,
name varchar(16) NOT NULL
);
COMMENT ON COLUMN harmony_task.initiated_by IS 'The task ID whose completion occasioned this task.';
COMMENT ON COLUMN harmony_task.owner_id IS 'The foreign key to harmony_machines.';
COMMENT ON COLUMN harmony_task.name IS 'The name of the task type.';
COMMENT ON COLUMN harmony_task.owner_id IS 'may be null if between owners or not yet taken';
COMMENT ON COLUMN harmony_task.update_time IS 'When it was last modified. not a heartbeat';
CREATE TABLE harmony_task_history (
id SERIAL PRIMARY KEY NOT NULL,
task_id INTEGER NOT NULL,
name VARCHAR(16) NOT NULL,
posted TIMESTAMP NOT NULL,
work_start TIMESTAMP NOT NULL,
work_end TIMESTAMP NOT NULL,
result BOOLEAN NOT NULL,
err varchar
);
COMMENT ON COLUMN harmony_task_history.result IS 'Use to detemine if this was a successful run.';
CREATE TABLE harmony_task_follow (
id SERIAL PRIMARY KEY NOT NULL,
owner_id INTEGER NOT NULL REFERENCES harmony_machines (id) ON DELETE CASCADE,
to_type VARCHAR(16) NOT NULL,
from_type VARCHAR(16) NOT NULL
);
CREATE TABLE harmony_task_impl (
id SERIAL PRIMARY KEY NOT NULL,
owner_id INTEGER NOT NULL REFERENCES harmony_machines (id) ON DELETE CASCADE,
name VARCHAR(16) NOT NULL
);

48
lib/harmony/harmonydb/sql/20230823.sql Normal file
View File

@ -0,0 +1,48 @@
create table wdpost_partition_tasks
(
task_id bigint not null
constraint wdpost_partition_tasks_pk
primary key,
sp_id bigint not null,
proving_period_start bigint not null,
deadline_index bigint not null,
partition_index bigint not null,
constraint wdpost_partition_tasks_identity_key
unique (sp_id, proving_period_start, deadline_index, partition_index)
);
comment on column wdpost_partition_tasks.task_id is 'harmonytask task ID';
comment on column wdpost_partition_tasks.sp_id is 'storage provider ID';
comment on column wdpost_partition_tasks.proving_period_start is 'proving period start';
comment on column wdpost_partition_tasks.deadline_index is 'deadline index within the proving period';
comment on column wdpost_partition_tasks.partition_index is 'partition index within the deadline';
create table wdpost_proofs
(
sp_id bigint not null,
proving_period_start bigint not null,
deadline bigint not null,
partition bigint not null,
submit_at_epoch bigint not null,
submit_by_epoch bigint not null,
proof_params bytea,
submit_task_id bigint,
message_cid text,
constraint wdpost_proofs_identity_key
unique (sp_id, proving_period_start, deadline, partition)
);
create table wdpost_recovery_tasks
(
task_id bigint not null
constraint wdpost_recovery_tasks_pk
primary key,
sp_id bigint not null,
proving_period_start bigint not null,
deadline_index bigint not null,
partition_index bigint not null,
constraint wdpost_recovery_tasks_identity_key
unique (sp_id, proving_period_start, deadline_index, partition_index)
);

5
lib/harmony/harmonydb/sql/20230919.sql Normal file
View File

@ -0,0 +1,5 @@
CREATE TABLE harmony_config (
id SERIAL PRIMARY KEY NOT NULL,
title VARCHAR(300) UNIQUE NOT NULL,
config TEXT NOT NULL
);

55
lib/harmony/harmonydb/sql/20231103.sql Normal file
View File

@ -0,0 +1,55 @@
create table message_sends
(
from_key text not null,
to_addr text not null,
send_reason text not null,
send_task_id bigint not null,
unsigned_data bytea not null,
unsigned_cid text not null,
nonce bigint,
signed_data bytea,
signed_json jsonb,
signed_cid text,
send_time timestamp default null,
send_success boolean default null,
send_error text,
constraint message_sends_pk
primary key (send_task_id, from_key)
);
comment on column message_sends.from_key is 'text f[1/3/4]... address';
comment on column message_sends.to_addr is 'text f[0/1/2/3/4]... address';
comment on column message_sends.send_reason is 'optional description of send reason';
comment on column message_sends.send_task_id is 'harmony task id of the send task';
comment on column message_sends.unsigned_data is 'unsigned message data';
comment on column message_sends.unsigned_cid is 'unsigned message cid';
comment on column message_sends.nonce is 'assigned message nonce, set while the send task is executing';
comment on column message_sends.signed_data is 'signed message data, set while the send task is executing';
comment on column message_sends.signed_cid is 'signed message cid, set while the send task is executing';
comment on column message_sends.send_time is 'time when the send task was executed, set after pushing the message to the network';
comment on column message_sends.send_success is 'whether this message was broadcasted to the network already, null if not yet attempted, true if successful, false if failed';
comment on column message_sends.send_error is 'error message if send_success is false';
create unique index message_sends_success_index
on message_sends (from_key, nonce)
where send_success is not false;
comment on index message_sends_success_index is
'message_sends_success_index enforces sender/nonce uniqueness, it is a conditional index that only indexes rows where send_success is not false. This allows us to have multiple rows with the same sender/nonce, as long as only one of them was successfully broadcasted (true) to the network or is in the process of being broadcasted (null).';
create table message_send_locks
(
from_key text not null,
task_id bigint not null,
claimed_at timestamp not null,
constraint message_send_locks_pk
primary key (from_key)
);

39
lib/harmony/harmonydb/sql/20231110.sql Normal file
View File

@ -0,0 +1,39 @@
create table mining_tasks
(
task_id bigint not null
constraint mining_tasks_pk
primary key,
sp_id bigint not null,
epoch bigint not null,
base_compute_time timestamp not null,
won bool not null default false,
mined_cid text,
mined_header jsonb,
mined_at timestamp,
submitted_at timestamp,
constraint mining_tasks_sp_epoch
unique (sp_id, epoch)
);
create table mining_base_block
(
id bigserial not null
constraint mining_base_block_pk
primary key,
task_id bigint not null
constraint mining_base_block_mining_tasks_task_id_fk
references mining_tasks
on delete cascade,
sp_id bigint,
block_cid text not null,
no_win bool not null default false,
constraint mining_base_block_pk2
unique (sp_id, task_id, block_cid)
);
CREATE UNIQUE INDEX mining_base_block_cid_k ON mining_base_block (sp_id, block_cid) WHERE no_win = false;

1
lib/harmony/harmonydb/sql/20231113.sql Normal file
View File

@ -0,0 +1 @@
ALTER TABLE harmony_task_history ADD COLUMN completed_by_host_and_port varchar(300) NOT NULL;

8
lib/harmony/harmonydb/sql/20231120-testing1.sql Normal file
View File

@ -0,0 +1,8 @@
CREATE TABLE harmony_test (
task_id bigint
constraint harmony_test_pk
primary key,
options text,
result text
);
ALTER TABLE wdpost_proofs ADD COLUMN test_task_id bigint;

158
lib/harmony/harmonydb/userfuncs.go Normal file
View File

@ -0,0 +1,158 @@
package harmonydb
import (
"context"
"errors"
"github.com/georgysavva/scany/v2/pgxscan"
"github.com/jackc/pgerrcode"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
)
// rawStringOnly is _intentionally_private_ to force only basic strings in SQL queries.
// In any package, raw strings will satisfy compilation. Ex:
//
// harmonydb.Exec("INSERT INTO version (number) VALUES (1)")
//
// This prevents SQL injection attacks where the input contains query fragments.
type rawStringOnly string
// Exec executes changes (INSERT, DELETE, or UPDATE).
// Note, for CREATE & DROP please keep these permanent and express
// them in the ./sql/ files (next number).
func (db *DB) Exec(ctx context.Context, sql rawStringOnly, arguments ...any) (count int, err error) {
res, err := db.pgx.Exec(ctx, string(sql), arguments...)
return int(res.RowsAffected()), err
}
type Qry interface {
Next() bool
Err() error
Close()
Scan(...any) error
Values() ([]any, error)
}
// Query offers Next/Err/Close/Scan/Values/StructScan
type Query struct {
Qry
}
// Query allows iterating returned values to save memory consumption
// with the downside of needing to `defer q.Close()`. For a simpler interface,
// try Select()
// Next() must be called to advance the row cursor, including the first time:
// Ex:
// q, err := db.Query(ctx, "SELECT id, name FROM users")
// handleError(err)
// defer q.Close()
//
// for q.Next() {
// var id int
// var name string
// handleError(q.Scan(&id, &name))
// fmt.Println(id, name)
// }
func (db *DB) Query(ctx context.Context, sql rawStringOnly, arguments ...any) (*Query, error) {
q, err := db.pgx.Query(ctx, string(sql), arguments...)
return &Query{q}, err
}
func (q *Query) StructScan(s any) error {
return pgxscan.ScanRow(s, q.Qry.(pgx.Rows))
}
type Row interface {
Scan(...any) error
}
// QueryRow gets 1 row using column order matching.
// This is a timesaver for the special case of wanting the first row returned only.
// EX:
//
// var name, pet string
// var ID = 123
// err := db.QueryRow(ctx, "SELECT name, pet FROM users WHERE ID=?", ID).Scan(&name, &pet)
func (db *DB) QueryRow(ctx context.Context, sql rawStringOnly, arguments ...any) Row {
return db.pgx.QueryRow(ctx, string(sql), arguments...)
}
/*
Select multiple rows into a slice using name matching
Ex:
type user struct {
Name string
ID int
Number string `db:"tel_no"`
}
var users []user
pet := "cat"
err := db.Select(ctx, &users, "SELECT name, id, tel_no FROM customers WHERE pet=?", pet)
*/
func (db *DB) Select(ctx context.Context, sliceOfStructPtr any, sql rawStringOnly, arguments ...any) error {
return pgxscan.Select(ctx, db.pgx, sliceOfStructPtr, string(sql), arguments...)
}
type Tx struct {
pgx.Tx
ctx context.Context
}
// BeginTransaction is how you can access transactions using this library.
// The entire transaction happens in the function passed in.
// The return must be true or a rollback will occur.
func (db *DB) BeginTransaction(ctx context.Context, f func(*Tx) (commit bool, err error)) (didCommit bool, retErr error) {
tx, err := db.pgx.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return false, err
}
var commit bool
defer func() { // Panic clean-up.
if !commit {
if tmp := tx.Rollback(ctx); tmp != nil {
retErr = tmp
}
}
}()
commit, err = f(&Tx{tx, ctx})
if err != nil {
return false, err
}
if commit {
err = tx.Commit(ctx)
if err != nil {
return false, err
}
return true, nil
}
return false, nil
}
// Exec in a transaction.
func (t *Tx) Exec(sql rawStringOnly, arguments ...any) (count int, err error) {
res, err := t.Tx.Exec(t.ctx, string(sql), arguments...)
return int(res.RowsAffected()), err
}
// Query in a transaction.
func (t *Tx) Query(sql rawStringOnly, arguments ...any) (*Query, error) {
q, err := t.Tx.Query(t.ctx, string(sql), arguments...)
return &Query{q}, err
}
// QueryRow in a transaction.
func (t *Tx) QueryRow(sql rawStringOnly, arguments ...any) Row {
return t.Tx.QueryRow(t.ctx, string(sql), arguments...)
}
// Select in a transaction.
func (t *Tx) Select(sliceOfStructPtr any, sql rawStringOnly, arguments ...any) error {
return pgxscan.Select(t.ctx, t.Tx, sliceOfStructPtr, string(sql), arguments...)
}
func IsErrUniqueContraint(err error) bool {
var e2 *pgconn.PgError
return errors.As(err, &e2) && e2.Code == pgerrcode.UniqueViolation
}

71
lib/harmony/harmonytask/doc.go Normal file
View File

@ -0,0 +1,71 @@
/*
Package harmonytask implements a pure (no task logic), distributed
task manager. This clean interface allows a task implementer to completely
avoid being concerned with task scheduling and management.
It's based on the idea of tasks as small units of work broken from other
work by hardware, parallelizabilty, reliability, or any other reason.
Workers will be Greedy: vaccuuming up their favorite jobs from a list.
Once 1 task is accepted, harmonydb tries to get other task runner
machines to accept work (round robin) before trying again to accept.
*
Mental Model:
Things that block tasks:
- task not registered for any running server
- max was specified and reached
- resource exhaustion
- CanAccept() interface (per-task implmentation) does not accept it.
Ways tasks start:
- DB Read every 3 seconds
- Task was added (to db) by this process
Ways tasks get added:
- Async Listener task (for chain, etc)
- Followers: Tasks get added because another task completed
When Follower collectors run:
- If both sides are process-local, then this process will pick it up.
- If properly registered already, the http endpoint will be tried to start it.
- Otherwise, at the listen interval during db scrape it will be found.
How duplicate tasks are avoided:
- that's up to the task definition, but probably a unique key
*
To use:
1.Implement TaskInterface for a new task.
2. Have New() receive this & all other ACTIVE implementations.
*
*
As we are not expecting DBAs in this database, it's important to know
what grows uncontrolled. The only growing harmony_* table is
harmony_task_history (somewhat quickly). These will need a
clean-up for after the task data could never be acted upon.
but the design **requires** extraInfo tables to grow until the task's
info could not possibly be used by a following task, including slow
release rollout. This would normally be in the order of months old.
*
Other possible enhancements include more collaborative coordination
to assign a task to machines closer to the data.
__Database_Behavior__
harmony_task is the list of work that has not been completed.
AddTaskFunc manages the additions, but is designed to have its
transactions failed-out on overlap with a similar task already written.
It's up to the TaskInterface implementer to discover this overlap via
some other table it uses (since overlap can mean very different things).
harmony_task_history
This holds transactions that completed or saw too many retries. It also
serves as input for subsequent (follower) tasks to kick off. This is not
done machine-internally because a follower may not be on the same machine
as the previous task.
harmony_task_machines
Managed by lib/harmony/resources, this is a reference to machines registered
via the resources. This registration does not obligate the machine to
anything, but serves as a discovery mechanism. Paths are hostnames + ports
which are presumed to support http, but this assumption is only used by
the task system.
*/
package harmonytask

302
lib/harmony/harmonytask/harmonytask.go Normal file
View File

@ -0,0 +1,302 @@
package harmonytask
import (
"context"
"fmt"
"strconv"
"sync/atomic"
"time"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/lib/harmony/resources"
)
// Consts (except for unit test)
var POLL_DURATION = time.Second * 3 // Poll for Work this frequently
var CLEANUP_FREQUENCY = 5 * time.Minute // Check for dead workers this often * everyone
var FOLLOW_FREQUENCY = 1 * time.Minute // Check for work to follow this often
type TaskTypeDetails struct {
// Max returns how many tasks this machine can run of this type.
// Zero (default) or less means unrestricted.
Max int
// Name is the task name to be added to the task list.
Name string
// Peak costs to Do() the task.
Cost resources.Resources
// Max Failure count before the job is dropped.
// 0 = retry forever
MaxFailures uint
// Follow another task's completion via this task's creation.
// The function should populate extraInfo from data
// available from the previous task's tables, using the given TaskID.
// It should also return success if the trigger succeeded.
// NOTE: if refatoring tasks, see if your task is
// necessary. Ex: Is the sector state correct for your stage to run?
Follows map[string]func(TaskID, AddTaskFunc) (bool, error)
}
// TaskInterface must be implemented in order to have a task used by harmonytask.
type TaskInterface interface {
// Do the task assigned. Call stillOwned before making single-writer-only
// changes to ensure the work has not been stolen.
// This is the ONLY function that should attempt to do the work, and must
// ONLY be called by harmonytask.
// Indicate if the task no-longer needs scheduling with done=true including
// cases where it's past the deadline.
Do(taskID TaskID, stillOwned func() bool) (done bool, err error)
// CanAccept should return if the task can run on this machine. It should
// return null if the task type is not allowed on this machine.
// It should select the task it most wants to accomplish.
// It is also responsible for determining & reserving disk space (including scratch).
CanAccept([]TaskID, *TaskEngine) (*TaskID, error)
// TypeDetails() returns static details about how this task behaves and
// how this machine will run it. Read once at the beginning.
TypeDetails() TaskTypeDetails
// This listener will consume all external sources continuously for work.
// Do() may also be called from a backlog of work. This must not
// start doing the work (it still must be scheduled).
// Note: Task de-duplication should happen in ExtraInfoFunc by
// returning false, typically by determining from the tx that the work
// exists already. The easy way is to have a unique joint index
// across all fields that will be common.
// Adder should typically only add its own task type, but multiple
// is possible for when 1 trigger starts 2 things.
// Usage Example:
// func (b *BazType)Adder(addTask AddTaskFunc) {
// for {
// bazMaker := <- bazChannel
// addTask("baz", func(t harmonytask.TaskID, txn db.Transaction) (bool, error) {
// _, err := txn.Exec(`INSERT INTO bazInfoTable (taskID, qix, mot)
// VALUES ($1,$2,$3)`, id, bazMaker.qix, bazMaker.mot)
// if err != nil {
// scream(err)
// return false
// }
// return true
// })
// }
// }
Adder(AddTaskFunc)
}
// AddTaskFunc is responsible for adding a task's details "extra info" to the DB.
// It should return true if the task should be added, false if it was already there.
// This is typically accomplished with a "unique" index on your detals table that
// would cause the insert to fail.
// The error indicates that instead of a conflict (which we should ignore) that we
// actually have a serious problem that needs to be logged with context.
type AddTaskFunc func(extraInfo func(TaskID, *harmonydb.Tx) (shouldCommit bool, seriousError error))
type TaskEngine struct {
ctx context.Context
handlers []*taskTypeHandler
db *harmonydb.DB
reg *resources.Reg
grace context.CancelFunc
taskMap map[string]*taskTypeHandler
ownerID int
follows map[string][]followStruct
lastFollowTime time.Time
lastCleanup atomic.Value
hostAndPort string
}
type followStruct struct {
f func(TaskID, AddTaskFunc) (bool, error)
h *taskTypeHandler
name string
}
type TaskID int
// New creates all the task definitions. Note that TaskEngine
// knows nothing about the tasks themselves and serves to be a
// generic container for common work
func New(
db *harmonydb.DB,
impls []TaskInterface,
hostnameAndPort string) (*TaskEngine, error) {
reg, err := resources.Register(db, hostnameAndPort)
if err != nil {
return nil, fmt.Errorf("cannot get resources: %w", err)
}
ctx, grace := context.WithCancel(context.Background())
e := &TaskEngine{
ctx: ctx,
grace: grace,
db: db,
reg: reg,
ownerID: reg.Resources.MachineID, // The current number representing "hostAndPort"
taskMap: make(map[string]*taskTypeHandler, len(impls)),
follows: make(map[string][]followStruct),
hostAndPort: hostnameAndPort,
}
e.lastCleanup.Store(time.Now())
for _, c := range impls {
h := taskTypeHandler{
TaskInterface: c,
TaskTypeDetails: c.TypeDetails(),
TaskEngine: e,
}
e.handlers = append(e.handlers, &h)
e.taskMap[h.TaskTypeDetails.Name] = &h
}
// resurrect old work
{
var taskRet []struct {
ID int
Name string
}
err := db.Select(e.ctx, &taskRet, `SELECT id, name from harmony_task WHERE owner_id=$1`, e.ownerID)
if err != nil {
return nil, err
}
for _, w := range taskRet {
// edge-case: if old assignments are not available tasks, unlock them.
h := e.taskMap[w.Name]
if h == nil {
_, err := db.Exec(e.ctx, `UPDATE harmony_task SET owner=NULL WHERE id=$1`, w.ID)
if err != nil {
log.Errorw("Cannot remove self from owner field", "error", err)
continue // not really fatal, but not great
}
}
if !h.considerWork("recovered", []TaskID{TaskID(w.ID)}) {
log.Error("Strange: Unable to accept previously owned task: ", w.ID, w.Name)
}
}
}
for _, h := range e.handlers {
go h.Adder(h.AddTask)
}
go e.poller()
return e, nil
}
// GracefullyTerminate hangs until all present tasks have completed.
// Call this to cleanly exit the process. As some processes are long-running,
// passing a deadline will ignore those still running (to be picked-up later).
func (e *TaskEngine) GracefullyTerminate(deadline time.Duration) {
e.grace()
e.reg.Shutdown()
deadlineChan := time.NewTimer(deadline).C
top:
for _, h := range e.handlers {
if h.Count.Load() > 0 {
select {
case <-deadlineChan:
return
default:
time.Sleep(time.Millisecond)
goto top
}
}
}
}
func (e *TaskEngine) poller() {
for {
select {
case <-time.NewTicker(POLL_DURATION).C: // Find work periodically
case <-e.ctx.Done(): ///////////////////// Graceful exit
return
}
e.pollerTryAllWork()
if time.Since(e.lastFollowTime) > FOLLOW_FREQUENCY {
e.followWorkInDB()
}
}
}
// followWorkInDB implements "Follows"
func (e *TaskEngine) followWorkInDB() {
// Step 1: What are we following?
var lastFollowTime time.Time
lastFollowTime, e.lastFollowTime = e.lastFollowTime, time.Now()
for fromName, srcs := range e.follows {
var cList []int // Which work is done (that we follow) since we last checked?
err := e.db.Select(e.ctx, &cList, `SELECT h.task_id FROM harmony_task_history
WHERE h.work_end>$1 AND h.name=$2`, lastFollowTime, fromName)
if err != nil {
log.Error("Could not query DB: ", err)
return
}
for _, src := range srcs {
for _, workAlreadyDone := range cList { // Were any tasks made to follow these tasks?
var ct int
err := e.db.QueryRow(e.ctx, `SELECT COUNT(*) FROM harmony_task
WHERE name=$1 AND previous_task=$2`, src.h.Name, workAlreadyDone).Scan(&ct)
if err != nil {
log.Error("Could not query harmony_task: ", err)
return // not recoverable here
}
if ct > 0 {
continue
}
// we need to create this task
b, err := src.h.Follows[fromName](TaskID(workAlreadyDone), src.h.AddTask)
if err != nil {
log.Errorw("Could not follow: ", "error", err)
continue
}
if !b {
// But someone may have beaten us to it.
log.Debugf("Unable to add task %s following Task(%d, %s)", src.h.Name, workAlreadyDone, fromName)
}
}
}
}
}
// pollerTryAllWork starts the next 1 task
func (e *TaskEngine) pollerTryAllWork() {
if time.Since(e.lastCleanup.Load().(time.Time)) > CLEANUP_FREQUENCY {
e.lastCleanup.Store(time.Now())
resources.CleanupMachines(e.ctx, e.db)
}
for _, v := range e.handlers {
if v.AssertMachineHasCapacity() != nil {
continue
}
var unownedTasks []TaskID
err := e.db.Select(e.ctx, &unownedTasks, `SELECT id
FROM harmony_task
WHERE owner_id IS NULL AND name=$1
ORDER BY update_time`, v.Name)
if err != nil {
log.Error("Unable to read work ", err)
continue
}
if len(unownedTasks) > 0 {
accepted := v.considerWork("poller", unownedTasks)
if accepted {
return // accept new work slowly and in priority order
}
log.Warn("Work not accepted for " + strconv.Itoa(len(unownedTasks)) + " " + v.Name + " task(s)")
}
}
}
// ResourcesAvailable determines what resources are still unassigned.
func (e *TaskEngine) ResourcesAvailable() resources.Resources {
tmp := e.reg.Resources
for _, t := range e.handlers {
ct := t.Count.Load()
tmp.Cpu -= int(ct) * t.Cost.Cpu
tmp.Gpu -= float64(ct) * t.Cost.Gpu
tmp.Ram -= uint64(ct) * t.Cost.Ram
}
return tmp
}

230
lib/harmony/harmonytask/task_type_handler.go Normal file
View File

@ -0,0 +1,230 @@
package harmonytask
import (
"context"
"errors"
"fmt"
"runtime"
"strconv"
"sync/atomic"
"time"
logging "github.com/ipfs/go-log/v2"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
)
var log = logging.Logger("harmonytask")
type taskTypeHandler struct {
TaskInterface
TaskTypeDetails
TaskEngine *TaskEngine
Count atomic.Int32
}
func (h *taskTypeHandler) AddTask(extra func(TaskID, *harmonydb.Tx) (bool, error)) {
var tID TaskID
_, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) (bool, error) {
// create taskID (from DB)
_, err := tx.Exec(`INSERT INTO harmony_task (name, added_by, posted_time)
VALUES ($1, $2, CURRENT_TIMESTAMP) `, h.Name, h.TaskEngine.ownerID)
if err != nil {
return false, fmt.Errorf("could not insert into harmonyTask: %w", err)
}
err = tx.QueryRow("SELECT id FROM harmony_task ORDER BY update_time DESC LIMIT 1").Scan(&tID)
if err != nil {
return false, fmt.Errorf("Could not select ID: %v", err)
}
return extra(tID, tx)
})
if err != nil {
if harmonydb.IsErrUniqueContraint(err) {
log.Debugf("addtask(%s) saw unique constraint, so it's added already.", h.Name)
return
}
log.Error("Could not add task. AddTasFunc failed: %v", err)
return
}
}
// considerWork is called to attempt to start work on a task-id of this task type.
// It presumes single-threaded calling, so there should not be a multi-threaded re-entry.
// The only caller should be the one work poller thread. This does spin off other threads,
// but those should not considerWork. Work completing may lower the resource numbers
// unexpectedly, but that will not invalidate work being already able to fit.
func (h *taskTypeHandler) considerWork(from string, ids []TaskID) (workAccepted bool) {
top:
if len(ids) == 0 {
return true // stop looking for takers
}
// 1. Can we do any more of this task type?
// NOTE: 0 is the default value, so this way people don't need to worry about
// this setting unless they want to limit the number of tasks of this type.
if h.Max > 0 && int(h.Count.Load()) >= h.Max {
log.Debugw("did not accept task", "name", h.Name, "reason", "at max already")
return false
}
// 2. Can we do any more work? From here onward, we presume the resource
// story will not change, so single-threaded calling is best.
err := h.AssertMachineHasCapacity()
if err != nil {
log.Debugw("did not accept task", "name", h.Name, "reason", "at capacity already: "+err.Error())
return false
}
// 3. What does the impl say?
tID, err := h.CanAccept(ids, h.TaskEngine)
if err != nil {
log.Error(err)
return false
}
if tID == nil {
log.Infow("did not accept task", "task_id", ids[0], "reason", "CanAccept() refused", "name", h.Name)
return false
}
// 4. Can we claim the work for our hostname?
ct, err := h.TaskEngine.db.Exec(h.TaskEngine.ctx, "UPDATE harmony_task SET owner_id=$1 WHERE id=$2 AND owner_id IS NULL", h.TaskEngine.ownerID, *tID)
if err != nil {
log.Error(err)
return false
}
if ct == 0 {
log.Infow("did not accept task", "task_id", strconv.Itoa(int(*tID)), "reason", "already Taken", "name", h.Name)
var tryAgain = make([]TaskID, 0, len(ids)-1)
for _, id := range ids {
if id != *tID {
tryAgain = append(tryAgain, id)
}
}
ids = tryAgain
goto top
}
h.Count.Add(1)
go func() {
log.Infow("Beginning work on Task", "id", *tID, "from", from, "name", h.Name)
var done bool
var doErr error
workStart := time.Now()
defer func() {
if r := recover(); r != nil {
stackSlice := make([]byte, 4092)
sz := runtime.Stack(stackSlice, false)
log.Error("Recovered from a serious error "+
"while processing "+h.Name+" task "+strconv.Itoa(int(*tID))+": ", r,
" Stack: ", string(stackSlice[:sz]))
}
h.Count.Add(-1)
h.recordCompletion(*tID, workStart, done, doErr)
if done {
for _, fs := range h.TaskEngine.follows[h.Name] { // Do we know of any follows for this task type?
if _, err := fs.f(*tID, fs.h.AddTask); err != nil {
log.Error("Could not follow", "error", err, "from", h.Name, "to", fs.name)
}
}
}
}()
done, doErr = h.Do(*tID, func() bool {
var owner int
// Background here because we don't want GracefulRestart to block this save.
err := h.TaskEngine.db.QueryRow(context.Background(),
`SELECT owner_id FROM harmony_task WHERE id=$1`, *tID).Scan(&owner)
if err != nil {
log.Error("Cannot determine ownership: ", err)
return false
}
return owner == h.TaskEngine.ownerID
})
if doErr != nil {
log.Errorw("Do() returned error", "type", h.Name, "id", strconv.Itoa(int(*tID)), "error", doErr)
}
}()
return true
}
func (h *taskTypeHandler) recordCompletion(tID TaskID, workStart time.Time, done bool, doErr error) {
workEnd := time.Now()
cm, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) (bool, error) {
var postedTime time.Time
err := tx.QueryRow(`SELECT posted_time FROM harmony_task WHERE id=$1`, tID).Scan(&postedTime)
if err != nil {
return false, fmt.Errorf("could not log completion: %w ", err)
}
result := "unspecified error"
if done {
_, err = tx.Exec("DELETE FROM harmony_task WHERE id=$1", tID)
if err != nil {
return false, fmt.Errorf("could not log completion: %w", err)
}
result = ""
} else {
if doErr != nil {
result = "error: " + doErr.Error()
}
var deleteTask bool
if h.MaxFailures > 0 {
ct := uint(0)
err = tx.QueryRow(`SELECT count(*) FROM harmony_task_history
WHERE task_id=$1 AND result=FALSE`, tID).Scan(&ct)
if err != nil {
return false, fmt.Errorf("could not read task history: %w", err)
}
if ct >= h.MaxFailures {
deleteTask = true
}
}
if deleteTask {
_, err = tx.Exec("DELETE FROM harmony_task WHERE id=$1", tID)
if err != nil {
return false, fmt.Errorf("could not delete failed job: %w", err)
}
// Note: Extra Info is left laying around for later review & clean-up
} else {
_, err := tx.Exec(`UPDATE harmony_task SET owner_id=NULL WHERE id=$1`, tID)
if err != nil {
return false, fmt.Errorf("could not disown failed task: %v %v", tID, err)
}
}
}
_, err = tx.Exec(`INSERT INTO harmony_task_history
(task_id, name, posted, work_start, work_end, result, completed_by_host_and_port, err)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, tID, h.Name, postedTime, workStart, workEnd, done, h.TaskEngine.hostAndPort, result)
if err != nil {
return false, fmt.Errorf("could not write history: %w", err)
}
return true, nil
})
if err != nil {
log.Error("Could not record transaction: ", err)
return
}
if !cm {
log.Error("Committing the task records failed")
}
}
func (h *taskTypeHandler) AssertMachineHasCapacity() error {
r := h.TaskEngine.ResourcesAvailable()
if r.Cpu-h.Cost.Cpu < 0 {
return errors.New("Did not accept " + h.Name + " task: out of cpu")
}
if h.Cost.Ram > r.Ram {
return errors.New("Did not accept " + h.Name + " task: out of RAM")
}
if r.Gpu-h.Cost.Gpu < 0 {
return errors.New("Did not accept " + h.Name + " task: out of available GPU")
}
return nil
}

22
lib/harmony/resources/getGPU.go Normal file
View File

@ -0,0 +1,22 @@
//go:build !darwin
// +build !darwin
package resources
import (
"strings"
ffi "github.com/filecoin-project/filecoin-ffi"
)
func getGPUDevices() float64 { // GPU boolean
gpus, err := ffi.GetGPUDevices()
if err != nil {
logger.Errorf("getting gpu devices failed: %+v", err)
}
all := strings.ToLower(strings.Join(gpus, ","))
if len(gpus) > 1 || strings.Contains(all, "ati") || strings.Contains(all, "nvidia") {
return float64(len(gpus))
}
return 0
}

8
lib/harmony/resources/getGPU_darwin.go Normal file
View File

@ -0,0 +1,8 @@
//go:build darwin
// +build darwin
package resources
func getGPUDevices() float64 {
return 10000.0 // unserious value intended for non-production use.
}

22
lib/harmony/resources/memsys.go Normal file
View File

@ -0,0 +1,22 @@
//go:build darwin || freebsd || openbsd || dragonfly || netbsd
// +build darwin freebsd openbsd dragonfly netbsd
package resources
import (
"encoding/binary"
"syscall"
)
func sysctlUint64(name string) (uint64, error) {
s, err := syscall.Sysctl(name)
if err != nil {
return 0, err
}
// hack because the string conversion above drops a \0
b := []byte(s)
if len(b) < 8 {
b = append(b, 0)
}
return binary.LittleEndian.Uint64(b), nil
}

17
lib/harmony/resources/miniopencl/cl.h Normal file
View File

@ -0,0 +1,17 @@
#ifndef CL_H
#define CL_H
#define CL_USE_DEPRECATED_OPENCL_1_1_APIS
#define CL_USE_DEPRECATED_OPENCL_1_2_APIS
#define CL_USE_DEPRECATED_OPENCL_2_0_APIS
#define CL_TARGET_OPENCL_VERSION 300
#ifdef __APPLE__
#include "OpenCL/opencl.h"
#else
#include "CL/opencl.h"
#endif
#endif /* CL_H */

93
lib/harmony/resources/miniopencl/mini_opencl.go Normal file
View File

@ -0,0 +1,93 @@
// Package cl was borrowed from the go-opencl library which is more complex and
// doesn't compile well for our needs.
package cl
// #include "cl.h"
import "C"
import (
"fmt"
"unsafe"
)
const maxPlatforms = 32
type Platform struct {
id C.cl_platform_id
}
// Obtain the list of platforms available.
func GetPlatforms() ([]*Platform, error) {
var platformIds [maxPlatforms]C.cl_platform_id
var nPlatforms C.cl_uint
err := C.clGetPlatformIDs(C.cl_uint(maxPlatforms), &platformIds[0], &nPlatforms)
if err == -1001 { // No platforms found
return nil, nil
}
if err != C.CL_SUCCESS {
return nil, toError(err)
}
platforms := make([]*Platform, nPlatforms)
for i := 0; i < int(nPlatforms); i++ {
platforms[i] = &Platform{id: platformIds[i]}
}
return platforms, nil
}
const maxDeviceCount = 64
type DeviceType uint
const (
DeviceTypeAll DeviceType = C.CL_DEVICE_TYPE_ALL
)
type Device struct {
id C.cl_device_id
}
func (p *Platform) GetAllDevices() ([]*Device, error) {
var deviceIds [maxDeviceCount]C.cl_device_id
var numDevices C.cl_uint
var platformId C.cl_platform_id
if p != nil {
platformId = p.id
}
if err := C.clGetDeviceIDs(platformId, C.cl_device_type(DeviceTypeAll), C.cl_uint(maxDeviceCount), &deviceIds[0], &numDevices); err != C.CL_SUCCESS {
return nil, toError(err)
}
if numDevices > maxDeviceCount {
numDevices = maxDeviceCount
}
devices := make([]*Device, numDevices)
for i := 0; i < int(numDevices); i++ {
devices[i] = &Device{id: deviceIds[i]}
}
return devices, nil
}
func toError(code C.cl_int) error {
return ErrOther(code)
}
type ErrOther int
func (e ErrOther) Error() string {
return fmt.Sprintf("OpenCL: error %d", int(e))
}
// Size of global device memory in bytes.
func (d *Device) GlobalMemSize() int64 {
val, _ := d.getInfoUlong(C.CL_DEVICE_GLOBAL_MEM_SIZE, true)
return val
}
func (d *Device) getInfoUlong(param C.cl_device_info, panicOnError bool) (int64, error) {
var val C.cl_ulong
if err := C.clGetDeviceInfo(d.id, param, C.size_t(unsafe.Sizeof(val)), unsafe.Pointer(&val), nil); err != C.CL_SUCCESS {
if panicOnError {
panic("Should never fail")
}
return 0, toError(err)
}
return int64(val), nil
}

142
lib/harmony/resources/resources.go Normal file
View File

@ -0,0 +1,142 @@
package resources
import (
"bytes"
"context"
"os/exec"
"regexp"
"runtime"
"sync/atomic"
"time"
logging "github.com/ipfs/go-log/v2"
"github.com/pbnjay/memory"
"golang.org/x/sys/unix"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
)
var LOOKS_DEAD_TIMEOUT = 10 * time.Minute // Time w/o minute heartbeats
type Resources struct {
Cpu int
Gpu float64
Ram uint64
MachineID int
}
type Reg struct {
Resources
shutdown atomic.Bool
}
var logger = logging.Logger("harmonytask")
var lotusRE = regexp.MustCompile("lotus-worker|lotus-harmony|yugabyted|yb-master|yb-tserver")
func Register(db *harmonydb.DB, hostnameAndPort string) (*Reg, error) {
var reg Reg
var err error
reg.Resources, err = getResources()
if err != nil {
return nil, err
}
ctx := context.Background()
{ // Learn our owner_id while updating harmony_machines
var ownerID *int
// Upsert query with last_contact update, fetch the machine ID
// (note this isn't a simple insert .. on conflict because host_and_port isn't unique)
err := db.QueryRow(ctx, `
WITH upsert AS (
UPDATE harmony_machines
SET cpu = $2, ram = $3, gpu = $4, last_contact = CURRENT_TIMESTAMP
WHERE host_and_port = $1
RETURNING id
),
inserted AS (
INSERT INTO harmony_machines (host_and_port, cpu, ram, gpu, last_contact)
SELECT $1, $2, $3, $4, CURRENT_TIMESTAMP
WHERE NOT EXISTS (SELECT id FROM upsert)
RETURNING id
)
SELECT id FROM upsert
UNION ALL
SELECT id FROM inserted;
`, hostnameAndPort, reg.Cpu, reg.Ram, reg.Gpu).Scan(&ownerID)
if err != nil {
return nil, xerrors.Errorf("inserting machine entry: %w", err)
}
if ownerID == nil {
return nil, xerrors.Errorf("no owner id")
}
reg.MachineID = *ownerID
cleaned := CleanupMachines(context.Background(), db)
logger.Infow("Cleaned up machines", "count", cleaned)
}
go func() {
for {
time.Sleep(time.Minute)
if reg.shutdown.Load() {
return
}
_, err := db.Exec(ctx, `UPDATE harmony_machines SET last_contact=CURRENT_TIMESTAMP`)
if err != nil {
logger.Error("Cannot keepalive ", err)
}
}
}()
return &reg, nil
}
func CleanupMachines(ctx context.Context, db *harmonydb.DB) int {
ct, err := db.Exec(ctx,
`DELETE FROM harmony_machines WHERE last_contact < CURRENT_TIMESTAMP - INTERVAL '1 MILLISECOND' * $1 `,
LOOKS_DEAD_TIMEOUT.Milliseconds()) // ms enables unit testing to change timeout.
if err != nil {
logger.Warn("unable to delete old machines: ", err)
}
return ct
}
func (res *Reg) Shutdown() {
res.shutdown.Store(true)
}
func getResources() (res Resources, err error) {
b, err := exec.Command(`ps`, `-ef`).CombinedOutput()
if err != nil {
logger.Warn("Could not safety check for 2+ processes: ", err)
} else {
found := 0
for _, b := range bytes.Split(b, []byte("\n")) {
if lotusRE.Match(b) {
found++
}
}
if found > 1 {
logger.Warn("lotus-provider's defaults are for running alone. Use task maximums or CGroups.")
}
}
res = Resources{
Cpu: runtime.NumCPU(),
Ram: memory.FreeMemory(),
Gpu: getGPUDevices(),
}
return res, nil
}
func DiskFree(path string) (uint64, error) {
s := unix.Statfs_t{}
err := unix.Statfs(path, &s)
if err != nil {
return 0, err
}
return s.Bfree * uint64(s.Bsize), nil
}

19
lib/harmony/taskhelp/common.go Normal file
View File

@ -0,0 +1,19 @@
package taskhelp
// SubsetIf returns a subset of the slice for which the predicate is true.
// It does not allocate memory, but rearranges the list in place.
// A non-zero list input will always return a non-zero list.
// The return value is the subset and a boolean indicating whether the subset was sliced.
func SliceIfFound[T any](slice []T, f func(T) bool) ([]T, bool) {
ct := 0
for i, v := range slice {
if f(v) {
slice[ct], slice[i] = slice[i], slice[ct]
ct++
}
}
if ct == 0 {
return slice, false
}
return slice[:ct], true
}

47
lib/promise/promise.go Normal file
View File

@ -0,0 +1,47 @@
package promise
import (
"context"
"sync"
)
type Promise[T any] struct {
val T
done chan struct{}
mu sync.Mutex
}
func (p *Promise[T]) Set(val T) {
p.mu.Lock()
defer p.mu.Unlock()
// Set value
p.val = val
// Initialize the done channel if it hasn't been initialized
if p.done == nil {
p.done = make(chan struct{})
}
// Signal that the value is set
close(p.done)
}
func (p *Promise[T]) Val(ctx context.Context) T {
p.mu.Lock()
// Initialize the done channel if it hasn't been initialized
if p.done == nil {
p.done = make(chan struct{})
}
p.mu.Unlock()
select {
case <-ctx.Done():
return *new(T)
case <-p.done:
p.mu.Lock()
val := p.val
p.mu.Unlock()
return val
}
}

65
lib/promise/promise_test.go Normal file
View File

@ -0,0 +1,65 @@
package promise
import (
"context"
"sync"
"testing"
"time"
)
func TestPromiseSet(t *testing.T) {
p := &Promise[int]{}
p.Set(42)
if p.val != 42 {
t.Fatalf("expected 42, got %v", p.val)
}
}
func TestPromiseVal(t *testing.T) {
p := &Promise[int]{}
p.Set(42)
ctx := context.Background()
val := p.Val(ctx)
if val != 42 {
t.Fatalf("expected 42, got %v", val)
}
}
func TestPromiseValWaitsForSet(t *testing.T) {
p := &Promise[int]{}
var val int
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
ctx := context.Background()
val = p.Val(ctx)
}()
time.Sleep(100 * time.Millisecond) // Give some time for the above goroutine to execute
p.Set(42)
wg.Wait()
if val != 42 {
t.Fatalf("expected 42, got %v", val)
}
}
func TestPromiseValContextCancel(t *testing.T) {
p := &Promise[int]{}
ctx, cancel := context.WithCancel(context.Background())
cancel() // Cancel the context
val := p.Val(ctx)
var zeroValue int
if val != zeroValue {
t.Fatalf("expected zero-value, got %v", val)
}
}

79
metrics/metrics.go
View File

@ -705,46 +705,55 @@ var (
}
)
var views = []*view.View{
InfoView,
PeerCountView,
APIRequestDurationView,
GraphsyncReceivingPeersCountView,
GraphsyncReceivingActiveCountView,
GraphsyncReceivingCountCountView,
GraphsyncReceivingTotalMemoryAllocatedView,
GraphsyncReceivingTotalPendingAllocationsView,
GraphsyncReceivingPeersPendingView,
GraphsyncSendingPeersCountView,
GraphsyncSendingActiveCountView,
GraphsyncSendingCountCountView,
GraphsyncSendingTotalMemoryAllocatedView,
GraphsyncSendingTotalPendingAllocationsView,
GraphsyncSendingPeersPendingView,
RcmgrAllowConnView,
RcmgrBlockConnView,
RcmgrAllowStreamView,
RcmgrBlockStreamView,
RcmgrAllowPeerView,
RcmgrBlockPeerView,
RcmgrAllowProtoView,
RcmgrBlockProtoView,
RcmgrBlockProtoPeerView,
RcmgrAllowSvcView,
RcmgrBlockSvcView,
RcmgrBlockSvcPeerView,
RcmgrAllowMemView,
RcmgrBlockMemView,
}
// DefaultViews is an array of OpenCensus views for metric gathering purposes
var DefaultViews = func() []*view.View {
views := []*view.View{
InfoView,
PeerCountView,
APIRequestDurationView,
GraphsyncReceivingPeersCountView,
GraphsyncReceivingActiveCountView,
GraphsyncReceivingCountCountView,
GraphsyncReceivingTotalMemoryAllocatedView,
GraphsyncReceivingTotalPendingAllocationsView,
GraphsyncReceivingPeersPendingView,
GraphsyncSendingPeersCountView,
GraphsyncSendingActiveCountView,
GraphsyncSendingCountCountView,
GraphsyncSendingTotalMemoryAllocatedView,
GraphsyncSendingTotalPendingAllocationsView,
GraphsyncSendingPeersPendingView,
RcmgrAllowConnView,
RcmgrBlockConnView,
RcmgrAllowStreamView,
RcmgrBlockStreamView,
RcmgrAllowPeerView,
RcmgrBlockPeerView,
RcmgrAllowProtoView,
RcmgrBlockProtoView,
RcmgrBlockProtoPeerView,
RcmgrAllowSvcView,
RcmgrBlockSvcView,
RcmgrBlockSvcPeerView,
RcmgrAllowMemView,
RcmgrBlockMemView,
}
views = append(views, blockstore.DefaultViews...)
views = append(views, rpcmetrics.DefaultViews...)
return views
}()
// RegisterViews adds views to the default list without modifying this file.
func RegisterViews(v ...*view.View) {
views = append(views, v...)
}
func init() {
RegisterViews(blockstore.DefaultViews...)
RegisterViews(rpcmetrics.DefaultViews...)
}
var ChainNodeViews = append([]*view.View{
ChainNodeHeightView,
ChainNodeHeightExpectedView,

6
metrics/proxy/proxy.go
View File

@ -10,6 +10,12 @@ import (
"github.com/filecoin-project/lotus/metrics"
)
func MetricedAPI[T, P any](a T) *P {
var out P
proxy(a, &out)
return &out
}
func MetricedStorMinerAPI(a api.StorageMiner) api.StorageMiner {
var out api.StorageMinerStruct
proxy(a, &out)

39
miner/miner.go
View File

@ -71,7 +71,7 @@ func NewMiner(api v1api.FullNode, epp gen.WinningPoStProver, addr address.Addres
api: api,
epp: epp,
address: addr,
waitFunc: func(ctx context.Context, baseTime uint64) (func(bool, abi.ChainEpoch, error), abi.ChainEpoch, error) {
propagationWaitFunc: func(ctx context.Context, baseTime uint64) (func(bool, abi.ChainEpoch, error), abi.ChainEpoch, error) {
// wait around for half the block time in case other parents come in
//
// if we're mining a block in the past via catch-up/rush mining,
@ -114,7 +114,7 @@ type Miner struct {
stop chan struct{}
stopping chan struct{}
waitFunc waitFunc
propagationWaitFunc waitFunc
// lastWork holds the last MiningBase we built upon.
lastWork *MiningBase
@ -205,15 +205,21 @@ func (m *Miner) mine(ctx context.Context) {
ctx, span := trace.StartSpan(ctx, "/mine")
defer span.End()
// Perform the Winning PoSt warmup in a separate goroutine.
go m.doWinPoStWarmup(ctx)
var lastBase MiningBase
// Start the main mining loop.
minerLoop:
for {
// Prepare a context for a single node operation.
ctx := cliutil.OnSingleNode(ctx)
// Handle stop signals.
select {
case <-m.stop:
// If a stop signal is received, clean up and exit the mining loop.
stopping := m.stopping
m.stop = nil
m.stopping = nil
@ -223,10 +229,11 @@ minerLoop:
default:
}
var base *MiningBase
var base *MiningBase // NOTE: This points to m.lastWork; Incrementing nulls here will increment it there.
var onDone func(bool, abi.ChainEpoch, error)
var injectNulls abi.ChainEpoch
// Look for the best mining candidate.
for {
prebase, err := m.GetBestMiningCandidate(ctx)
if err != nil {
@ -237,6 +244,7 @@ minerLoop:
continue
}
// Check if we have a new base or if the current base is still valid.
if base != nil && base.TipSet.Height() == prebase.TipSet.Height() && base.NullRounds == prebase.NullRounds {
base = prebase
break
@ -253,13 +261,13 @@ minerLoop:
// best mining candidate at that time.
// Wait until propagation delay period after block we plan to mine on
onDone, injectNulls, err = m.waitFunc(ctx, prebase.TipSet.MinTimestamp())
onDone, injectNulls, err = m.propagationWaitFunc(ctx, prebase.TipSet.MinTimestamp())
if err != nil {
log.Error(err)
continue
}
// just wait for the beacon entry to become available before we select our final mining base
// Ensure the beacon entry is available before finalizing the mining base.
_, err = m.api.StateGetBeaconEntry(ctx, prebase.TipSet.Height()+prebase.NullRounds+1)
if err != nil {
log.Errorf("failed getting beacon entry: %s", err)
@ -272,8 +280,9 @@ minerLoop:
base = prebase
}
base.NullRounds += injectNulls // testing
base.NullRounds += injectNulls // Adjust for testing purposes.
// Check for repeated mining candidates and handle sleep for the next round.
if base.TipSet.Equals(lastBase.TipSet) && lastBase.NullRounds == base.NullRounds {
log.Warnf("BestMiningCandidate from the previous round: %s (nulls:%d)", lastBase.TipSet.Cids(), lastBase.NullRounds)
if !m.niceSleep(time.Duration(build.BlockDelaySecs) * time.Second) {
@ -282,6 +291,7 @@ minerLoop:
continue
}
// Attempt to mine a block.
b, err := m.mineOne(ctx, base)
if err != nil {
log.Errorf("mining block failed: %+v", err)
@ -299,9 +309,12 @@ minerLoop:
}
onDone(b != nil, h, nil)
// Process the mined block.
if b != nil {
// Record the event of mining a block.
m.journal.RecordEvent(m.evtTypes[evtTypeBlockMined], func() interface{} {
return map[string]interface{}{
// Data about the mined block.
"parents": base.TipSet.Cids(),
"nulls": base.NullRounds,
"epoch": b.Header.Height,
@ -312,19 +325,23 @@ minerLoop:
btime := time.Unix(int64(b.Header.Timestamp), 0)
now := build.Clock.Now()
// Handle timing for broadcasting the block.
switch {
case btime == now:
// block timestamp is perfectly aligned with time.
case btime.After(now):
// Wait until it's time to broadcast the block.
if !m.niceSleep(build.Clock.Until(btime)) {
log.Warnf("received interrupt while waiting to broadcast block, will shutdown after block is sent out")
build.Clock.Sleep(build.Clock.Until(btime))
}
default:
// Log if the block was mined in the past.
log.Warnw("mined block in the past",
"block-time", btime, "time", build.Clock.Now(), "difference", build.Clock.Since(btime))
}
// Check for slash filter conditions.
if os.Getenv("LOTUS_MINER_NO_SLASHFILTER") != "_yes_i_know_i_can_and_probably_will_lose_all_my_fil_and_power_" && !build.IsNearUpgrade(base.TipSet.Height(), build.UpgradeWatermelonFixHeight) {
witness, fault, err := m.sf.MinedBlock(ctx, b.Header, base.TipSet.Height()+base.NullRounds)
if err != nil {
@ -339,25 +356,27 @@ minerLoop:
}
}
// Check for blocks created at the same height.
if _, ok := m.minedBlockHeights.Get(b.Header.Height); ok {
log.Warnw("Created a block at the same height as another block we've created", "height", b.Header.Height, "miner", b.Header.Miner, "parents", b.Header.Parents)
continue
}
// Add the block height to the mined block heights.
m.minedBlockHeights.Add(b.Header.Height, true)
// Submit the newly mined block.
if err := m.api.SyncSubmitBlock(ctx, b); err != nil {
log.Errorf("failed to submit newly mined block: %+v", err)
}
} else {
// If no block was mined, increase the null rounds and wait for the next epoch.
base.NullRounds++
// Wait until the next epoch, plus the propagation delay, so a new tipset
// has enough time to form.
//
// See: https://github.com/filecoin-project/lotus/issues/1845
// Calculate the time for the next round.
nextRound := time.Unix(int64(base.TipSet.MinTimestamp()+build.BlockDelaySecs*uint64(base.NullRounds))+int64(build.PropagationDelaySecs), 0)
// Wait for the next round or stop signal.
select {
case <-build.Clock.After(build.Clock.Until(nextRound)):
case <-m.stop:

14
miner/testminer.go
View File

@ -28,13 +28,13 @@ func NewTestMiner(nextCh <-chan MineReq, addr address.Address) func(v1api.FullNo
}
m := &Miner{
api: api,
waitFunc: chanWaiter(nextCh),
epp: epp,
minedBlockHeights: arc,
address: addr,
sf: slashfilter.New(ds.NewMapDatastore()),
journal: journal.NilJournal(),
api: api,
propagationWaitFunc: chanWaiter(nextCh),
epp: epp,
minedBlockHeights: arc,
address: addr,
sf: slashfilter.New(ds.NewMapDatastore()),
journal: journal.NilJournal(),
}
if err := m.Start(context.TODO()); err != nil {

30
node/builder_miner.go
View File

@ -19,6 +19,7 @@ import (
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/gen/slashfilter"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/markets/dagstore"
"github.com/filecoin-project/lotus/markets/dealfilter"
"github.com/filecoin-project/lotus/markets/idxprov"
@ -102,6 +103,7 @@ func ConfigStorageMiner(c interface{}) Option {
If(cfg.Subsystems.EnableSealing, Error(xerrors.Errorf("sealing can only be enabled on a mining node"))),
If(cfg.Subsystems.EnableSectorStorage, Error(xerrors.Errorf("sealing can only be enabled on a mining node"))),
),
If(cfg.Subsystems.EnableMining,
If(!cfg.Subsystems.EnableSealing, Error(xerrors.Errorf("sealing can't be disabled on a mining node yet"))),
If(!cfg.Subsystems.EnableSectorStorage, Error(xerrors.Errorf("sealing can't be disabled on a mining node yet"))),
@ -116,19 +118,35 @@ func ConfigStorageMiner(c interface{}) Option {
// Mining / proving
Override(new(*slashfilter.SlashFilter), modules.NewSlashFilter),
Override(new(*miner.Miner), modules.SetupBlockProducer),
Override(new(gen.WinningPoStProver), storage.NewWinningPoStProver),
If(!cfg.Subsystems.DisableWinningPoSt,
Override(new(*miner.Miner), modules.SetupBlockProducer),
Override(new(gen.WinningPoStProver), storage.NewWinningPoStProver),
),
Override(PreflightChecksKey, modules.PreflightChecks),
Override(new(*sealing.Sealing), modules.SealingPipeline(cfg.Fees)),
Override(new(*wdpost.WindowPoStScheduler), modules.WindowPostScheduler(cfg.Fees, cfg.Proving)),
If(!cfg.Subsystems.DisableWindowPoSt, Override(new(*wdpost.WindowPoStScheduler), modules.WindowPostScheduler(cfg.Fees, cfg.Proving))),
Override(new(sectorblocks.SectorBuilder), From(new(*sealing.Sealing))),
),
If(cfg.Subsystems.EnableSectorStorage,
// Sector storage
Override(new(*paths.Index), paths.NewIndex),
Override(new(paths.SectorIndex), From(new(*paths.Index))),
If(cfg.Subsystems.EnableSectorIndexDB,
Override(new(*paths.DBIndex), paths.NewDBIndex),
Override(new(paths.SectorIndex), From(new(*paths.DBIndex))),
// sector index db is the only thing on lotus-miner that will use harmonydb
Override(new(*harmonydb.DB), func(cfg config.HarmonyDB, id harmonydb.ITestID) (*harmonydb.DB, error) {
return harmonydb.NewFromConfigWithITestID(cfg)(id)
}),
),
If(!cfg.Subsystems.EnableSectorIndexDB,
Override(new(*paths.MemIndex), paths.NewMemIndex),
Override(new(paths.SectorIndex), From(new(*paths.MemIndex))),
),
Override(new(*sectorstorage.Manager), modules.SectorStorage),
Override(new(sectorstorage.Unsealer), From(new(*sectorstorage.Manager))),
Override(new(sectorstorage.SectorManager), From(new(*sectorstorage.Manager))),
@ -230,6 +248,8 @@ func ConfigStorageMiner(c interface{}) Option {
Override(new(config.SealerConfig), cfg.Storage),
Override(new(config.ProvingConfig), cfg.Proving),
Override(new(config.HarmonyDB), cfg.HarmonyDB),
Override(new(harmonydb.ITestID), harmonydb.ITestID("")),
Override(new(*ctladdr.AddressSelector), modules.AddressSelector(&cfg.Addresses)),
)
}

2
node/config/cfgdocgen/gen.go
View File

@ -104,7 +104,7 @@ var Doc = map[string][]DocField{
for _, typeName := range outt {
typ := out[typeName]
fmt.Printf("\t\"%s\": []DocField{\n", typeName)
fmt.Printf("\t\"%s\": {\n", typeName)
for _, f := range typ {
fmt.Println("\t\t{")

42
node/config/def.go
View File

@ -238,6 +238,7 @@ func DefaultStorageMiner() *StorageMiner {
EnableSealing: true,
EnableSectorStorage: true,
EnableMarkets: false,
EnableSectorIndexDB: false,
},
Fees: MinerFeeConfig{
@ -274,6 +275,13 @@ func DefaultStorageMiner() *StorageMiner {
MaxConcurrentUnseals: 5,
GCInterval: Duration(1 * time.Minute),
},
HarmonyDB: HarmonyDB{
Hosts: []string{"127.0.0.1"},
Username: "yugabyte",
Password: "yugabyte",
Database: "yugabyte",
Port: "5433",
},
}
cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http"
@ -340,3 +348,37 @@ func DefaultUserRaftConfig() *UserRaftConfig {
return &cfg
}
func DefaultLotusProvider() *LotusProviderConfig {
return &LotusProviderConfig{
Subsystems: ProviderSubsystemsConfig{},
Fees: LotusProviderFees{
DefaultMaxFee: DefaultDefaultMaxFee,
MaxPreCommitGasFee: types.MustParseFIL("0.025"),
MaxCommitGasFee: types.MustParseFIL("0.05"),
MaxPreCommitBatchGasFee: BatchFeeConfig{
Base: types.MustParseFIL("0"),
PerSector: types.MustParseFIL("0.02"),
},
MaxCommitBatchGasFee: BatchFeeConfig{
Base: types.MustParseFIL("0"),
PerSector: types.MustParseFIL("0.03"), // enough for 6 agg and 1nFIL base fee
},
MaxTerminateGasFee: types.MustParseFIL("0.5"),
MaxWindowPoStGasFee: types.MustParseFIL("5"),
MaxPublishDealsFee: types.MustParseFIL("0.05"),
},
Addresses: LotusProviderAddresses{
PreCommitControl: []string{},
CommitControl: []string{},
TerminateControl: []string{},
},
Proving: ProvingConfig{
ParallelCheckLimit: 32,
PartitionCheckTimeout: Duration(20 * time.Minute),
SingleCheckTimeout: Duration(10 * time.Minute),
},
}
}

6
node/config/def_test.go
View File

@ -71,6 +71,12 @@ func TestDefaultMinerRoundtrip(t *testing.T) {
fmt.Println(s)
// Differs between test envs
c.HarmonyDB = HarmonyDB{}
c2.(*StorageMiner).HarmonyDB = HarmonyDB{}
fmt.Println(c)
fmt.Println(c2)
require.True(t, reflect.DeepEqual(c, c2))
}

314
node/config/doc_gen.go
View File

@ -9,7 +9,7 @@ type DocField struct {
}
var Doc = map[string][]DocField{
"API": []DocField{
"API": {
{
Name: "ListenAddress",
Type: "string",
@ -29,7 +29,23 @@ var Doc = map[string][]DocField{
Comment: ``,
},
},
"Backup": []DocField{
"ApisConfig": {
{
Name: "ChainApiInfo",
Type: "[]string",
Comment: `ChainApiInfo is the API endpoint for the Lotus daemon.`,
},
{
Name: "StorageRPCSecret",
Type: "string",
Comment: `RPC Secret for the storage subsystem.
If integrating with lotus-miner this must match the value from
cat ~/.lotusminer/keystore/MF2XI2BNNJ3XILLQOJUXMYLUMU | jq -r .PrivateKey`,
},
},
"Backup": {
{
Name: "DisableMetadataLog",
Type: "bool",
@ -41,7 +57,7 @@ Note that in case of metadata corruption it might be much harder to recover
your node if metadata log is disabled`,
},
},
"BatchFeeConfig": []DocField{
"BatchFeeConfig": {
{
Name: "Base",
Type: "types.FIL",
@ -55,7 +71,7 @@ your node if metadata log is disabled`,
Comment: ``,
},
},
"Chainstore": []DocField{
"Chainstore": {
{
Name: "EnableSplitstore",
Type: "bool",
@ -69,7 +85,7 @@ your node if metadata log is disabled`,
Comment: ``,
},
},
"Client": []DocField{
"Client": {
{
Name: "UseIpfs",
Type: "bool",
@ -117,7 +133,7 @@ without existing payment channels with available funds will fail instead
of automatically performing on-chain operations.`,
},
},
"Common": []DocField{
"Common": {
{
Name: "API",
Type: "API",
@ -149,7 +165,7 @@ of automatically performing on-chain operations.`,
Comment: ``,
},
},
"DAGStoreConfig": []DocField{
"DAGStoreConfig": {
{
Name: "RootDir",
Type: "string",
@ -206,7 +222,7 @@ representation, e.g. 1m, 5m, 1h.
Default value: 1 minute.`,
},
},
"DealmakingConfig": []DocField{
"DealmakingConfig": {
{
Name: "ConsiderOnlineStorageDeals",
Type: "bool",
@ -341,7 +357,7 @@ see https://lotus.filecoin.io/storage-providers/advanced-configurations/market/#
Comment: ``,
},
},
"Events": []DocField{
"Events": {
{
Name: "DisableRealTimeFilterAPI",
Type: "bool",
@ -394,7 +410,7 @@ the database must already exist and be writeable. If a relative path is provided
relative to the CWD (current working directory).`,
},
},
"FaultReporterConfig": []DocField{
"FaultReporterConfig": {
{
Name: "EnableConsensusFaultReporter",
Type: "bool",
@ -423,7 +439,7 @@ ReportConsensusFault messages. It will pay for gas fees, and receive any
rewards. This address should have adequate funds to cover gas fees.`,
},
},
"FeeConfig": []DocField{
"FeeConfig": {
{
Name: "DefaultMaxFee",
Type: "types.FIL",
@ -431,7 +447,7 @@ rewards. This address should have adequate funds to cover gas fees.`,
Comment: ``,
},
},
"FevmConfig": []DocField{
"FevmConfig": {
{
Name: "EnableEthRPC",
Type: "bool",
@ -453,7 +469,7 @@ Set to 0 to keep all mappings`,
Comment: ``,
},
},
"FullNode": []DocField{
"FullNode": {
{
Name: "Client",
Type: "Client",
@ -503,7 +519,40 @@ Set to 0 to keep all mappings`,
Comment: ``,
},
},
"IndexConfig": []DocField{
"HarmonyDB": {
{
Name: "Hosts",
Type: "[]string",
Comment: `HOSTS is a list of hostnames to nodes running YugabyteDB
in a cluster. Only 1 is required`,
},
{
Name: "Username",
Type: "string",
Comment: `The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default.`,
},
{
Name: "Password",
Type: "string",
Comment: `The password for the related username. Blank for default.`,
},
{
Name: "Database",
Type: "string",
Comment: `The database (logical partition) within Yugabyte. Blank for default.`,
},
{
Name: "Port",
Type: "string",
Comment: `The port to find Yugabyte. Blank for default.`,
},
},
"IndexConfig": {
{
Name: "EnableMsgIndex",
Type: "bool",
@ -512,7 +561,7 @@ Set to 0 to keep all mappings`,
EnableMsgIndex enables indexing of messages on chain.`,
},
},
"IndexProviderConfig": []DocField{
"IndexProviderConfig": {
{
Name: "Enable",
Type: "bool",
@ -557,7 +606,15 @@ starts. By default, the cache is rehydrated from previously cached entries store
datastore if any is present.`,
},
},
"Libp2p": []DocField{
"JournalConfig": {
{
Name: "DisabledEvents",
Type: "string",
Comment: `Events of the form: "system1:event1,system1:event2[,...]"`,
},
},
"Libp2p": {
{
Name: "ListenAddresses",
Type: "[]string",
@ -624,7 +681,7 @@ count towards this limit.`,
closed by the connection manager.`,
},
},
"Logging": []DocField{
"Logging": {
{
Name: "SubsystemLevels",
Type: "map[string]string",
@ -632,7 +689,137 @@ closed by the connection manager.`,
Comment: `SubsystemLevels specify per-subsystem log levels`,
},
},
"MinerAddressConfig": []DocField{
"LotusProviderAddresses": {
{
Name: "PreCommitControl",
Type: "[]string",
Comment: `Addresses to send PreCommit messages from`,
},
{
Name: "CommitControl",
Type: "[]string",
Comment: `Addresses to send Commit messages from`,
},
{
Name: "TerminateControl",
Type: "[]string",
Comment: ``,
},
{
Name: "DisableOwnerFallback",
Type: "bool",
Comment: `DisableOwnerFallback disables usage of the owner address for messages
sent automatically`,
},
{
Name: "DisableWorkerFallback",
Type: "bool",
Comment: `DisableWorkerFallback disables usage of the worker address for messages
sent automatically, if control addresses are configured.
A control address that doesn't have enough funds will still be chosen
over the worker address if this flag is set.`,
},
{
Name: "MinerAddresses",
Type: "[]string",
Comment: `MinerAddresses are the addresses of the miner actors to use for sending messages`,
},
},
"LotusProviderConfig": {
{
Name: "Subsystems",
Type: "ProviderSubsystemsConfig",
Comment: ``,
},
{
Name: "Fees",
Type: "LotusProviderFees",
Comment: ``,
},
{
Name: "Addresses",
Type: "LotusProviderAddresses",
Comment: ``,
},
{
Name: "Proving",
Type: "ProvingConfig",
Comment: ``,
},
{
Name: "Journal",
Type: "JournalConfig",
Comment: ``,
},
{
Name: "Apis",
Type: "ApisConfig",
Comment: ``,
},
},
"LotusProviderFees": {
{
Name: "DefaultMaxFee",
Type: "types.FIL",
Comment: ``,
},
{
Name: "MaxPreCommitGasFee",
Type: "types.FIL",
Comment: ``,
},
{
Name: "MaxCommitGasFee",
Type: "types.FIL",
Comment: ``,
},
{
Name: "MaxPreCommitBatchGasFee",
Type: "BatchFeeConfig",
Comment: `maxBatchFee = maxBase + maxPerSector * nSectors`,
},
{
Name: "MaxCommitBatchGasFee",
Type: "BatchFeeConfig",
Comment: ``,
},
{
Name: "MaxTerminateGasFee",
Type: "types.FIL",
Comment: ``,
},
{
Name: "MaxWindowPoStGasFee",
Type: "types.FIL",
Comment: `WindowPoSt is a high-value operation, so the default fee should be high.`,
},
{
Name: "MaxPublishDealsFee",
Type: "types.FIL",
Comment: ``,
},
},
"MinerAddressConfig": {
{
Name: "PreCommitControl",
Type: "[]string",
@ -674,7 +861,7 @@ A control address that doesn't have enough funds will still be chosen
over the worker address if this flag is set.`,
},
},
"MinerFeeConfig": []DocField{
"MinerFeeConfig": {
{
Name: "MaxPreCommitGasFee",
Type: "types.FIL",
@ -730,7 +917,7 @@ over the worker address if this flag is set.`,
Comment: ``,
},
},
"MinerSubsystemConfig": []DocField{
"MinerSubsystemConfig": {
{
Name: "EnableMining",
Type: "bool",
@ -755,6 +942,14 @@ over the worker address if this flag is set.`,
Comment: ``,
},
{
Name: "EnableSectorIndexDB",
Type: "bool",
Comment: `When enabled, the sector index will reside in an external database
as opposed to the local KV store in the miner process
This is useful to allow workers to bypass the lotus miner to access sector information`,
},
{
Name: "SealerApiInfo",
Type: "string",
@ -767,8 +962,59 @@ over the worker address if this flag is set.`,
Comment: ``,
},
{
Name: "DisableWindowPoSt",
Type: "bool",
Comment: `When window post is enabled, the miner will automatically submit window post proofs
for all sectors that are eligible for window post
IF WINDOW POST IS DISABLED, THE MINER WILL NOT SUBMIT WINDOW POST PROOFS
THIS WILL RESULT IN FAULTS AND PENALTIES IF NO OTHER MECHANISM IS RUNNING
TO SUBMIT WINDOW POST PROOFS.
Note: This option is entirely disabling the window post scheduler,
not just the builtin PoSt computation like Proving.DisableBuiltinWindowPoSt.
This option will stop lotus-miner from performing any actions related
to window post, including scheduling, submitting proofs, and recovering
sectors.`,
},
{
Name: "DisableWinningPoSt",
Type: "bool",
Comment: `When winning post is disabled, the miner process will NOT attempt to mine
blocks. This should only be set when there's an external process mining
blocks on behalf of the miner.
When disabled and no external block producers are configured, all potential
block rewards will be missed!`,
},
},
"ProvingConfig": []DocField{
"ProviderSubsystemsConfig": {
{
Name: "EnableWindowPost",
Type: "bool",
Comment: ``,
},
{
Name: "WindowPostMaxTasks",
Type: "int",
Comment: ``,
},
{
Name: "EnableWinningPost",
Type: "bool",
Comment: ``,
},
{
Name: "WinningPostMaxTasks",
Type: "int",
Comment: ``,
},
},
"ProvingConfig": {
{
Name: "ParallelCheckLimit",
Type: "int",
@ -892,7 +1138,7 @@ Note that setting this value lower may result in less efficient gas use - more m
to prove each deadline, resulting in more total gas use (but each message will have lower gas limit)`,
},
},
"Pubsub": []DocField{
"Pubsub": {
{
Name: "Bootstrapper",
Type: "bool",
@ -952,7 +1198,7 @@ This property is used only if ElasticSearchTracer propery is set.`,
Comment: `Auth token that will be passed with logs to elasticsearch - used for weighted peers score.`,
},
},
"RetrievalPricing": []DocField{
"RetrievalPricing": {
{
Name: "Strategy",
Type: "string",
@ -972,7 +1218,7 @@ This property is used only if ElasticSearchTracer propery is set.`,
Comment: ``,
},
},
"RetrievalPricingDefault": []DocField{
"RetrievalPricingDefault": {
{
Name: "VerifiedDealsFreeTransfer",
Type: "bool",
@ -983,7 +1229,7 @@ This parameter is ONLY applicable if the retrieval pricing policy strategy has b
default value is true`,
},
},
"RetrievalPricingExternal": []DocField{
"RetrievalPricingExternal": {
{
Name: "Path",
Type: "string",
@ -992,7 +1238,7 @@ default value is true`,
This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "external".`,
},
},
"SealerConfig": []DocField{
"SealerConfig": {
{
Name: "ParallelFetchLimit",
Type: "int",
@ -1093,7 +1339,7 @@ to use when evaluating tasks against this worker. An empty value defaults
to "hardware".`,
},
},
"SealingConfig": []DocField{
"SealingConfig": {
{
Name: "MaxWaitDealsSectors",
Type: "uint64",
@ -1305,7 +1551,7 @@ Submitting a smaller number of prove commits per epoch would reduce the possibil
Comment: `UseSyntheticPoRep, when set to true, will reduce the amount of cache data held on disk after the completion of PreCommit 2 to 11GiB.`,
},
},
"Splitstore": []DocField{
"Splitstore": {
{
Name: "ColdStoreType",
Type: "string",
@ -1372,7 +1618,7 @@ is set. Moving GC will not occur when total moving size exceeds
HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer`,
},
},
"StorageMiner": []DocField{
"StorageMiner": {
{
Name: "Subsystems",
Type: "MinerSubsystemConfig",
@ -1425,10 +1671,16 @@ HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer`,
Name: "DAGStore",
Type: "DAGStoreConfig",
Comment: ``,
},
{
Name: "HarmonyDB",
Type: "HarmonyDB",
Comment: ``,
},
},
"UserRaftConfig": []DocField{
"UserRaftConfig": {
{
Name: "ClusterModeEnabled",
Type: "bool",
@ -1490,7 +1742,7 @@ copies that we keep as backups (renaming) after cleanup.`,
Comment: `Tracing enables propagation of contexts across binary boundaries.`,
},
},
"Wallet": []DocField{
"Wallet": {
{
Name: "RemoteBackend",
Type: "string",

12
node/config/load.go
View File

@ -124,6 +124,7 @@ func ValidateSplitstoreSet(cfgRaw string) error {
type cfgUpdateOpts struct {
comment bool
keepUncommented func(string) bool
noEnv bool
}
// UpdateCfgOpt is a functional option for updating the config
@ -149,6 +150,13 @@ func DefaultKeepUncommented() UpdateCfgOpt {
return KeepUncommented(MatchEnableSplitstoreField)
}
func NoEnv() UpdateCfgOpt {
return func(opts *cfgUpdateOpts) error {
opts.noEnv = true
return nil
}
}
// ConfigUpdate takes in a config and a default config and optionally comments out default values
func ConfigUpdate(cfgCur, cfgDef interface{}, opts ...UpdateCfgOpt) ([]byte, error) {
var updateOpts cfgUpdateOpts
@ -236,7 +244,9 @@ func ConfigUpdate(cfgCur, cfgDef interface{}, opts ...UpdateCfgOpt) ([]byte, err
outLines = append(outLines, pad+"# type: "+doc.Type)
}
outLines = append(outLines, pad+"# env var: LOTUS_"+strings.ToUpper(strings.ReplaceAll(section, ".", "_"))+"_"+strings.ToUpper(lf[0]))
if !updateOpts.noEnv {
outLines = append(outLines, pad+"# env var: LOTUS_"+strings.ToUpper(strings.ReplaceAll(section, ".", "_"))+"_"+strings.ToUpper(lf[0]))
}
}
}

26
node/config/storage.go
View File

@ -2,8 +2,11 @@ package config
import (
"encoding/json"
"errors"
"io"
"io/fs"
"os"
"path"
"golang.org/x/xerrors"
@ -36,14 +39,31 @@ func StorageFromReader(reader io.Reader) (*storiface.StorageConfig, error) {
return &cfg, nil
}
func WriteStorageFile(path string, config storiface.StorageConfig) error {
func WriteStorageFile(filePath string, config storiface.StorageConfig) error {
b, err := json.MarshalIndent(config, "", " ")
if err != nil {
return xerrors.Errorf("marshaling storage config: %w", err)
}
if err := os.WriteFile(path, b, 0644); err != nil {
return xerrors.Errorf("persisting storage config (%s): %w", path, err)
info, err := os.Stat(filePath)
if err != nil {
if !errors.Is(err, fs.ErrNotExist) {
return xerrors.Errorf("statting storage config (%s): %w", filePath, err)
}
if path.Base(filePath) == "." {
filePath = path.Join(filePath, "storage.json")
}
} else {
if info.IsDir() || path.Base(filePath) == "." {
filePath = path.Join(filePath, "storage.json")
}
}
if err := os.MkdirAll(path.Dir(filePath), 0755); err != nil {
return xerrors.Errorf("making storage config parent directory: %w", err)
}
if err := os.WriteFile(filePath, b, 0644); err != nil {
return xerrors.Errorf("persisting storage config (%s): %w", filePath, err)
}
return nil

109
node/config/types.go
View File

@ -62,6 +62,40 @@ type StorageMiner struct {
Fees MinerFeeConfig
Addresses MinerAddressConfig
DAGStore DAGStoreConfig
HarmonyDB HarmonyDB
}
type LotusProviderConfig struct {
Subsystems ProviderSubsystemsConfig
Fees LotusProviderFees
Addresses LotusProviderAddresses
Proving ProvingConfig
Journal JournalConfig
Apis ApisConfig
}
type ApisConfig struct {
// ChainApiInfo is the API endpoint for the Lotus daemon.
ChainApiInfo []string
// RPC Secret for the storage subsystem.
// If integrating with lotus-miner this must match the value from
// cat ~/.lotusminer/keystore/MF2XI2BNNJ3XILLQOJUXMYLUMU | jq -r .PrivateKey
StorageRPCSecret string
}
type JournalConfig struct {
//Events of the form: "system1:event1,system1:event2[,...]"
DisabledEvents string
}
type ProviderSubsystemsConfig struct {
EnableWindowPost bool
WindowPostMaxTasks int
EnableWinningPost bool
WinningPostMaxTasks int
}
type DAGStoreConfig struct {
@ -109,8 +143,32 @@ type MinerSubsystemConfig struct {
EnableSectorStorage bool
EnableMarkets bool
// When enabled, the sector index will reside in an external database
// as opposed to the local KV store in the miner process
// This is useful to allow workers to bypass the lotus miner to access sector information
EnableSectorIndexDB bool
SealerApiInfo string // if EnableSealing == false
SectorIndexApiInfo string // if EnableSectorStorage == false
// When window post is enabled, the miner will automatically submit window post proofs
// for all sectors that are eligible for window post
// IF WINDOW POST IS DISABLED, THE MINER WILL NOT SUBMIT WINDOW POST PROOFS
// THIS WILL RESULT IN FAULTS AND PENALTIES IF NO OTHER MECHANISM IS RUNNING
// TO SUBMIT WINDOW POST PROOFS.
// Note: This option is entirely disabling the window post scheduler,
// not just the builtin PoSt computation like Proving.DisableBuiltinWindowPoSt.
// This option will stop lotus-miner from performing any actions related
// to window post, including scheduling, submitting proofs, and recovering
// sectors.
DisableWindowPoSt bool
// When winning post is disabled, the miner process will NOT attempt to mine
// blocks. This should only be set when there's an external process mining
// blocks on behalf of the miner.
// When disabled and no external block producers are configured, all potential
// block rewards will be missed!
DisableWinningPoSt bool
}
type DealmakingConfig struct {
@ -494,6 +552,20 @@ type MinerFeeConfig struct {
MaximizeWindowPoStFeeCap bool
}
type LotusProviderFees struct {
DefaultMaxFee types.FIL
MaxPreCommitGasFee types.FIL
MaxCommitGasFee types.FIL
// maxBatchFee = maxBase + maxPerSector * nSectors
MaxPreCommitBatchGasFee BatchFeeConfig
MaxCommitBatchGasFee BatchFeeConfig
MaxTerminateGasFee types.FIL
// WindowPoSt is a high-value operation, so the default fee should be high.
MaxWindowPoStGasFee types.FIL
MaxPublishDealsFee types.FIL
}
type MinerAddressConfig struct {
// Addresses to send PreCommit messages from
PreCommitControl []string
@ -512,6 +584,26 @@ type MinerAddressConfig struct {
DisableWorkerFallback bool
}
type LotusProviderAddresses struct {
// Addresses to send PreCommit messages from
PreCommitControl []string
// Addresses to send Commit messages from
CommitControl []string
TerminateControl []string
// DisableOwnerFallback disables usage of the owner address for messages
// sent automatically
DisableOwnerFallback bool
// DisableWorkerFallback disables usage of the worker address for messages
// sent automatically, if control addresses are configured.
// A control address that doesn't have enough funds will still be chosen
// over the worker address if this flag is set.
DisableWorkerFallback bool
// MinerAddresses are the addresses of the miner actors to use for sending messages
MinerAddresses []string
}
// API contains configs for API endpoint
type API struct {
// Binding address for the Lotus API
@ -735,6 +827,23 @@ type IndexConfig struct {
EnableMsgIndex bool
}
type HarmonyDB struct {
// HOSTS is a list of hostnames to nodes running YugabyteDB
// in a cluster. Only 1 is required
Hosts []string
// The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default.
Username string
// The password for the related username. Blank for default.
Password string
// The database (logical partition) within Yugabyte. Blank for default.
Database string
// The port to find Yugabyte. Blank for default.
Port string
}
type FaultReporterConfig struct {
// EnableConsensusFaultReporter controls whether the node will monitor and
// report consensus faults. When enabled, the node will watch for malicious

3
node/impl/storminer.go
View File

@ -45,6 +45,7 @@ import (
lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
mktsdagstore "github.com/filecoin-project/lotus/markets/dagstore"
"github.com/filecoin-project/lotus/markets/storageadapter"
"github.com/filecoin-project/lotus/miner"
@ -122,6 +123,8 @@ type StorageMinerAPI struct {
GetSealingConfigFunc dtypes.GetSealingConfigFunc `optional:"true"`
GetExpectedSealDurationFunc dtypes.GetExpectedSealDurationFunc `optional:"true"`
SetExpectedSealDurationFunc dtypes.SetExpectedSealDurationFunc `optional:"true"`
HarmonyDB *harmonydb.DB `optional:"true"`
}
var _ api.StorageMiner = &StorageMinerAPI{}

3
node/modules/storageminer.go
View File

@ -311,12 +311,11 @@ func WindowPostScheduler(fc config.MinerFeeConfig, pc config.ProvingConfig) func
verif = params.Verifier
j = params.Journal
as = params.AddrSel
maddr = address.Address(params.Maddr)
)
ctx := helpers.LifecycleCtx(mctx, lc)
fps, err := wdpost.NewWindowedPoStScheduler(api, fc, pc, as, sealer, verif, sealer, j, maddr)
fps, err := wdpost.NewWindowedPoStScheduler(api, fc, pc, as, sealer, verif, sealer, j, []dtypes.MinerAddress{params.Maddr})
if err != nil {
return nil, err

26
node/repo/fsrepo.go
View File

@ -185,6 +185,30 @@ func (worker) APIInfoEnvVars() (primary string, fallbacks []string, deprecated [
return "WORKER_API_INFO", nil, nil
}
type provider struct{}
var Provider provider
func (provider) Type() string {
return "Provider"
}
func (provider) Config() interface{} {
return &struct{}{}
}
func (provider) APIFlags() []string {
return []string{"provider-api-url"}
}
func (provider) RepoFlags() []string {
return []string{"provider-repo"}
}
func (provider) APIInfoEnvVars() (primary string, fallbacks []string, deprecated []string) {
return "PROVIDER_API_INFO", nil, nil
}
var Wallet wallet
type wallet struct {
@ -322,7 +346,7 @@ func (fsr *FsRepo) APIEndpoint() (multiaddr.Multiaddr, error) {
f, err := os.Open(p)
if os.IsNotExist(err) {
return nil, ErrNoAPIEndpoint
return nil, xerrors.Errorf("No file (%s): %w", p, ErrNoAPIEndpoint)
} else if err != nil {
return nil, err
}

4
node/repo/repo_test.go
View File

@ -16,7 +16,7 @@ import (
func basicTest(t *testing.T, repo Repo) {
apima, err := repo.APIEndpoint()
if assert.Error(t, err) {
assert.Equal(t, ErrNoAPIEndpoint, err)
assert.ErrorContains(t, err, ErrNoAPIEndpoint.Error())
}
assert.Nil(t, apima, "with no api endpoint, return should be nil")
@ -72,7 +72,7 @@ func basicTest(t *testing.T, repo Repo) {
apima, err = repo.APIEndpoint()
if assert.Error(t, err) {
assert.Equal(t, ErrNoAPIEndpoint, err, "after closing repo, api should be nil")
assert.ErrorContains(t, err, ErrNoAPIEndpoint.Error(), "after closing repo, api should be nil")
}
assert.Nil(t, apima, "with closed repo, apima should be set back to nil")

51
provider/address.go Normal file
View File

@ -0,0 +1,51 @@
package provider
import (
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/storage/ctladdr"
)
func AddressSelector(addrConf *config.LotusProviderAddresses) func() (*ctladdr.AddressSelector, error) {
return func() (*ctladdr.AddressSelector, error) {
as := &ctladdr.AddressSelector{}
if addrConf == nil {
return as, nil
}
as.DisableOwnerFallback = addrConf.DisableOwnerFallback
as.DisableWorkerFallback = addrConf.DisableWorkerFallback
for _, s := range addrConf.PreCommitControl {
addr, err := address.NewFromString(s)
if err != nil {
return nil, xerrors.Errorf("parsing precommit control address: %w", err)
}
as.PreCommitControl = append(as.PreCommitControl, addr)
}
for _, s := range addrConf.CommitControl {
addr, err := address.NewFromString(s)
if err != nil {
return nil, xerrors.Errorf("parsing commit control address: %w", err)
}
as.CommitControl = append(as.CommitControl, addr)
}
for _, s := range addrConf.TerminateControl {
addr, err := address.NewFromString(s)
if err != nil {
return nil, xerrors.Errorf("parsing terminate control address: %w", err)
}
as.TerminateControl = append(as.TerminateControl, addr)
}
return as, nil
}
}

50
provider/builder.go Normal file
View File

@ -0,0 +1,50 @@
package provider
import (
"context"
"time"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/node/config"
dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/provider/chainsched"
"github.com/filecoin-project/lotus/provider/lpmessage"
"github.com/filecoin-project/lotus/provider/lpwindow"
"github.com/filecoin-project/lotus/storage/ctladdr"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
//var log = logging.Logger("provider")
func WindowPostScheduler(ctx context.Context, fc config.LotusProviderFees, pc config.ProvingConfig,
api api.FullNode, verif storiface.Verifier, lw *sealer.LocalWorker, sender *lpmessage.Sender,
as *ctladdr.AddressSelector, addresses []dtypes.MinerAddress, db *harmonydb.DB,
stor paths.Store, idx paths.SectorIndex, max int) (*lpwindow.WdPostTask, *lpwindow.WdPostSubmitTask, *lpwindow.WdPostRecoverDeclareTask, error) {
chainSched := chainsched.New(api)
// todo config
ft := lpwindow.NewSimpleFaultTracker(stor, idx, 32, 5*time.Second, 300*time.Second)
computeTask, err := lpwindow.NewWdPostTask(db, api, ft, lw, verif, chainSched, addresses, max)
if err != nil {
return nil, nil, nil, err
}
submitTask, err := lpwindow.NewWdPostSubmitTask(chainSched, sender, db, api, fc.MaxWindowPoStGasFee, as)
if err != nil {
return nil, nil, nil, err
}
recoverTask, err := lpwindow.NewWdPostRecoverDeclareTask(sender, db, api, ft, as, chainSched, fc.MaxWindowPoStGasFee, addresses)
if err != nil {
return nil, nil, nil, err
}
go chainSched.Run(ctx)
return computeTask, submitTask, recoverTask, nil
}

136
provider/chainsched/chain_sched.go Normal file
View File

@ -0,0 +1,136 @@
package chainsched
import (
"context"
"time"
logging "github.com/ipfs/go-log/v2"
"go.opencensus.io/trace"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
)
var log = logging.Logger("chainsched")
type NodeAPI interface {
ChainHead(context.Context) (*types.TipSet, error)
ChainNotify(context.Context) (<-chan []*api.HeadChange, error)
}
type ProviderChainSched struct {
api NodeAPI
callbacks []UpdateFunc
started bool
}
func New(api NodeAPI) *ProviderChainSched {
return &ProviderChainSched{
api: api,
}
}
type UpdateFunc func(ctx context.Context, revert, apply *types.TipSet) error
func (s *ProviderChainSched) AddHandler(ch UpdateFunc) error {
if s.started {
return xerrors.Errorf("cannot add handler after start")
}
s.callbacks = append(s.callbacks, ch)
return nil
}
func (s *ProviderChainSched) Run(ctx context.Context) {
s.started = true
var (
notifs <-chan []*api.HeadChange
err error
gotCur bool
)
// not fine to panic after this point
for {
if notifs == nil {
notifs, err = s.api.ChainNotify(ctx)
if err != nil {
log.Errorf("ChainNotify error: %+v", err)
build.Clock.Sleep(10 * time.Second)
continue
}
gotCur = false
log.Info("restarting window post scheduler")
}
select {
case changes, ok := <-notifs:
if !ok {
log.Warn("window post scheduler notifs channel closed")
notifs = nil
continue
}
if !gotCur {
if len(changes) != 1 {
log.Errorf("expected first notif to have len = 1")
continue
}
chg := changes[0]
if chg.Type != store.HCCurrent {
log.Errorf("expected first notif to tell current ts")
continue
}
ctx, span := trace.StartSpan(ctx, "ProviderChainSched.headChange")
s.update(ctx, nil, chg.Val)
span.End()
gotCur = true
continue
}
ctx, span := trace.StartSpan(ctx, "ProviderChainSched.headChange")
var lowest, highest *types.TipSet = nil, nil
for _, change := range changes {
if change.Val == nil {
log.Errorf("change.Val was nil")
}
switch change.Type {
case store.HCRevert:
lowest = change.Val
case store.HCApply:
highest = change.Val
}
}
s.update(ctx, lowest, highest)
span.End()
case <-ctx.Done():
return
}
}
}
func (s *ProviderChainSched) update(ctx context.Context, revert, apply *types.TipSet) {
if apply == nil {
log.Error("no new tipset in window post ProviderChainSched.update")
return
}
for _, ch := range s.callbacks {
if err := ch(ctx, revert, apply); err != nil {
log.Errorf("handling head updates in provider chain sched: %+v", err)
}
}
}

Some files were not shown because too many files have changed in this diff Show More