cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #5 from filecoin-project/master

Browse Source 
merge
This commit is contained in:
chunqizhi 2020-07-03 20:15:19 +08:00 committed by GitHub
commit 3e7654d459
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
84 changed files with 3448 additions and 714 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -35,3 +35,4 @@ build/paramfetch.sh
bin/ipget
bin/tmp/*
.idea
scratchpad

8
Makefile
View File

@ -6,7 +6,7 @@ all: build
unexport GOFLAGS
GOVERSION:=$(shell go version | cut -d' ' -f 3 | cut -d. -f 2)
ifeq ($(shell expr $(GOVERSION) \< 13), 1)
ifeq ($(shell expr $(GOVERSION) \< 14), 1)
$(warning Your Golang version is go 1.$(GOVERSION))
$(error Update Golang to version $(shell grep '^go' go.mod))
endif
@ -191,6 +191,12 @@ health:
.PHONY: health
BINS+=health
testground:
go build -tags testground -o /dev/null ./cmd/lotus
.PHONY: testground
BINS+=testground
# MISC
buildall: $(BINS)

2
api/api_full.go
View File

@ -206,7 +206,7 @@ type FullNode interface {
StateCall(context.Context, *types.Message, types.TipSetKey) (*InvocResult, error)
StateReplay(context.Context, types.TipSetKey, cid.Cid) (*InvocResult, error)
StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error)
StateReadState(ctx context.Context, act *types.Actor, tsk types.TipSetKey) (*ActorState, error)
StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error)
StateListMessages(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error)
StateNetworkName(context.Context) (dtypes.NetworkName, error)

9
api/api_storage.go
View File

@ -56,9 +56,16 @@ type StorageMiner interface {
DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error
DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error)
DealsSetAcceptingStorageDeals(context.Context, bool) error
DealsConsiderOnlineStorageDeals(context.Context) (bool, error)
DealsSetConsiderOnlineStorageDeals(context.Context, bool) error
DealsConsiderOnlineRetrievalDeals(context.Context) (bool, error)
DealsSetConsiderOnlineRetrievalDeals(context.Context, bool) error
DealsPieceCidBlocklist(context.Context) ([]cid.Cid, error)
DealsSetPieceCidBlocklist(context.Context, []cid.Cid) error
DealsConsiderOfflineStorageDeals(context.Context) (bool, error)
DealsSetConsiderOfflineStorageDeals(context.Context, bool) error
DealsConsiderOfflineRetrievalDeals(context.Context) (bool, error)
DealsSetConsiderOfflineRetrievalDeals(context.Context, bool) error
StorageAddLocal(ctx context.Context, path string) error
}

55
api/apistruct/struct.go
View File

@ -139,7 +139,7 @@ type FullNodeStruct struct {
StateCall func(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) `perm:"read"`
StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"`
StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"`
StateReadState func(context.Context, *types.Actor, types.TipSetKey) (*api.ActorState, error) `perm:"read"`
StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"`
StatePledgeCollateral func(context.Context, types.TipSetKey) (types.BigInt, error) `perm:"read"`
StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"`
StateSearchMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"`
@ -224,11 +224,18 @@ type StorageMinerStruct struct {
StorageLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error `perm:"admin"`
StorageTryLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) (bool, error) `perm:"admin"`
DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"`
DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"`
DealsSetAcceptingStorageDeals func(context.Context, bool) error `perm:"admin"`
DealsPieceCidBlocklist func(context.Context) ([]cid.Cid, error) `perm:"admin"`
DealsSetPieceCidBlocklist func(context.Context, []cid.Cid) error `perm:"read"`
DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"`
DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"`
DealsConsiderOnlineStorageDeals func(context.Context) (bool, error) `perm:"read"`
DealsSetConsiderOnlineStorageDeals func(context.Context, bool) error `perm:"admin"`
DealsConsiderOnlineRetrievalDeals func(context.Context) (bool, error) `perm:"read"`
DealsSetConsiderOnlineRetrievalDeals func(context.Context, bool) error `perm:"admin"`
DealsConsiderOfflineStorageDeals func(context.Context) (bool, error) `perm:"read"`
DealsSetConsiderOfflineStorageDeals func(context.Context, bool) error `perm:"admin"`
DealsConsiderOfflineRetrievalDeals func(context.Context) (bool, error) `perm:"read"`
DealsSetConsiderOfflineRetrievalDeals func(context.Context, bool) error `perm:"admin"`
DealsPieceCidBlocklist func(context.Context) ([]cid.Cid, error) `perm:"read"`
DealsSetPieceCidBlocklist func(context.Context, []cid.Cid) error `perm:"admin"`
StorageAddLocal func(ctx context.Context, path string) error `perm:"admin"`
}
@ -617,8 +624,8 @@ func (c *FullNodeStruct) StateGetActor(ctx context.Context, actor address.Addres
return c.Internal.StateGetActor(ctx, actor, tsk)
}
func (c *FullNodeStruct) StateReadState(ctx context.Context, act *types.Actor, tsk types.TipSetKey) (*api.ActorState, error) {
return c.Internal.StateReadState(ctx, act, tsk)
func (c *FullNodeStruct) StateReadState(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*api.ActorState, error) {
return c.Internal.StateReadState(ctx, addr, tsk)
}
func (c *FullNodeStruct) StatePledgeCollateral(ctx context.Context, tsk types.TipSetKey) (types.BigInt, error) {
@ -877,8 +884,20 @@ func (c *StorageMinerStruct) DealsList(ctx context.Context) ([]storagemarket.Sto
return c.Internal.DealsList(ctx)
}
func (c *StorageMinerStruct) DealsSetAcceptingStorageDeals(ctx context.Context, b bool) error {
return c.Internal.DealsSetAcceptingStorageDeals(ctx, b)
func (c *StorageMinerStruct) DealsConsiderOnlineStorageDeals(ctx context.Context) (bool, error) {
return c.Internal.DealsConsiderOnlineStorageDeals(ctx)
}
func (c *StorageMinerStruct) DealsSetConsiderOnlineStorageDeals(ctx context.Context, b bool) error {
return c.Internal.DealsSetConsiderOnlineStorageDeals(ctx, b)
}
func (c *StorageMinerStruct) DealsConsiderOnlineRetrievalDeals(ctx context.Context) (bool, error) {
return c.Internal.DealsConsiderOnlineRetrievalDeals(ctx)
}
func (c *StorageMinerStruct) DealsSetConsiderOnlineRetrievalDeals(ctx context.Context, b bool) error {
return c.Internal.DealsSetConsiderOnlineRetrievalDeals(ctx, b)
}
func (c *StorageMinerStruct) DealsPieceCidBlocklist(ctx context.Context) ([]cid.Cid, error) {
@ -889,6 +908,22 @@ func (c *StorageMinerStruct) DealsSetPieceCidBlocklist(ctx context.Context, cids
return c.Internal.DealsSetPieceCidBlocklist(ctx, cids)
}
func (c *StorageMinerStruct) DealsConsiderOfflineStorageDeals(ctx context.Context) (bool, error) {
return c.Internal.DealsConsiderOfflineStorageDeals(ctx)
}
func (c *StorageMinerStruct) DealsSetConsiderOfflineStorageDeals(ctx context.Context, b bool) error {
return c.Internal.DealsSetConsiderOfflineStorageDeals(ctx, b)
}
func (c *StorageMinerStruct) DealsConsiderOfflineRetrievalDeals(ctx context.Context) (bool, error) {
return c.Internal.DealsConsiderOfflineRetrievalDeals(ctx)
}
func (c *StorageMinerStruct) DealsSetConsiderOfflineRetrievalDeals(ctx context.Context, b bool) error {
return c.Internal.DealsSetConsiderOfflineRetrievalDeals(ctx, b)
}
func (c *StorageMinerStruct) StorageAddLocal(ctx context.Context, path string) error {
return c.Internal.StorageAddLocal(ctx, path)
}

16
api/docgen/docgen.go
View File

@ -16,6 +16,7 @@ import (
"github.com/filecoin-project/go-bitfield"
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/apistruct"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/modules/dtypes"
@ -334,6 +335,9 @@ func main() {
}
}
permStruct := reflect.TypeOf(apistruct.FullNodeStruct{}.Internal)
commonPermStruct := reflect.TypeOf(apistruct.CommonStruct{}.Internal)
for _, g := range groupslice {
g := g
fmt.Printf("## %s\n", g.GroupName)
@ -347,6 +351,18 @@ func main() {
fmt.Printf("### %s\n", m.Name)
fmt.Printf("%s\n\n", m.Comment)
meth, ok := permStruct.FieldByName(m.Name)
if !ok {
meth, ok = commonPermStruct.FieldByName(m.Name)
if !ok {
panic("no perms for method: " + m.Name)
}
}
perms := meth.Tag.Get("perm")
fmt.Printf("Perms: %s\n\n", perms)
if strings.Count(m.InputExample, "\n") > 0 {
fmt.Printf("Inputs:\n```json\n%s\n```\n\n", m.InputExample)
} else {

13
api/test/deals.go
View File

@ -8,6 +8,7 @@ import (
"math/rand"
"os"
"path/filepath"
"sync/atomic"
"testing"
"time"
@ -52,11 +53,11 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport
}
time.Sleep(time.Second)
mine := true
mine := int64(1)
done := make(chan struct{})
go func() {
defer close(done)
for mine {
for atomic.LoadInt64(&mine) == 1 {
time.Sleep(blocktime)
if err := sn[0].MineOne(ctx, func(bool) {}); err != nil {
t.Error(err)
@ -66,7 +67,7 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport
makeDeal(t, ctx, 6, client, miner, carExport)
mine = false
atomic.AddInt64(&mine, -1)
fmt.Println("shutting down mining")
<-done
}
@ -89,12 +90,12 @@ func TestDoubleDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) {
}
time.Sleep(time.Second)
mine := true
mine := int64(1)
done := make(chan struct{})
go func() {
defer close(done)
for mine {
for atomic.LoadInt64(&mine) == 1 {
time.Sleep(blocktime)
if err := sn[0].MineOne(ctx, func(bool) {}); err != nil {
t.Error(err)
@ -105,7 +106,7 @@ func TestDoubleDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) {
makeDeal(t, ctx, 6, client, miner, false)
makeDeal(t, ctx, 7, client, miner, false)
mine = false
atomic.AddInt64(&mine, -1)
fmt.Println("shutting down mining")
<-done
}

5
api/test/mining.go
View File

@ -126,6 +126,7 @@ func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExpo
minedTwo := make(chan struct{})
go func() {
doneMinedTwo := false
defer close(done)
prevExpect := 0
@ -175,9 +176,9 @@ func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExpo
time.Sleep(blocktime)
}
if prevExpect == 2 && expect == 2 && minedTwo != nil {
if prevExpect == 2 && expect == 2 && !doneMinedTwo {
close(minedTwo)
minedTwo = nil
doneMinedTwo = true
}
prevExpect = expect

5
build/params_2k.go
View File

@ -20,10 +20,9 @@ func init() {
BuildType |= Build2k
}
// Seconds
const BlockDelay = 2
const BlockDelaySecs = uint64(2)
const PropagationDelay = 3
const PropagationDelaySecs = uint64(3)
// SlashablePowerDelay is the number of epochs after ElectionPeriodStart, after
// which the miner is slashed

38
build/params_shared_funcs.go Normal file
View File

@ -0,0 +1,38 @@
package build
import (
"sort"
"github.com/libp2p/go-libp2p-core/protocol"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
func DefaultSectorSize() abi.SectorSize {
szs := make([]abi.SectorSize, 0, len(miner.SupportedProofTypes))
for spt := range miner.SupportedProofTypes {
ss, err := spt.SectorSize()
if err != nil {
panic(err)
}
szs = append(szs, ss)
}
sort.Slice(szs, func(i, j int) bool {
return szs[i] < szs[j]
})
return szs[0]
}
// Core network constants
func BlocksTopic(netName dtypes.NetworkName) string { return "/fil/blocks/" + string(netName) }
func MessagesTopic(netName dtypes.NetworkName) string { return "/fil/msgs/" + string(netName) }
func DhtProtocolName(netName dtypes.NetworkName) protocol.ID {
return protocol.ID("/fil/kad/" + string(netName))
}

52
build/params_shared.go → build/params_shared_vals.go
View File

@ -1,10 +1,9 @@
// +build !testground
package build
import (
"math/big"
"sort"
"github.com/libp2p/go-libp2p-core/protocol"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin"
@ -13,32 +12,6 @@ import (
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
func DefaultSectorSize() abi.SectorSize {
szs := make([]abi.SectorSize, 0, len(miner.SupportedProofTypes))
for spt := range miner.SupportedProofTypes {
ss, err := spt.SectorSize()
if err != nil {
panic(err)
}
szs = append(szs, ss)
}
sort.Slice(szs, func(i, j int) bool {
return szs[i] < szs[j]
})
return szs[0]
}
// Core network constants
func BlocksTopic(netName dtypes.NetworkName) string { return "/fil/blocks/" + string(netName) }
func MessagesTopic(netName dtypes.NetworkName) string { return "/fil/msgs/" + string(netName) }
func DhtProtocolName(netName dtypes.NetworkName) protocol.ID {
return protocol.ID("/fil/kad/" + string(netName))
}
// /////
// Storage
@ -48,8 +21,7 @@ const UnixfsLinksPerLevel = 1024
// /////
// Consensus / Network
// Seconds
const AllowableClockDrift = 1
const AllowableClockDriftSecs = uint64(1)
// Epochs
const ForkLengthThreshold = Finality
@ -59,12 +31,12 @@ var BlocksPerEpoch = uint64(builtin.ExpectedLeadersPerEpoch)
// Epochs
const Finality = miner.ChainFinalityish
const MessageConfidence = 5
const MessageConfidence = uint64(5)
// constants for Weight calculation
// The ratio of weight contributed by short-term vs long-term factors in a given round
const WRatioNum = int64(1)
const WRatioDen = 2
const WRatioDen = uint64(2)
// /////
// Proofs
@ -82,25 +54,25 @@ const MaxSealLookback = SealRandomnessLookbackLimit + 2000 // TODO: Get from spe
// Mining
// Epochs
const TicketRandomnessLookback = 1
const TicketRandomnessLookback = abi.ChainEpoch(1)
const WinningPoStSectorSetLookback = 10
const WinningPoStSectorSetLookback = abi.ChainEpoch(10)
// /////
// Devnet settings
const TotalFilecoin = 2_000_000_000
const MiningRewardTotal = 1_400_000_000
const TotalFilecoin = uint64(2_000_000_000)
const MiningRewardTotal = uint64(1_400_000_000)
const FilecoinPrecision = 1_000_000_000_000_000_000
const FilecoinPrecision = uint64(1_000_000_000_000_000_000)
var InitialRewardBalance *big.Int
// TODO: Move other important consts here
func init() {
InitialRewardBalance = big.NewInt(MiningRewardTotal)
InitialRewardBalance = InitialRewardBalance.Mul(InitialRewardBalance, big.NewInt(FilecoinPrecision))
InitialRewardBalance = big.NewInt(int64(MiningRewardTotal))
InitialRewardBalance = InitialRewardBalance.Mul(InitialRewardBalance, big.NewInt(int64(FilecoinPrecision)))
}
// Sync

73
build/params_testground.go Normal file
View File

@ -0,0 +1,73 @@
// +build testground
// This file makes hardcoded parameters (const) configurable as vars.
//
// Its purpose is to unlock various degrees of flexibility and parametrization
// when writing Testground plans for Lotus.
//
package build
import (
"math/big"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
)
var (
UnixfsChunkSize = uint64(1 << 20)
UnixfsLinksPerLevel = 1024
BlocksPerEpoch = uint64(builtin.ExpectedLeadersPerEpoch)
BlockMessageLimit = 512
BlockGasLimit = int64(100_000_000_000)
BlockDelaySecs = uint64(builtin.EpochDurationSeconds)
PropagationDelaySecs = uint64(6)
AllowableClockDriftSecs = uint64(1)
Finality = miner.ChainFinalityish
ForkLengthThreshold = Finality
SlashablePowerDelay = 20
InteractivePoRepConfidence = 6
MessageConfidence uint64 = 5
WRatioNum = int64(1)
WRatioDen = uint64(2)
BadBlockCacheSize = 1 << 15
BlsSignatureCacheSize = 40000
VerifSigCacheSize = 32000
SealRandomnessLookback = Finality
SealRandomnessLookbackLimit = SealRandomnessLookback + 2000
MaxSealLookback = SealRandomnessLookbackLimit + 2000
TicketRandomnessLookback = abi.ChainEpoch(1)
WinningPoStSectorSetLookback = abi.ChainEpoch(10)
TotalFilecoin uint64 = 2_000_000_000
MiningRewardTotal uint64 = 1_400_000_000
FilecoinPrecision uint64 = 1_000_000_000_000_000_000
InitialRewardBalance = func() *big.Int {
v := big.NewInt(int64(MiningRewardTotal))
v = v.Mul(v, big.NewInt(int64(FilecoinPrecision)))
return v
}()
DrandConfig = dtypes.DrandConfig{
Servers: []string{
"https://pl-eu.testnet.drand.sh",
"https://pl-us.testnet.drand.sh",
"https://pl-sin.testnet.drand.sh",
},
ChainInfoJSON: `{"public_key":"922a2e93828ff83345bae533f5172669a26c02dc76d6bf59c80892e12ab1455c229211886f35bb56af6d5bea981024df","period":25,"genesis_time":1590445175,"hash":"138a324aa6540f93d0dad002aa89454b1bec2b6e948682cde6bd4db40f4b7c9b"}`,
}
)

6
build/params_testnet.go
View File

@ -1,5 +1,6 @@
// +build !debug
// +build !2k
// +build !testground
package build
@ -19,7 +20,6 @@ func init() {
}
}
// Seconds
const BlockDelay = builtin.EpochDurationSeconds
const BlockDelaySecs = uint64(builtin.EpochDurationSeconds)
const PropagationDelay = 6
const PropagationDelaySecs = uint64(6)

4
build/version.go
View File

@ -25,7 +25,7 @@ func buildType() string {
}
// BuildVersion is the local build version, set by build system
const BuildVersion = "0.4.0"
const BuildVersion = "0.4.1"
func UserVersion() string {
return BuildVersion + buildType() + CurrentCommit
@ -53,7 +53,7 @@ func (ve Version) EqMajorMinor(v2 Version) bool {
}
// APIVersion is a semver version of the rpc api exposed
var APIVersion Version = newVer(0, 4, 0)
var APIVersion Version = newVer(0, 5, 0)
//nolint:varcheck,deadcode
const (

19
chain/events/events.go
View File

@ -51,7 +51,7 @@ type Events struct {
readyOnce sync.Once
heightEvents
calledEvents
*hcEvents
}
func NewEvents(ctx context.Context, api eventAPI) *Events {
@ -74,18 +74,7 @@ func NewEvents(ctx context.Context, api eventAPI) *Events {
htHeights: map[abi.ChainEpoch][]uint64{},
},
calledEvents: calledEvents{
cs: api,
tsc: tsc,
ctx: ctx,
gcConfidence: uint64(gcConfidence),
confQueue: map[triggerH]map[msgH][]*queuedEvent{},
revertQueue: map[msgH][]triggerH{},
triggers: map[triggerID]*callHandler{},
matchers: map[triggerID][]MatchFunc{},
timeouts: map[abi.ChainEpoch]map[triggerID]int{},
},
hcEvents: newHCEvents(ctx, api, tsc, uint64(gcConfidence)),
}
e.ready.Add(1)
@ -143,7 +132,7 @@ func (e *Events) listenHeadChangesOnce(ctx context.Context) error {
}
e.readyOnce.Do(func() {
e.at = cur[0].Val.Height()
e.lastTs = cur[0].Val
e.ready.Done()
})
@ -186,5 +175,5 @@ func (e *Events) headChange(rev, app []*types.TipSet) error {
return err
}
return e.headChangeCalled(rev, app)
return e.processHeadChangeEvent(rev, app)
}

472
chain/events/events_called.go
View File

@ -13,6 +13,7 @@ import (
)
const NoTimeout = math.MaxInt64
const NoHeight = abi.ChainEpoch(-1)
type triggerID = uint64
@ -23,54 +24,60 @@ type msgH = abi.ChainEpoch
// message (msgH+confidence)
type triggerH = abi.ChainEpoch
// CalledHandler arguments:
// `ts` is the tipset, in which the `msg` is included.
type eventData interface{}
// EventHandler arguments:
// `prevTs` is the previous tipset, eg the "from" tipset for a state change.
// `ts` is the event tipset, eg the tipset in which the `msg` is included.
// `curH`-`ts.Height` = `confidence`
type CalledHandler func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (more bool, err error)
type EventHandler func(data eventData, prevTs, ts *types.TipSet, curH abi.ChainEpoch) (more bool, err error)
// CheckFunc is used for atomicity guarantees. If the condition the callbacks
// wait for has already happened in tipset `ts`
//
// If `done` is true, timeout won't be triggered
// If `more` is false, no messages will be sent to CalledHandler (RevertHandler
// If `more` is false, no messages will be sent to EventHandler (RevertHandler
// may still be called)
type CheckFunc func(ts *types.TipSet) (done bool, more bool, err error)
type MatchFunc func(msg *types.Message) (bool, error)
type callHandler struct {
// Keep track of information for an event handler
type handlerInfo struct {
confidence int
timeout abi.ChainEpoch
disabled bool // TODO: GC after gcConfidence reached
handle CalledHandler
handle EventHandler
revert RevertHandler
}
// When a change occurs, a queuedEvent is created and put into a queue
// until the required confidence is reached
type queuedEvent struct {
trigger triggerID
h abi.ChainEpoch
msg *types.Message
prevH abi.ChainEpoch
h abi.ChainEpoch
data eventData
called bool
}
type calledEvents struct {
// Manages chain head change events, which may be forward (new tipset added to
// chain) or backward (chain branch discarded in favour of heavier branch)
type hcEvents struct {
cs eventAPI
tsc *tipSetCache
ctx context.Context
gcConfidence uint64
at abi.ChainEpoch
lastTs *types.TipSet
lk sync.Mutex
ctr triggerID
triggers map[triggerID]*callHandler
matchers map[triggerID][]MatchFunc
triggers map[triggerID]*handlerInfo
// maps block heights to events
// [triggerH][msgH][event]
@ -81,27 +88,79 @@ type calledEvents struct {
// [timeoutH+confidence][triggerID]{calls}
timeouts map[abi.ChainEpoch]map[triggerID]int
messageEvents
watcherEvents
}
func (e *calledEvents) headChangeCalled(rev, app []*types.TipSet) error {
func newHCEvents(ctx context.Context, cs eventAPI, tsc *tipSetCache, gcConfidence uint64) *hcEvents {
e := hcEvents{
ctx: ctx,
cs: cs,
tsc: tsc,
gcConfidence: gcConfidence,
confQueue: map[triggerH]map[msgH][]*queuedEvent{},
revertQueue: map[msgH][]triggerH{},
triggers: map[triggerID]*handlerInfo{},
timeouts: map[abi.ChainEpoch]map[triggerID]int{},
}
e.messageEvents = newMessageEvents(ctx, &e, cs)
e.watcherEvents = newWatcherEvents(ctx, &e, cs)
return &e
}
// Called when there is a change to the head with tipsets to be
// reverted / applied
func (e *hcEvents) processHeadChangeEvent(rev, app []*types.TipSet) error {
e.lk.Lock()
defer e.lk.Unlock()
for _, ts := range rev {
e.handleReverts(ts)
e.at = ts.Height()
e.lastTs = ts
}
for _, ts := range app {
// called triggers
e.checkNewCalls(ts)
for ; e.at <= ts.Height(); e.at++ {
e.applyWithConfidence(ts, e.at)
// Check if the head change caused any state changes that we were
// waiting for
stateChanges := e.watcherEvents.checkStateChanges(e.lastTs, ts)
// Queue up calls until there have been enough blocks to reach
// confidence on the state changes
for tid, data := range stateChanges {
e.queueForConfidence(tid, data, e.lastTs, ts)
}
// Check if the head change included any new message calls
newCalls, err := e.messageEvents.checkNewCalls(ts)
if err != nil {
return err
}
// Queue up calls until there have been enough blocks to reach
// confidence on the message calls
for tid, data := range newCalls {
e.queueForConfidence(tid, data, nil, ts)
}
for at := e.lastTs.Height(); at <= ts.Height(); at++ {
// Apply any queued events and timeouts that were targeted at the
// current chain height
e.applyWithConfidence(ts, at)
e.applyTimeouts(ts)
}
// Update the latest known tipset
e.lastTs = ts
}
return nil
}
func (e *calledEvents) handleReverts(ts *types.TipSet) {
func (e *hcEvents) handleReverts(ts *types.TipSet) {
reverts, ok := e.revertQueue[ts.Height()]
if !ok {
return // nothing to do
@ -117,7 +176,7 @@ func (e *calledEvents) handleReverts(ts *types.TipSet) {
trigger := e.triggers[event.trigger]
if err := trigger.revert(e.ctx, ts); err != nil {
log.Errorf("reverting chain trigger (call %s.%d() @H %d, called @ %d) failed: %s", event.msg.To, event.msg.Method, ts.Height(), triggerH, err)
log.Errorf("reverting chain trigger (@H %d, triggered @ %d) failed: %s", ts.Height(), triggerH, err)
}
}
delete(e.confQueue[triggerH], ts.Height())
@ -125,42 +184,15 @@ func (e *calledEvents) handleReverts(ts *types.TipSet) {
delete(e.revertQueue, ts.Height())
}
func (e *calledEvents) checkNewCalls(ts *types.TipSet) {
pts, err := e.cs.ChainGetTipSet(e.ctx, ts.Parents()) // we actually care about messages in the parent tipset here
if err != nil {
log.Errorf("getting parent tipset in checkNewCalls: %s", err)
return
}
e.messagesForTs(pts, func(msg *types.Message) {
// TODO: provide receipts
for tid, matchFns := range e.matchers {
var matched bool
for _, matchFn := range matchFns {
ok, err := matchFn(msg)
if err != nil {
log.Errorf("event matcher failed: %s", err)
continue
}
matched = ok
if matched {
break
}
}
if matched {
e.queueForConfidence(tid, msg, ts)
break
}
}
})
}
func (e *calledEvents) queueForConfidence(trigID uint64, msg *types.Message, ts *types.TipSet) {
// Queue up events until the chain has reached a height that reflects the
// desired confidence
func (e *hcEvents) queueForConfidence(trigID uint64, data eventData, prevTs, ts *types.TipSet) {
trigger := e.triggers[trigID]
prevH := NoHeight
if prevTs != nil {
prevH = prevTs.Height()
}
appliedH := ts.Height()
triggerH := appliedH + abi.ChainEpoch(trigger.confidence)
@ -173,17 +205,19 @@ func (e *calledEvents) queueForConfidence(trigID uint64, msg *types.Message, ts
byOrigH[appliedH] = append(byOrigH[appliedH], &queuedEvent{
trigger: trigID,
prevH: prevH,
h: appliedH,
msg: msg,
data: data,
})
e.revertQueue[appliedH] = append(e.revertQueue[appliedH], triggerH)
}
func (e *calledEvents) applyWithConfidence(ts *types.TipSet, height abi.ChainEpoch) {
// Apply any events that were waiting for this chain height for confidence
func (e *hcEvents) applyWithConfidence(ts *types.TipSet, height abi.ChainEpoch) {
byOrigH, ok := e.confQueue[height]
if !ok {
return // no triggers at thin height
return // no triggers at this height
}
for origH, events := range byOrigH {
@ -202,15 +236,20 @@ func (e *calledEvents) applyWithConfidence(ts *types.TipSet, height abi.ChainEpo
continue
}
rec, err := e.cs.StateGetReceipt(e.ctx, event.msg.Cid(), ts.Key())
if err != nil {
log.Error(err)
return
// Previous tipset - this is relevant for example in a state change
// from one tipset to another
var prevTs *types.TipSet
if event.prevH != NoHeight {
prevTs, err = e.tsc.get(event.prevH)
if err != nil {
log.Errorf("events: applyWithConfidence didn't find tipset for previous event; wanted %d; current %d", event.prevH, height)
continue
}
}
more, err := trigger.handle(event.msg, rec, triggerTs, height)
more, err := trigger.handle(event.data, prevTs, triggerTs, height)
if err != nil {
log.Errorf("chain trigger (call %s.%d() @H %d, called @ %d) failed: %s", event.msg.To, event.msg.Method, origH, height, err)
log.Errorf("chain trigger (@H %d, triggered @ %d) failed: %s", origH, height, err)
continue // don't revert failed calls
}
@ -226,7 +265,8 @@ func (e *calledEvents) applyWithConfidence(ts *types.TipSet, height abi.ChainEpo
}
}
func (e *calledEvents) applyTimeouts(ts *types.TipSet) {
// Apply any timeouts that expire at this height
func (e *hcEvents) applyTimeouts(ts *types.TipSet) {
triggers, ok := e.timeouts[ts.Height()]
if !ok {
return // nothing to do
@ -256,12 +296,225 @@ func (e *calledEvents) applyTimeouts(ts *types.TipSet) {
}
}
func (e *calledEvents) messagesForTs(ts *types.TipSet, consume func(*types.Message)) {
// Listen for an event
// - CheckFunc: immediately checks if the event already occurred
// - EventHandler: called when the event has occurred, after confidence tipsets
// - RevertHandler: called if the chain head changes causing the event to revert
// - confidence: wait this many tipsets before calling EventHandler
// - timeout: at this chain height, timeout on waiting for this event
func (e *hcEvents) onHeadChanged(check CheckFunc, hnd EventHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch) (triggerID, error) {
e.lk.Lock()
defer e.lk.Unlock()
// Check if the event has already occurred
ts := e.tsc.best()
done, more, err := check(ts)
if err != nil {
return 0, xerrors.Errorf("called check error (h: %d): %w", ts.Height(), err)
}
if done {
timeout = NoTimeout
}
// Create a trigger for the event
id := e.ctr
e.ctr++
e.triggers[id] = &handlerInfo{
confidence: confidence,
timeout: timeout + abi.ChainEpoch(confidence),
disabled: !more,
handle: hnd,
revert: rev,
}
// If there's a timeout, set up a timeout check at that height
if timeout != NoTimeout {
if e.timeouts[timeout+abi.ChainEpoch(confidence)] == nil {
e.timeouts[timeout+abi.ChainEpoch(confidence)] = map[uint64]int{}
}
e.timeouts[timeout+abi.ChainEpoch(confidence)][id] = 0
}
return id, nil
}
// headChangeAPI is used to allow the composed event APIs to call back to hcEvents
// to listen for changes
type headChangeAPI interface {
onHeadChanged(check CheckFunc, hnd EventHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch) (triggerID, error)
}
// watcherEvents watches for a state change
type watcherEvents struct {
ctx context.Context
cs eventAPI
hcAPI headChangeAPI
lk sync.RWMutex
matchers map[triggerID]StateMatchFunc
}
func newWatcherEvents(ctx context.Context, hcAPI headChangeAPI, cs eventAPI) watcherEvents {
return watcherEvents{
ctx: ctx,
cs: cs,
hcAPI: hcAPI,
matchers: make(map[triggerID]StateMatchFunc),
}
}
// Run each of the matchers against the previous and current state to see if
// there's a change
func (we *watcherEvents) checkStateChanges(oldState, newState *types.TipSet) map[triggerID]eventData {
we.lk.RLock()
defer we.lk.RUnlock()
res := make(map[triggerID]eventData)
for tid, matchFn := range we.matchers {
ok, data, err := matchFn(oldState, newState)
if err != nil {
log.Errorf("event diff fn failed: %s", err)
continue
}
if ok {
res[tid] = data
}
}
return res
}
// StateChange represents a change in state
type StateChange interface{}
// StateChangeHandler arguments:
// `oldTs` is the state "from" tipset
// `newTs` is the state "to" tipset
// `states` is the change in state
// `curH`-`ts.Height` = `confidence`
type StateChangeHandler func(oldTs, newTs *types.TipSet, states StateChange, curH abi.ChainEpoch) (more bool, err error)
type StateMatchFunc func(oldTs, newTs *types.TipSet) (bool, StateChange, error)
// StateChanged registers a callback which is triggered when a specified state
// change occurs or a timeout is reached.
//
// * `CheckFunc` callback is invoked immediately with a recent tipset, it
// returns two booleans - `done`, and `more`.
//
// * `done` should be true when some on-chain state change we are waiting
// for has happened. When `done` is set to true, timeout trigger is disabled.
//
// * `more` should be false when we don't want to receive new notifications
// through StateChangeHandler. Note that notifications may still be delivered to
// RevertHandler
//
// * `StateChangeHandler` is called when the specified state change was observed
// on-chain, and a confidence threshold was reached, or the specified `timeout`
// height was reached with no state change observed. When this callback is
// invoked on a timeout, `oldState` and `newState` are set to nil.
// This callback returns a boolean specifying whether further notifications
// should be sent, like `more` return param from `CheckFunc` above.
//
// * `RevertHandler` is called after apply handler, when we drop the tipset
// containing the message. The tipset passed as the argument is the tipset
// that is being dropped. Note that the event dropped may be re-applied
// in a different tipset in small amount of time.
//
// * `StateMatchFunc` is called against each tipset state. If there is a match,
// the state change is queued up until the confidence interval has elapsed (and
// `StateChangeHandler` is called)
func (we *watcherEvents) StateChanged(check CheckFunc, scHnd StateChangeHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch, mf StateMatchFunc) error {
hnd := func(data eventData, prevTs, ts *types.TipSet, height abi.ChainEpoch) (bool, error) {
states, ok := data.(StateChange)
if data != nil && !ok {
panic("expected StateChange")
}
return scHnd(prevTs, ts, states, height)
}
id, err := we.hcAPI.onHeadChanged(check, hnd, rev, confidence, timeout)
if err != nil {
return err
}
we.lk.Lock()
defer we.lk.Unlock()
we.matchers[id] = mf
return nil
}
// messageEvents watches for message calls to actors
type messageEvents struct {
ctx context.Context
cs eventAPI
hcAPI headChangeAPI
lk sync.RWMutex
matchers map[triggerID][]MsgMatchFunc
}
func newMessageEvents(ctx context.Context, hcAPI headChangeAPI, cs eventAPI) messageEvents {
return messageEvents{
ctx: ctx,
cs: cs,
hcAPI: hcAPI,
matchers: map[triggerID][]MsgMatchFunc{},
}
}
// Check if there are any new actor calls
func (me *messageEvents) checkNewCalls(ts *types.TipSet) (map[triggerID]eventData, error) {
pts, err := me.cs.ChainGetTipSet(me.ctx, ts.Parents()) // we actually care about messages in the parent tipset here
if err != nil {
log.Errorf("getting parent tipset in checkNewCalls: %s", err)
return nil, err
}
me.lk.RLock()
defer me.lk.RUnlock()
res := make(map[triggerID]eventData)
me.messagesForTs(pts, func(msg *types.Message) {
// TODO: provide receipts
for tid, matchFns := range me.matchers {
var matched bool
for _, matchFn := range matchFns {
ok, err := matchFn(msg)
if err != nil {
log.Errorf("event matcher failed: %s", err)
continue
}
matched = ok
if matched {
break
}
}
if matched {
res[tid] = msg
break
}
}
})
return res, nil
}
// Get the messages in a tipset
func (me *messageEvents) messagesForTs(ts *types.TipSet, consume func(*types.Message)) {
seen := map[cid.Cid]struct{}{}
for _, tsb := range ts.Blocks() {
msgs, err := e.cs.ChainGetBlockMessages(context.TODO(), tsb.Cid())
msgs, err := me.cs.ChainGetBlockMessages(context.TODO(), tsb.Cid())
if err != nil {
log.Errorf("messagesForTs MessagesForBlock failed (ts.H=%d, Bcid:%s, B.Mcid:%s): %s", ts.Height(), tsb.Cid(), tsb.Messages, err)
// this is quite bad, but probably better than missing all the other updates
@ -290,20 +543,27 @@ func (e *calledEvents) messagesForTs(ts *types.TipSet, consume func(*types.Messa
}
}
// Called registers a callbacks which are triggered when a specified method is
// MsgHandler arguments:
// `ts` is the tipset, in which the `msg` is included.
// `curH`-`ts.Height` = `confidence`
type MsgHandler func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (more bool, err error)
type MsgMatchFunc func(msg *types.Message) (bool, error)
// Called registers a callback which is triggered when a specified method is
// called on an actor, or a timeout is reached.
//
// * `CheckFunc` callback is invoked immediately with a recent tipset, it
// returns two booleans - `done`, and `more`.
//
// * `done` should be true when some on-chain action we are waiting for has
// happened. When `done` is set to true, timeout trigger is disabled.
// * `done` should be true when some on-chain action we are waiting for has
// happened. When `done` is set to true, timeout trigger is disabled.
//
// * `more` should be false when we don't want to receive new notifications
// through CalledHandler. Note that notifications may still be delivered to
// RevertHandler
// * `more` should be false when we don't want to receive new notifications
// through MsgHandler. Note that notifications may still be delivered to
// RevertHandler
//
// * `CalledHandler` is called when the specified event was observed on-chain,
// * `MsgHandler` is called when the specified event was observed on-chain,
// and a confidence threshold was reached, or the specified `timeout` height
// was reached with no events observed. When this callback is invoked on a
// timeout, `msg` is set to nil. This callback returns a boolean specifying
@ -314,44 +574,38 @@ func (e *calledEvents) messagesForTs(ts *types.TipSet, consume func(*types.Messa
// containing the message. The tipset passed as the argument is the tipset
// that is being dropped. Note that the message dropped may be re-applied
// in a different tipset in small amount of time.
func (e *calledEvents) Called(check CheckFunc, hnd CalledHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch, mf MatchFunc) error {
e.lk.Lock()
defer e.lk.Unlock()
ts := e.tsc.best()
done, more, err := check(ts)
if err != nil {
return xerrors.Errorf("called check error (h: %d): %w", ts.Height(), err)
}
if done {
timeout = NoTimeout
}
id := e.ctr
e.ctr++
e.triggers[id] = &callHandler{
confidence: confidence,
timeout: timeout + abi.ChainEpoch(confidence),
disabled: !more,
handle: hnd,
revert: rev,
}
e.matchers[id] = append(e.matchers[id], mf)
if timeout != NoTimeout {
if e.timeouts[timeout+abi.ChainEpoch(confidence)] == nil {
e.timeouts[timeout+abi.ChainEpoch(confidence)] = map[uint64]int{}
//
// * `MsgMatchFunc` is called against each message. If there is a match, the
// message is queued up until the confidence interval has elapsed (and
// `MsgHandler` is called)
func (me *messageEvents) Called(check CheckFunc, msgHnd MsgHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch, mf MsgMatchFunc) error {
hnd := func(data eventData, prevTs, ts *types.TipSet, height abi.ChainEpoch) (bool, error) {
msg, ok := data.(*types.Message)
if data != nil && !ok {
panic("expected msg")
}
e.timeouts[timeout+abi.ChainEpoch(confidence)][id] = 0
rec, err := me.cs.StateGetReceipt(me.ctx, msg.Cid(), ts.Key())
if err != nil {
return false, err
}
return msgHnd(msg, rec, ts, height)
}
id, err := me.hcAPI.onHeadChanged(check, hnd, rev, confidence, timeout)
if err != nil {
return err
}
me.lk.Lock()
defer me.lk.Unlock()
me.matchers[id] = append(me.matchers[id], mf)
return nil
}
func (e *calledEvents) CalledMsg(ctx context.Context, hnd CalledHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch, msg types.ChainMsg) error {
return e.Called(e.CheckMsg(ctx, msg, hnd), hnd, rev, confidence, timeout, e.MatchMsg(msg.VMMessage()))
// Convenience function for checking and matching messages
func (me *messageEvents) CalledMsg(ctx context.Context, hnd MsgHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch, msg types.ChainMsg) error {
return me.Called(me.CheckMsg(ctx, msg, hnd), hnd, rev, confidence, timeout, me.MatchMsg(msg.VMMessage()))
}

14
chain/events/events_height.go
View File

@ -26,12 +26,15 @@ type heightEvents struct {
}
func (e *heightEvents) headChangeAt(rev, app []*types.TipSet) error {
ctx, span := trace.StartSpan(e.ctx, "events.HeightHeadChange")
defer span.End()
span.AddAttributes(trace.Int64Attribute("endHeight", int64(app[0].Height())))
span.AddAttributes(trace.Int64Attribute("reverts", int64(len(rev))))
span.AddAttributes(trace.Int64Attribute("applies", int64(len(app))))
e.lk.Lock()
defer e.lk.Unlock()
for _, ts := range rev {
// TODO: log error if h below gcconfidence
// revert height-based triggers
@ -40,7 +43,10 @@ func (e *heightEvents) headChangeAt(rev, app []*types.TipSet) error {
for _, tid := range e.htHeights[h] {
ctx, span := trace.StartSpan(ctx, "events.HeightRevert")
err := e.heightTriggers[tid].revert(ctx, ts)
rev := e.heightTriggers[tid].revert
e.lk.Unlock()
err := rev(ctx, ts)
e.lk.Lock()
e.heightTriggers[tid].called = false
span.End()
@ -98,8 +104,10 @@ func (e *heightEvents) headChangeAt(rev, app []*types.TipSet) error {
ctx, span := trace.StartSpan(ctx, "events.HeightApply")
span.AddAttributes(trace.BoolAttribute("immediate", false))
err = hnd.handle(ctx, incTs, h)
handle := hnd.handle
e.lk.Unlock()
err = handle(ctx, incTs, h)
e.lk.Lock()
span.End()
if err != nil {

249
chain/events/events_test.go
View File

@ -1004,8 +1004,6 @@ func TestRemoveTriggersOnMessage(t *testing.T) {
return false, true, nil
}, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (bool, error) {
require.Equal(t, false, applied)
fmt.Println(msg == nil)
fmt.Println(curH)
applied = true
return more, nil
}, func(_ context.Context, ts *types.TipSet) error {
@ -1067,3 +1065,250 @@ func TestRemoveTriggersOnMessage(t *testing.T) {
require.Equal(t, true, applied)
require.Equal(t, false, reverted)
}
type testStateChange struct {
from string
to string
}
func TestStateChanged(t *testing.T) {
fcs := &fakeCS{
t: t,
h: 1,
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
events := NewEvents(context.Background(), fcs)
more := true
var applied, reverted bool
var appliedData StateChange
var appliedOldTs *types.TipSet
var appliedNewTs *types.TipSet
var appliedH abi.ChainEpoch
var matchData StateChange
confidence := 3
timeout := abi.ChainEpoch(20)
err := events.StateChanged(func(ts *types.TipSet) (d bool, m bool, e error) {
return false, true, nil
}, func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) {
require.Equal(t, false, applied)
applied = true
appliedData = data
appliedOldTs = oldTs
appliedNewTs = newTs
appliedH = curH
return more, nil
}, func(_ context.Context, ts *types.TipSet) error {
reverted = true
return nil
}, confidence, timeout, func(oldTs, newTs *types.TipSet) (bool, StateChange, error) {
if matchData == nil {
return false, matchData, nil
}
d := matchData
matchData = nil
return true, d, nil
})
require.NoError(t, err)
// create few blocks to make sure nothing get's randomly called
fcs.advance(0, 4, nil) // H=5
require.Equal(t, false, applied)
require.Equal(t, false, reverted)
// create state change (but below confidence threshold)
matchData = testStateChange{from: "a", to: "b"}
fcs.advance(0, 3, nil)
require.Equal(t, false, applied)
require.Equal(t, false, reverted)
// create additional block so we are above confidence threshold
fcs.advance(0, 2, nil) // H=10 (confidence=3, apply)
require.Equal(t, true, applied)
require.Equal(t, false, reverted)
applied = false
// dip below confidence (should not apply again)
fcs.advance(2, 2, nil) // H=10 (confidence=3, apply)
require.Equal(t, false, applied)
require.Equal(t, false, reverted)
// Change happens from 5 -> 6
require.Equal(t, abi.ChainEpoch(5), appliedOldTs.Height())
require.Equal(t, abi.ChainEpoch(6), appliedNewTs.Height())
// Actually applied (with confidence) at 9
require.Equal(t, abi.ChainEpoch(9), appliedH)
// Make sure the state change was correctly passed through
rcvd := appliedData.(testStateChange)
require.Equal(t, "a", rcvd.from)
require.Equal(t, "b", rcvd.to)
}
func TestStateChangedRevert(t *testing.T) {
fcs := &fakeCS{
t: t,
h: 1,
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
events := NewEvents(context.Background(), fcs)
more := true
var applied, reverted bool
var matchData StateChange
confidence := 1
timeout := abi.ChainEpoch(20)
err := events.StateChanged(func(ts *types.TipSet) (d bool, m bool, e error) {
return false, true, nil
}, func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) {
require.Equal(t, false, applied)
applied = true
return more, nil
}, func(_ context.Context, ts *types.TipSet) error {
reverted = true
return nil
}, confidence, timeout, func(oldTs, newTs *types.TipSet) (bool, StateChange, error) {
if matchData == nil {
return false, matchData, nil
}
d := matchData
matchData = nil
return true, d, nil
})
require.NoError(t, err)
fcs.advance(0, 2, nil) // H=3
// Make a state change from TS at height 3 to TS at height 4
matchData = testStateChange{from: "a", to: "b"}
fcs.advance(0, 1, nil) // H=4
// Haven't yet reached confidence
require.Equal(t, false, applied)
require.Equal(t, false, reverted)
// Advance to reach confidence level
fcs.advance(0, 1, nil) // H=5
// Should now have called the handler
require.Equal(t, true, applied)
require.Equal(t, false, reverted)
applied = false
// Advance 3 more TS
fcs.advance(0, 3, nil) // H=8
require.Equal(t, false, applied)
require.Equal(t, false, reverted)
// Regress but not so far as to cause a revert
fcs.advance(3, 1, nil) // H=6
require.Equal(t, false, applied)
require.Equal(t, false, reverted)
// Regress back to state where change happened
fcs.advance(3, 1, nil) // H=4
// Expect revert to have happened
require.Equal(t, false, applied)
require.Equal(t, true, reverted)
}
func TestStateChangedTimeout(t *testing.T) {
fcs := &fakeCS{
t: t,
h: 1,
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
events := NewEvents(context.Background(), fcs)
called := false
err := events.StateChanged(func(ts *types.TipSet) (d bool, m bool, e error) {
return false, true, nil
}, func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) {
called = true
require.Nil(t, data)
require.Equal(t, abi.ChainEpoch(20), newTs.Height())
require.Equal(t, abi.ChainEpoch(23), curH)
return false, nil
}, func(_ context.Context, ts *types.TipSet) error {
t.Fatal("revert on timeout")
return nil
}, 3, 20, func(oldTs, newTs *types.TipSet) (bool, StateChange, error) {
return false, nil, nil
})
require.NoError(t, err)
fcs.advance(0, 21, nil)
require.False(t, called)
fcs.advance(0, 5, nil)
require.True(t, called)
called = false
// with check func reporting done
fcs = &fakeCS{
t: t,
h: 1,
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
events = NewEvents(context.Background(), fcs)
err = events.StateChanged(func(ts *types.TipSet) (d bool, m bool, e error) {
return true, true, nil
}, func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) {
called = true
require.Nil(t, data)
require.Equal(t, abi.ChainEpoch(20), newTs.Height())
require.Equal(t, abi.ChainEpoch(23), curH)
return false, nil
}, func(_ context.Context, ts *types.TipSet) error {
t.Fatal("revert on timeout")
return nil
}, 3, 20, func(oldTs, newTs *types.TipSet) (bool, StateChange, error) {
return false, nil, nil
})
require.NoError(t, err)
fcs.advance(0, 21, nil)
require.False(t, called)
fcs.advance(0, 5, nil)
require.False(t, called)
}

137
chain/events/state/predicates.go Normal file
View File

@ -0,0 +1,137 @@
package state
import (
"context"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-amt-ipld/v2"
"github.com/filecoin-project/lotus/api/apibstore"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/market"
"github.com/ipfs/go-cid"
cbor "github.com/ipfs/go-ipld-cbor"
)
// UserData is the data returned from the DiffFunc
type UserData interface{}
// ChainAPI abstracts out calls made by this class to external APIs
type ChainAPI interface {
apibstore.ChainIO
StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error)
}
// StatePredicates has common predicates for responding to state changes
type StatePredicates struct {
api ChainAPI
cst *cbor.BasicIpldStore
}
func NewStatePredicates(api ChainAPI) *StatePredicates {
return &StatePredicates{
api: api,
cst: cbor.NewCborStore(apibstore.NewAPIBlockstore(api)),
}
}
// DiffFunc check if there's a change form oldState to newState, and returns
// - changed: was there a change
// - user: user-defined data representing the state change
// - err
type DiffFunc func(ctx context.Context, oldState, newState *types.TipSet) (changed bool, user UserData, err error)
type DiffStateFunc func(ctx context.Context, oldActorStateHead, newActorStateHead cid.Cid) (changed bool, user UserData, err error)
// OnActorStateChanged calls diffStateFunc when the state changes for the given actor
func (sp *StatePredicates) OnActorStateChanged(addr address.Address, diffStateFunc DiffStateFunc) DiffFunc {
return func(ctx context.Context, oldState, newState *types.TipSet) (changed bool, user UserData, err error) {
oldActor, err := sp.api.StateGetActor(ctx, addr, oldState.Key())
if err != nil {
return false, nil, err
}
newActor, err := sp.api.StateGetActor(ctx, addr, newState.Key())
if err != nil {
return false, nil, err
}
if oldActor.Head.Equals(newActor.Head) {
return false, nil, nil
}
return diffStateFunc(ctx, oldActor.Head, newActor.Head)
}
}
type DiffStorageMarketStateFunc func(ctx context.Context, oldState *market.State, newState *market.State) (changed bool, user UserData, err error)
// OnStorageMarketActorChanged calls diffStorageMarketState when the state changes for the market actor
func (sp *StatePredicates) OnStorageMarketActorChanged(diffStorageMarketState DiffStorageMarketStateFunc) DiffFunc {
return sp.OnActorStateChanged(builtin.StorageMarketActorAddr, func(ctx context.Context, oldActorStateHead, newActorStateHead cid.Cid) (changed bool, user UserData, err error) {
var oldState market.State
if err := sp.cst.Get(ctx, oldActorStateHead, &oldState); err != nil {
return false, nil, err
}
var newState market.State
if err := sp.cst.Get(ctx, newActorStateHead, &newState); err != nil {
return false, nil, err
}
return diffStorageMarketState(ctx, &oldState, &newState)
})
}
type DiffDealStatesFunc func(ctx context.Context, oldDealStateRoot *amt.Root, newDealStateRoot *amt.Root) (changed bool, user UserData, err error)
// OnDealStateChanged calls diffDealStates when the market state changes
func (sp *StatePredicates) OnDealStateChanged(diffDealStates DiffDealStatesFunc) DiffStorageMarketStateFunc {
return func(ctx context.Context, oldState *market.State, newState *market.State) (changed bool, user UserData, err error) {
if oldState.States.Equals(newState.States) {
return false, nil, nil
}
oldRoot, err := amt.LoadAMT(ctx, sp.cst, oldState.States)
if err != nil {
return false, nil, err
}
newRoot, err := amt.LoadAMT(ctx, sp.cst, newState.States)
if err != nil {
return false, nil, err
}
return diffDealStates(ctx, oldRoot, newRoot)
}
}
// ChangedDeals is a set of changes to deal state
type ChangedDeals map[abi.DealID]DealStateChange
// DealStateChange is a change in deal state from -> to
type DealStateChange struct {
From market.DealState
To market.DealState
}
// DealStateChangedForIDs detects changes in the deal state AMT for the given deal IDs
func (sp *StatePredicates) DealStateChangedForIDs(dealIds []abi.DealID) DiffDealStatesFunc {
return func(ctx context.Context, oldDealStateRoot *amt.Root, newDealStateRoot *amt.Root) (changed bool, user UserData, err error) {
changedDeals := make(ChangedDeals)
for _, dealID := range dealIds {
var oldDeal, newDeal market.DealState
err := oldDealStateRoot.Get(ctx, uint64(dealID), &oldDeal)
if err != nil {
return false, nil, err
}
err = newDealStateRoot.Get(ctx, uint64(dealID), &newDeal)
if err != nil {
return false, nil, err
}
if oldDeal != newDeal {
changedDeals[dealID] = DealStateChange{oldDeal, newDeal}
}
}
if len(changedDeals) > 0 {
return true, changedDeals, nil
}
return false, nil, nil
}
}

201
chain/events/state/predicates_test.go Normal file
View File

@ -0,0 +1,201 @@
package state
import (
"context"
"testing"
"github.com/filecoin-project/specs-actors/actors/crypto"
"github.com/ipfs/go-hamt-ipld"
"github.com/filecoin-project/go-amt-ipld/v2"
"github.com/filecoin-project/specs-actors/actors/builtin/market"
ds "github.com/ipfs/go-datastore"
ds_sync "github.com/ipfs/go-datastore/sync"
bstore "github.com/ipfs/go-ipfs-blockstore"
cbornode "github.com/ipfs/go-ipld-cbor"
"golang.org/x/xerrors"
"github.com/ipfs/go-cid"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/specs-actors/actors/abi"
)
var dummyCid cid.Cid
func init() {
dummyCid, _ = cid.Parse("bafkqaaa")
}
type mockAPI struct {
ts map[types.TipSetKey]*types.Actor
bs bstore.Blockstore
}
func newMockAPI(bs bstore.Blockstore) *mockAPI {
return &mockAPI{
bs: bs,
ts: make(map[types.TipSetKey]*types.Actor),
}
}
func (m mockAPI) ChainHasObj(ctx context.Context, c cid.Cid) (bool, error) {
return m.bs.Has(c)
}
func (m mockAPI) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) {
blk, err := m.bs.Get(c)
if err != nil {
return nil, xerrors.Errorf("blockstore get: %w", err)
}
return blk.RawData(), nil
}
func (m mockAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) {
return m.ts[tsk], nil
}
func (m mockAPI) setActor(tsk types.TipSetKey, act *types.Actor) {
m.ts[tsk] = act
}
func TestPredicates(t *testing.T) {
ctx := context.Background()
bs := bstore.NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
store := cbornode.NewCborStore(bs)
oldDeals := map[abi.DealID]*market.DealState{
abi.DealID(1): {
SectorStartEpoch: 1,
LastUpdatedEpoch: 2,
SlashEpoch: 0,
},
abi.DealID(2): {
SectorStartEpoch: 4,
LastUpdatedEpoch: 5,
SlashEpoch: 0,
},
}
oldStateC := createMarketState(ctx, t, store, oldDeals)
newDeals := map[abi.DealID]*market.DealState{
abi.DealID(1): {
SectorStartEpoch: 1,
LastUpdatedEpoch: 3,
SlashEpoch: 0,
},
abi.DealID(2): {
SectorStartEpoch: 4,
LastUpdatedEpoch: 6,
SlashEpoch: 6,
},
}
newStateC := createMarketState(ctx, t, store, newDeals)
miner, err := address.NewFromString("t00")
require.NoError(t, err)
oldState, err := mockTipset(miner, 1)
require.NoError(t, err)
newState, err := mockTipset(miner, 2)
require.NoError(t, err)
api := newMockAPI(bs)
api.setActor(oldState.Key(), &types.Actor{Head: oldStateC})
api.setActor(newState.Key(), &types.Actor{Head: newStateC})
preds := NewStatePredicates(api)
dealIds := []abi.DealID{abi.DealID(1), abi.DealID(2)}
diffFn := preds.OnStorageMarketActorChanged(preds.OnDealStateChanged(preds.DealStateChangedForIDs(dealIds)))
// Diff a state against itself: expect no change
changed, _, err := diffFn(ctx, oldState, oldState)
require.NoError(t, err)
require.False(t, changed)
// Diff old state against new state
changed, val, err := diffFn(ctx, oldState, newState)
require.NoError(t, err)
require.True(t, changed)
changedDeals, ok := val.(ChangedDeals)
require.True(t, ok)
require.Len(t, changedDeals, 2)
require.Contains(t, changedDeals, abi.DealID(1))
require.Contains(t, changedDeals, abi.DealID(2))
deal1 := changedDeals[abi.DealID(1)]
if deal1.From.LastUpdatedEpoch != 2 || deal1.To.LastUpdatedEpoch != 3 {
t.Fatal("Unexpected change to LastUpdatedEpoch")
}
deal2 := changedDeals[abi.DealID(2)]
if deal2.From.SlashEpoch != 0 || deal2.To.SlashEpoch != 6 {
t.Fatal("Unexpected change to SlashEpoch")
}
// Test that OnActorStateChanged does not call the callback if the state has not changed
mockAddr, err := address.NewFromString("t01")
require.NoError(t, err)
actorDiffFn := preds.OnActorStateChanged(mockAddr, func(context.Context, cid.Cid, cid.Cid) (bool, UserData, error) {
t.Fatal("No state change so this should not be called")
return false, nil, nil
})
changed, _, err = actorDiffFn(ctx, oldState, oldState)
require.NoError(t, err)
require.False(t, changed)
// Test that OnDealStateChanged does not call the callback if the state has not changed
diffDealStateFn := preds.OnDealStateChanged(func(context.Context, *amt.Root, *amt.Root) (bool, UserData, error) {
t.Fatal("No state change so this should not be called")
return false, nil, nil
})
marketState := createEmptyMarketState(t, store)
changed, _, err = diffDealStateFn(ctx, marketState, marketState)
require.NoError(t, err)
require.False(t, changed)
}
func mockTipset(miner address.Address, timestamp uint64) (*types.TipSet, error) {
return types.NewTipSet([]*types.BlockHeader{{
Miner: miner,
Height: 5,
ParentStateRoot: dummyCid,
Messages: dummyCid,
ParentMessageReceipts: dummyCid,
BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS},
BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS},
Timestamp: timestamp,
}})
}
func createMarketState(ctx context.Context, t *testing.T, store *cbornode.BasicIpldStore, deals map[abi.DealID]*market.DealState) cid.Cid {
rootCid := createAMT(ctx, t, store, deals)
state := createEmptyMarketState(t, store)
state.States = rootCid
stateC, err := store.Put(ctx, state)
require.NoError(t, err)
return stateC
}
func createEmptyMarketState(t *testing.T, store *cbornode.BasicIpldStore) *market.State {
emptyArrayCid, err := amt.NewAMT(store).Flush(context.TODO())
require.NoError(t, err)
emptyMap, err := store.Put(context.TODO(), hamt.NewNode(store, hamt.UseTreeBitWidth(5)))
require.NoError(t, err)
return market.ConstructState(emptyArrayCid, emptyMap, emptyMap)
}
func createAMT(ctx context.Context, t *testing.T, store *cbornode.BasicIpldStore, deals map[abi.DealID]*market.DealState) cid.Cid {
root := amt.NewAMT(store)
for dealID, dealState := range deals {
err := root.Set(ctx, uint64(dealID), dealState)
require.NoError(t, err)
}
rootCid, err := root.Flush(ctx)
require.NoError(t, err)
return rootCid
}

8
chain/events/utils.go
View File

@ -8,11 +8,11 @@ import (
"github.com/filecoin-project/lotus/chain/types"
)
func (e *calledEvents) CheckMsg(ctx context.Context, smsg types.ChainMsg, hnd CalledHandler) CheckFunc {
func (me *messageEvents) CheckMsg(ctx context.Context, smsg types.ChainMsg, hnd MsgHandler) CheckFunc {
msg := smsg.VMMessage()
return func(ts *types.TipSet) (done bool, more bool, err error) {
fa, err := e.cs.StateGetActor(ctx, msg.From, ts.Key())
fa, err := me.cs.StateGetActor(ctx, msg.From, ts.Key())
if err != nil {
return false, true, err
}
@ -22,7 +22,7 @@ func (e *calledEvents) CheckMsg(ctx context.Context, smsg types.ChainMsg, hnd Ca
return false, true, nil
}
rec, err := e.cs.StateGetReceipt(ctx, smsg.VMMessage().Cid(), ts.Key())
rec, err := me.cs.StateGetReceipt(ctx, smsg.VMMessage().Cid(), ts.Key())
if err != nil {
return false, true, xerrors.Errorf("getting receipt in CheckMsg: %w", err)
}
@ -33,7 +33,7 @@ func (e *calledEvents) CheckMsg(ctx context.Context, smsg types.ChainMsg, hnd Ca
}
}
func (e *calledEvents) MatchMsg(inmsg *types.Message) MatchFunc {
func (me *messageEvents) MatchMsg(inmsg *types.Message) MsgMatchFunc {
return func(msg *types.Message) (bool, error) {
if msg.From == inmsg.From && msg.Nonce == inmsg.Nonce && !inmsg.Equals(msg) {
return false, xerrors.Errorf("matching msg %s from %s, nonce %d: got duplicate origin/nonce msg %d", inmsg.Cid(), inmsg.From, inmsg.Nonce, msg.Nonce)

6
chain/gen/gen.go
View File

@ -196,7 +196,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
*genm2,
},
NetworkName: "",
Timestamp: uint64(time.Now().Add(-500 * build.BlockDelay * time.Second).Unix()),
Timestamp: uint64(time.Now().Add(-500 * time.Duration(build.BlockDelaySecs) * time.Second).Unix()),
}
genb, err := genesis2.MakeGenesisBlock(context.TODO(), bs, sys, tpl)
@ -223,7 +223,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
miners := []address.Address{maddr1, maddr2}
beac := beacon.NewMockBeacon(time.Second)
//beac, err := drand.NewDrandBeacon(tpl.Timestamp, build.BlockDelay)
//beac, err := drand.NewDrandBeacon(tpl.Timestamp, build.BlockDelaySecs)
//if err != nil {
//return nil, xerrors.Errorf("creating drand beacon: %w", err)
//}
@ -414,7 +414,7 @@ func (cg *ChainGen) makeBlock(parents *types.TipSet, m address.Address, vrfticke
if cg.Timestamper != nil {
ts = cg.Timestamper(parents, height-parents.Height())
} else {
ts = parents.MinTimestamp() + uint64((height-parents.Height())*build.BlockDelay)
ts = parents.MinTimestamp() + uint64(height-parents.Height())*build.BlockDelaySecs
}
fblk, err := MinerCreateBlock(context.TODO(), cg.sm, cg.w, &api.BlockTemplate{

11
chain/gen/mining.go
View File

@ -150,6 +150,17 @@ func aggregateSignatures(sigs []crypto.Signature) (*crypto.Signature, error) {
}
aggSig := bls.Aggregate(blsSigs)
if aggSig == nil {
if len(sigs) > 0 {
return nil, xerrors.Errorf("bls.Aggregate returned nil with %d signatures", len(sigs))
}
return &crypto.Signature{
Type: crypto.SigTypeBLS,
Data: new(bls.Signature)[:],
}, nil
}
return &crypto.Signature{
Type: crypto.SigTypeBLS,
Data: aggSig[:],

2
chain/messagepool/messagepool.go
View File

@ -187,7 +187,7 @@ func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName) (*Messa
mp := &MessagePool{
closer: make(chan struct{}),
repubTk: time.NewTicker(build.BlockDelay * 10 * time.Second),
repubTk: time.NewTicker(time.Duration(build.BlockDelaySecs) * 10 * time.Second),
localAddrs: make(map[address.Address]struct{}),
pending: make(map[address.Address]*msgSet),
minGasPrice: types.NewInt(0),

55
chain/stmgr/utils.go
View File

@ -13,6 +13,7 @@ import (
"github.com/filecoin-project/go-address"
amt "github.com/filecoin-project/go-amt-ipld/v2"
"github.com/filecoin-project/go-bitfield"
"github.com/filecoin-project/sector-storage/ffiwrapper"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin"
@ -173,13 +174,34 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S
return nil, xerrors.Errorf("(get sectors) failed to load miner actor state: %w", err)
}
// TODO: Optimization: we could avoid loaditg the whole proving set here if we had AMT.GetNth with bitfield filtering
sectorSet, err := GetProvingSetRaw(ctx, sm, mas)
if err != nil {
return nil, xerrors.Errorf("getting proving set: %w", err)
cst := cbor.NewCborStore(sm.cs.Blockstore())
var deadlines miner.Deadlines
if err := cst.Get(ctx, mas.Deadlines, &deadlines); err != nil {
return nil, xerrors.Errorf("failed to load deadlines: %w", err)
}
if len(sectorSet) == 0 {
notProving, err := abi.BitFieldUnion(mas.Faults, mas.Recoveries)
if err != nil {
return nil, xerrors.Errorf("failed to union faults and recoveries: %w", err)
}
allSectors, err := bitfield.MultiMerge(append(deadlines.Due[:], mas.NewSectors)...)
if err != nil {
return nil, xerrors.Errorf("merging deadline bitfields failed: %w", err)
}
provingSectors, err := bitfield.SubtractBitField(allSectors, notProving)
if err != nil {
return nil, xerrors.Errorf("failed to subtract non-proving sectors from set: %w", err)
}
numProvSect, err := provingSectors.Count()
if err != nil {
return nil, xerrors.Errorf("failed to count bits: %w", err)
}
// TODO(review): is this right? feels fishy to me
if numProvSect == 0 {
return nil, nil
}
@ -198,17 +220,34 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S
return nil, xerrors.Errorf("getting miner ID: %w", err)
}
ids, err := pv.GenerateWinningPoStSectorChallenge(ctx, wpt, abi.ActorID(mid), rand, uint64(len(sectorSet)))
ids, err := pv.GenerateWinningPoStSectorChallenge(ctx, wpt, abi.ActorID(mid), rand, numProvSect)
if err != nil {
return nil, xerrors.Errorf("generating winning post challenges: %w", err)
}
sectors, err := provingSectors.All(miner.SectorsMax)
if err != nil {
return nil, xerrors.Errorf("failed to enumerate all sector IDs: %w", err)
}
sectorAmt, err := amt.LoadAMT(ctx, cst, mas.Sectors)
if err != nil {
return nil, xerrors.Errorf("failed to load sectors amt: %w", err)
}
out := make([]abi.SectorInfo, len(ids))
for i, n := range ids {
sid := sectors[n]
var sinfo miner.SectorOnChainInfo
if err := sectorAmt.Get(ctx, sid, &sinfo); err != nil {
return nil, xerrors.Errorf("failed to get sector %d: %w", sid, err)
}
out[i] = abi.SectorInfo{
SealProof: spt,
SectorNumber: sectorSet[n].ID,
SealedCID: sectorSet[n].Info.Info.SealedCID,
SectorNumber: sinfo.Info.SectorNumber,
SealedCID: sinfo.Info.SealedCID,
}
}

27
chain/sync.go
View File

@ -49,6 +49,10 @@ import (
"github.com/filecoin-project/lotus/metrics"
)
// Blocks that are more than MaxHeightDrift epochs above
//the theoretical max height based on systime are quickly rejected
const MaxHeightDrift = 5
var log = logging.Logger("chain")
var LocalIncoming = "incoming"
@ -157,6 +161,11 @@ func (syncer *Syncer) InformNewHead(from peer.ID, fts *store.FullTipSet) bool {
return false
}
if syncer.IsEpochBeyondCurrMax(fts.TipSet().Height()) {
log.Errorf("Received block with impossibly large height %d", fts.TipSet().Height())
return false
}
for _, b := range fts.Blocks {
if reason, ok := syncer.bad.Has(b.Cid()); ok {
log.Warnf("InformNewHead called on block marked as bad: %s (reason: %s)", b.Cid(), reason)
@ -657,18 +666,18 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
// fast checks first
now := uint64(time.Now().Unix())
if h.Timestamp > now+build.AllowableClockDrift {
if h.Timestamp > now+build.AllowableClockDriftSecs {
return xerrors.Errorf("block was from the future (now=%d, blk=%d): %w", now, h.Timestamp, ErrTemporal)
}
if h.Timestamp > now {
log.Warn("Got block from the future, but within threshold", h.Timestamp, time.Now().Unix())
}
if h.Timestamp < baseTs.MinTimestamp()+(build.BlockDelay*uint64(h.Height-baseTs.Height())) {
if h.Timestamp < baseTs.MinTimestamp()+(build.BlockDelaySecs*uint64(h.Height-baseTs.Height())) {
log.Warn("timestamp funtimes: ", h.Timestamp, baseTs.MinTimestamp(), h.Height, baseTs.Height())
diff := (baseTs.MinTimestamp() + (build.BlockDelay * uint64(h.Height-baseTs.Height()))) - h.Timestamp
diff := (baseTs.MinTimestamp() + (build.BlockDelaySecs * uint64(h.Height-baseTs.Height()))) - h.Timestamp
return xerrors.Errorf("block was generated too soon (h.ts:%d < base.mints:%d + BLOCK_DELAY:%d * deltaH:%d; diff %d)", h.Timestamp, baseTs.MinTimestamp(), build.BlockDelay, h.Height-baseTs.Height(), diff)
return xerrors.Errorf("block was generated too soon (h.ts:%d < base.mints:%d + BLOCK_DELAY:%d * deltaH:%d; diff %d)", h.Timestamp, baseTs.MinTimestamp(), build.BlockDelaySecs, h.Height-baseTs.Height(), diff)
}
msgsCheck := async.Err(func() error {
@ -1517,3 +1526,13 @@ func (syncer *Syncer) getLatestBeaconEntry(_ context.Context, ts *types.TipSet)
return nil, xerrors.Errorf("found NO beacon entries in the 20 blocks prior to given tipset")
}
func (syncer *Syncer) IsEpochBeyondCurrMax(epoch abi.ChainEpoch) bool {
g, err := syncer.store.GetGenesis()
if err != nil {
return false
}
now := uint64(time.Now().Unix())
return epoch > (abi.ChainEpoch((now-g.Timestamp)/build.BlockDelaySecs) + MaxHeightDrift)
}

2
chain/sync_test.go
View File

@ -408,7 +408,7 @@ func TestSyncBadTimestamp(t *testing.T) {
base := tu.g.CurTipset
tu.g.Timestamper = func(pts *types.TipSet, tl abi.ChainEpoch) uint64 {
return pts.MinTimestamp() + (build.BlockDelay / 2)
return pts.MinTimestamp() + (build.BlockDelaySecs / 2)
}
fmt.Println("BASE: ", base.Cids())

18
chain/types/execresult.go
View File

@ -21,16 +21,16 @@ type ExecutionTrace struct {
type GasTrace struct {
Name string
Location []Loc
TotalGas int64
ComputeGas int64
StorageGas int64
TotalVirtualGas int64
VirtualComputeGas int64
VirtualStorageGas int64
Location []Loc `json:"loc"`
TotalGas int64 `json:"tg"`
ComputeGas int64 `json:"cg"`
StorageGas int64 `json:"sg"`
TotalVirtualGas int64 `json:"vtg"`
VirtualComputeGas int64 `json:"vcg"`
VirtualStorageGas int64 `json:"vsg"`
TimeTaken time.Duration
Extra interface{} `json:",omitempty"`
TimeTaken time.Duration `json:"tt"`
Extra interface{} `json:"ex,omitempty"`
Callers []uintptr `json:"-"`
}

4
chain/types/fil.go
View File

@ -11,7 +11,7 @@ import (
type FIL BigInt
func (f FIL) String() string {
r := new(big.Rat).SetFrac(f.Int, big.NewInt(build.FilecoinPrecision))
r := new(big.Rat).SetFrac(f.Int, big.NewInt(int64(build.FilecoinPrecision)))
if r.Sign() == 0 {
return "0"
}
@ -33,7 +33,7 @@ func ParseFIL(s string) (FIL, error) {
return FIL{}, fmt.Errorf("failed to parse %q as a decimal number", s)
}
r = r.Mul(r, big.NewRat(build.FilecoinPrecision, 1))
r = r.Mul(r, big.NewRat(int64(build.FilecoinPrecision), 1))
if !r.IsInt() {
return FIL{}, fmt.Errorf("invalid FIL value: %q", s)
}

1
chain/types/message.go
View File

@ -21,6 +21,7 @@ type ChainMsg interface {
Cid() cid.Cid
VMMessage() *Message
ToStorageBlock() (block.Block, error)
// FIXME: This is the *message* length, this name is misleading.
ChainLength() int
}

24
chain/vm/gas.go
View File

@ -143,30 +143,40 @@ func (ps pricedSyscalls) VerifySignature(signature crypto.Signature, signer addr
return err
}
ps.chargeGas(c)
defer ps.chargeGas(gasOnActorExec)
return ps.under.VerifySignature(signature, signer, plaintext)
}
// Hashes input data using blake2b with 256 bit output.
func (ps pricedSyscalls) HashBlake2b(data []byte) [32]byte {
ps.chargeGas(ps.pl.OnHashing(len(data)))
defer ps.chargeGas(gasOnActorExec)
return ps.under.HashBlake2b(data)
}
// Computes an unsealed sector CID (CommD) from its constituent piece CIDs (CommPs) and sizes.
func (ps pricedSyscalls) ComputeUnsealedSectorCID(reg abi.RegisteredSealProof, pieces []abi.PieceInfo) (cid.Cid, error) {
ps.chargeGas(ps.pl.OnComputeUnsealedSectorCid(reg, pieces))
defer ps.chargeGas(gasOnActorExec)
return ps.under.ComputeUnsealedSectorCID(reg, pieces)
}
// Verifies a sector seal proof.
func (ps pricedSyscalls) VerifySeal(vi abi.SealVerifyInfo) error {
ps.chargeGas(ps.pl.OnVerifySeal(vi))
defer ps.chargeGas(gasOnActorExec)
return ps.under.VerifySeal(vi)
}
// Verifies a proof of spacetime.
func (ps pricedSyscalls) VerifyPoSt(vi abi.WindowPoStVerifyInfo) error {
ps.chargeGas(ps.pl.OnVerifyPost(vi))
defer ps.chargeGas(gasOnActorExec)
return ps.under.VerifyPoSt(vi)
}
@ -182,19 +192,21 @@ func (ps pricedSyscalls) VerifyPoSt(vi abi.WindowPoStVerifyInfo) error {
// Returns nil and an error if the headers don't prove a fault.
func (ps pricedSyscalls) VerifyConsensusFault(h1 []byte, h2 []byte, extra []byte) (*runtime.ConsensusFault, error) {
ps.chargeGas(ps.pl.OnVerifyConsensusFault())
defer ps.chargeGas(gasOnActorExec)
return ps.under.VerifyConsensusFault(h1, h2, extra)
}
func (ps pricedSyscalls) BatchVerifySeals(inp map[address.Address][]abi.SealVerifyInfo) (map[address.Address][]bool, error) {
var gasChargeSum GasCharge
gasChargeSum.Name = "BatchVerifySeals"
ps.chargeGas(gasChargeSum) // TODO: this is only called by the cron actor. Should we even charge gas?
count := int64(0)
for _, svis := range inp {
for _, svi := range svis {
ch := ps.pl.OnVerifySeal(svi)
ps.chargeGas(newGasCharge("BatchVerifySingle", 0, 0).WithVirtual(ch.VirtualCompute+ch.ComputeGas, 0))
}
count += int64(len(svis))
}
gasChargeSum = gasChargeSum.WithExtra(count).WithVirtual(129778623*count+716683250, 0)
ps.chargeGas(gasChargeSum) // TODO: this is only called by the cron actor. Should we even charge gas?
defer ps.chargeGas(gasOnActorExec)
return ps.under.BatchVerifySeals(inp)
}

56
chain/vm/gas_v0.go
View File

@ -4,6 +4,7 @@ import (
"fmt"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/crypto"
)
@ -86,39 +87,57 @@ var _ Pricelist = (*pricelistV0)(nil)
// OnChainMessage returns the gas used for storing a message of a given size in the chain.
func (pl *pricelistV0) OnChainMessage(msgSize int) GasCharge {
return newGasCharge("OnChainMessage", 0, pl.onChainMessageBase+pl.onChainMessagePerByte*int64(msgSize))
return newGasCharge("OnChainMessage", 0, pl.onChainMessageBase+pl.onChainMessagePerByte*int64(msgSize)).WithVirtual(77302, 0)
}
// OnChainReturnValue returns the gas used for storing the response of a message in the chain.
func (pl *pricelistV0) OnChainReturnValue(dataSize int) GasCharge {
return newGasCharge("OnChainReturnValue", 0, int64(dataSize)*pl.onChainReturnValuePerByte)
return newGasCharge("OnChainReturnValue", 0, int64(dataSize)*pl.onChainReturnValuePerByte).WithVirtual(107294, 0)
}
// OnMethodInvocation returns the gas used when invoking a method.
func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.MethodNum) GasCharge {
ret := pl.sendBase
extra := ""
virtGas := int64(1072944)
if value != abi.NewTokenAmount(0) {
// TODO: fix this, it is comparing pointers instead of values
// see vv
ret += pl.sendTransferFunds
}
if big.Cmp(value, abi.NewTokenAmount(0)) != 0 {
virtGas += 497495
if methodNum == builtin.MethodSend {
// transfer only
virtGas += 973940
}
extra += "t"
}
if methodNum != builtin.MethodSend {
ret += pl.sendInvokeMethod
extra += "i"
// running actors is cheaper becase we hand over to actors
virtGas += -295779
}
return newGasCharge("OnMethodInvocation", ret, 0).WithVirtual(ret*15000, 0)
return newGasCharge("OnMethodInvocation", ret, 0).WithVirtual(virtGas, 0).WithExtra(extra)
}
// OnIpldGet returns the gas used for storing an object
func (pl *pricelistV0) OnIpldGet(dataSize int) GasCharge {
return newGasCharge("OnIpldGet", pl.ipldGetBase+int64(dataSize)*pl.ipldGetPerByte, 0).WithExtra(dataSize).WithVirtual(pl.ipldGetBase*13750+(pl.ipldGetPerByte*100), 0)
return newGasCharge("OnIpldGet", pl.ipldGetBase+int64(dataSize)*pl.ipldGetPerByte, 0).
WithExtra(dataSize).WithVirtual(433685, 0)
}
// OnIpldPut returns the gas used for storing an object
func (pl *pricelistV0) OnIpldPut(dataSize int) GasCharge {
return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte).WithExtra(dataSize).WithVirtual(pl.ipldPutBase*8700+(pl.ipldPutPerByte*100), 0)
return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte).
WithExtra(dataSize).WithVirtual(88970, 0)
}
// OnCreateActor returns the gas used for creating an actor
func (pl *pricelistV0) OnCreateActor() GasCharge {
return newGasCharge("OnCreateActor", pl.createActorBase, pl.createActorExtra)
return newGasCharge("OnCreateActor", pl.createActorBase, pl.createActorExtra).WithVirtual(65636, 0)
}
// OnDeleteActor returns the gas used for deleting an actor
@ -127,39 +146,52 @@ func (pl *pricelistV0) OnDeleteActor() GasCharge {
}
// OnVerifySignature
func (pl *pricelistV0) OnVerifySignature(sigType crypto.SigType, planTextSize int) (GasCharge, error) {
costFn, ok := pl.verifySignature[sigType]
if !ok {
return GasCharge{}, fmt.Errorf("cost function for signature type %d not supported", sigType)
}
sigName, _ := sigType.Name()
return newGasCharge("OnVerifySignature", costFn(int64(planTextSize)), 0).WithExtra(sigName), nil
virtGas := int64(0)
switch sigType {
case crypto.SigTypeBLS:
virtGas = 220138570
case crypto.SigTypeSecp256k1:
virtGas = 7053730
}
return newGasCharge("OnVerifySignature", costFn(int64(planTextSize)), 0).
WithExtra(map[string]interface{}{
"type": sigName,
"size": planTextSize,
}).WithVirtual(virtGas, 0), nil
}
// OnHashing
func (pl *pricelistV0) OnHashing(dataSize int) GasCharge {
return newGasCharge("OnHashing", pl.hashingBase+int64(dataSize)*pl.hashingPerByte, 0)
return newGasCharge("OnHashing", pl.hashingBase+int64(dataSize)*pl.hashingPerByte, 0).WithExtra(dataSize).WithVirtual(77300, 0)
}
// OnComputeUnsealedSectorCid
func (pl *pricelistV0) OnComputeUnsealedSectorCid(proofType abi.RegisteredSealProof, pieces []abi.PieceInfo) GasCharge {
// TODO: this needs more cost tunning, check with @lotus
return newGasCharge("OnComputeUnsealedSectorCid", pl.computeUnsealedSectorCidBase, 0).WithVirtual(pl.computeUnsealedSectorCidBase*24500, 0)
return newGasCharge("OnComputeUnsealedSectorCid", pl.computeUnsealedSectorCidBase, 0).WithVirtual(382370, 0)
}
// OnVerifySeal
func (pl *pricelistV0) OnVerifySeal(info abi.SealVerifyInfo) GasCharge {
// TODO: this needs more cost tunning, check with @lotus
return newGasCharge("OnVerifySeal", pl.verifySealBase, 0).WithVirtual(pl.verifySealBase*177500, 0)
return newGasCharge("OnVerifySeal", pl.verifySealBase, 0).WithVirtual(199954003, 0)
}
// OnVerifyPost
func (pl *pricelistV0) OnVerifyPost(info abi.WindowPoStVerifyInfo) GasCharge {
// TODO: this needs more cost tunning, check with @lotus
return newGasCharge("OnVerifyPost", pl.verifyPostBase, 0)
return newGasCharge("OnVerifyPost", pl.verifyPostBase, 0).WithVirtual(2629471704, 0).WithExtra(len(info.ChallengedSectors))
}
// OnVerifyConsensusFault
func (pl *pricelistV0) OnVerifyConsensusFault() GasCharge {
return newGasCharge("OnVerifyConsensusFault", pl.verifyConsensusFault, 0)
return newGasCharge("OnVerifyConsensusFault", pl.verifyConsensusFault, 0).WithVirtual(551935, 0)
}

8
chain/vm/runtime.go
View File

@ -264,6 +264,7 @@ func (rt *Runtime) CreateActor(codeID cid.Cid, address address.Address) {
if err != nil {
panic(aerrors.Fatalf("creating actor entry: %v", err))
}
_ = rt.chargeGasSafe(gasOnActorExec)
}
func (rt *Runtime) DeleteActor(addr address.Address) {
@ -284,10 +285,10 @@ func (rt *Runtime) DeleteActor(addr address.Address) {
if err := rt.state.DeleteActor(rt.Message().Receiver()); err != nil {
panic(aerrors.Fatalf("failed to delete actor: %s", err))
}
_ = rt.chargeGasSafe(gasOnActorExec)
}
func (rt *Runtime) Syscalls() vmr.Syscalls {
// TODO: Make sure this is wrapped in something that charges gas for each of the calls
return rt.sys
}
@ -312,7 +313,7 @@ func (rt *Runtime) Context() context.Context {
}
func (rt *Runtime) Abortf(code exitcode.ExitCode, msg string, args ...interface{}) {
log.Warnf("Abortf: ", fmt.Sprintf(msg, args...))
log.Warnf("Abortf: " + fmt.Sprintf(msg, args...))
panic(aerrors.NewfSkip(2, code, msg, args...))
}
@ -367,6 +368,7 @@ func (rt *Runtime) Send(to address.Address, method abi.MethodNum, m vmr.CBORMars
log.Warnf("vmctx send failed: to: %s, method: %d: ret: %d, err: %s", to, method, ret, err)
return nil, err.RetCode()
}
_ = rt.chargeGasSafe(gasOnActorExec)
return &dumbWrapperType{ret}, 0
}
@ -408,7 +410,7 @@ func (rt *Runtime) internalSend(from, to address.Address, method abi.MethodNum,
if subrt != nil {
rt.numActorsCreated = subrt.numActorsCreated
}
rt.executionTrace.Subcalls = append(rt.executionTrace.Subcalls, subrt.executionTrace) //&er)
rt.executionTrace.Subcalls = append(rt.executionTrace.Subcalls, subrt.executionTrace)
return ret, errSend
}

6
chain/vm/vm.go
View File

@ -35,6 +35,7 @@ import (
)
var log = logging.Logger("vm")
var gasOnActorExec = newGasCharge("OnActorExec", 0, 0)
// ResolveToKeyAddr returns the public key type of address (`BLS`/`SECP256K1`) of an account actor identified by `addr`.
func ResolveToKeyAddr(state types.StateTree, cst cbor.IpldStore, addr address.Address) (address.Address, aerrors.ActorError) {
@ -68,11 +69,13 @@ type gasChargingBlocks struct {
}
func (bs *gasChargingBlocks) Get(c cid.Cid) (block.Block, error) {
bs.chargeGas(newGasCharge("OnIpldGetStart", 0, 0))
blk, err := bs.under.Get(c)
if err != nil {
return nil, aerrors.Escalate(err, "failed to get block from blockstore")
}
bs.chargeGas(bs.pricelist.OnIpldGet(len(blk.RawData())))
bs.chargeGas(gasOnActorExec)
return blk, nil
}
@ -83,6 +86,7 @@ func (bs *gasChargingBlocks) Put(blk block.Block) error {
if err := bs.under.Put(blk); err != nil {
return aerrors.Escalate(err, "failed to write data to disk")
}
bs.chargeGas(gasOnActorExec)
return nil
}
@ -231,7 +235,9 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime,
if msg.Method != 0 {
var ret []byte
_ = rt.chargeGasSafe(gasOnActorExec)
ret, err := vm.Invoke(toActor, rt, msg.Method, msg.Params)
_ = rt.chargeGasSafe(newGasCharge("OnActorExecDone", 0, 0))
return ret, err
}
return nil, nil

7
cli/state.go
View File

@ -742,12 +742,7 @@ var stateReadStateCmd = &cli.Command{
return err
}
act, err := api.StateGetActor(ctx, addr, ts.Key())
if err != nil {
return err
}
as, err := api.StateReadState(ctx, act, ts.Key())
as, err := api.StateReadState(ctx, addr, ts.Key())
if err != nil {
return err
}

2
cli/sync.go
View File

@ -186,7 +186,7 @@ func SyncWait(ctx context.Context, napi api.FullNode) error {
fmt.Printf("\r\x1b[2KWorker %d: Target: %s\tState: %s\tHeight: %d", working, target, chain.SyncStageString(ss.Stage), ss.Height)
if time.Now().Unix()-int64(head.MinTimestamp()) < build.BlockDelay {
if time.Now().Unix()-int64(head.MinTimestamp()) < int64(build.BlockDelaySecs) {
fmt.Println("\nDone!")
return nil
}

401
cmd/lotus-bench/import.go
View File

@ -1,12 +1,15 @@
package main
import (
"bufio"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"math"
"net/http"
_ "net/http/pprof"
"os"
"runtime"
"runtime/pprof"
@ -149,6 +152,7 @@ var importBenchCmd = &cli.Command{
if err != nil {
return err
}
stripCallers(trace)
lastTse = &TipSetExec{
TipSet: cur.Key(),
@ -168,6 +172,21 @@ var importBenchCmd = &cli.Command{
},
}
func walkExecutionTrace(et *types.ExecutionTrace) {
for _, gc := range et.GasCharges {
gc.Callers = nil
}
for _, sub := range et.Subcalls {
walkExecutionTrace(&sub) //nolint:scopelint,gosec
}
}
func stripCallers(trace []*api.InvocResult) {
for _, t := range trace {
walkExecutionTrace(&t.ExecutionTrace)
}
}
type Invocation struct {
TipSet types.TipSetKey
Invoc *api.InvocResult
@ -210,19 +229,222 @@ func compStats(vals []float64) (float64, float64) {
return av, math.Sqrt(varsum / float64(len(vals)))
}
func tallyGasCharges(charges map[string][]float64, et *types.ExecutionTrace) {
for _, gc := range et.GasCharges {
type stats struct {
timeTaken meanVar
gasRatio meanVar
compGas := gc.ComputeGas + gc.VirtualComputeGas
ratio := float64(compGas) / float64(gc.TimeTaken.Nanoseconds())
extraCovar *covar
}
charges[gc.Name] = append(charges[gc.Name], 1/(ratio/GasPerNs))
//fmt.Printf("%s: %d, %s: %0.2f\n", gc.Name, compGas, gc.TimeTaken, 1/(ratio/GasPerNs))
for _, sub := range et.Subcalls {
tallyGasCharges(charges, &sub)
}
type covar struct {
meanX float64
meanY float64
c float64
n float64
m2x float64
m2y float64
}
func (cov1 *covar) Covariance() float64 {
return cov1.c / (cov1.n - 1)
}
func (cov1 *covar) VarianceX() float64 {
return cov1.m2x / (cov1.n - 1)
}
func (v1 *covar) StddevX() float64 {
return math.Sqrt(v1.VarianceX())
}
func (cov1 *covar) VarianceY() float64 {
return cov1.m2y / (cov1.n - 1)
}
func (v1 *covar) StddevY() float64 {
return math.Sqrt(v1.VarianceY())
}
func (cov1 *covar) AddPoint(x, y float64) {
cov1.n += 1
dx := x - cov1.meanX
cov1.meanX += dx / cov1.n
dx2 := x - cov1.meanX
cov1.m2x += dx * dx2
dy := y - cov1.meanY
cov1.meanY += dy / cov1.n
dy2 := y - cov1.meanY
cov1.m2y += dy * dy2
cov1.c += dx * dy
}
func (cov1 *covar) Combine(cov2 *covar) {
if cov1.n == 0 {
*cov1 = *cov2
return
}
if cov2.n == 0 {
return
}
if cov1.n == 1 {
cpy := *cov2
cpy.AddPoint(cov2.meanX, cov2.meanY)
*cov1 = cpy
return
}
if cov2.n == 1 {
cov1.AddPoint(cov2.meanX, cov2.meanY)
}
out := covar{}
out.n = cov1.n + cov2.n
dx := cov1.meanX - cov2.meanX
out.meanX = cov1.meanX - dx*cov2.n/out.n
out.m2x = cov1.m2x + cov2.m2x + dx*dx*cov1.n*cov2.n/out.n
dy := cov1.meanY - cov2.meanY
out.meanY = cov1.meanY - dy*cov2.n/out.n
out.m2y = cov1.m2y + cov2.m2y + dy*dy*cov1.n*cov2.n/out.n
out.c = cov1.c + cov2.c + dx*dy*cov1.n*cov2.n/out.n
*cov1 = out
}
func (cov1 *covar) A() float64 {
return cov1.Covariance() / cov1.VarianceX()
}
func (cov1 *covar) B() float64 {
return cov1.meanY - cov1.meanX*cov1.A()
}
func (cov1 *covar) Correl() float64 {
return cov1.Covariance() / cov1.StddevX() / cov1.StddevY()
}
type meanVar struct {
n float64
mean float64
m2 float64
}
func (v1 *meanVar) AddPoint(value float64) {
// based on https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm
v1.n += 1
delta := value - v1.mean
v1.mean += delta / v1.n
delta2 := value - v1.mean
v1.m2 += delta * delta2
}
func (v1 *meanVar) Variance() float64 {
return v1.m2 / (v1.n - 1)
}
func (v1 *meanVar) Mean() float64 {
return v1.mean
}
func (v1 *meanVar) Stddev() float64 {
return math.Sqrt(v1.Variance())
}
func (v1 *meanVar) Combine(v2 *meanVar) {
if v1.n == 0 {
*v1 = *v2
return
}
if v2.n == 0 {
return
}
if v1.n == 1 {
cpy := *v2
cpy.AddPoint(v1.mean)
*v1 = cpy
return
}
if v2.n == 1 {
v1.AddPoint(v2.mean)
return
}
newCount := v1.n + v2.n
delta := v2.mean - v1.mean
meanDelta := delta * v2.n / newCount
m2 := v1.m2 + v2.m2 + delta*meanDelta*v1.n
v1.n = newCount
v1.mean += meanDelta
v1.m2 = m2
}
func getExtras(ex interface{}) (*string, *float64) {
if t, ok := ex.(string); ok {
return &t, nil
}
if size, ok := ex.(float64); ok {
return nil, &size
}
if exMap, ok := ex.(map[string]interface{}); ok {
t, tok := exMap["type"].(string)
size, sok := exMap["size"].(float64)
if tok && sok {
return &t, &size
}
if tok {
return &t, nil
}
if sok {
return nil, &size
}
return nil, nil
}
return nil, nil
}
func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
for i, gc := range et.GasCharges {
name := gc.Name
if name == "OnIpldGetStart" {
continue
}
tt := float64(gc.TimeTaken.Nanoseconds())
if name == "OnIpldGet" {
prev := et.GasCharges[i-1]
if prev.Name != "OnIpldGetStart" {
log.Warn("OnIpldGet without OnIpldGetStart")
}
tt += float64(prev.TimeTaken.Nanoseconds())
}
eType, eSize := getExtras(gc.Extra)
if eType != nil {
name += "-" + *eType
}
compGas := gc.VirtualComputeGas
if compGas == 0 {
compGas = 1
}
s := charges[name]
if s == nil {
s = new(stats)
charges[name] = s
}
if eSize != nil {
if s.extraCovar == nil {
s.extraCovar = &covar{}
}
s.extraCovar.AddPoint(*eSize, tt)
}
s.timeTaken.AddPoint(tt)
ratio := tt / float64(compGas) * GasPerNs
s.gasRatio.AddPoint(ratio)
}
for _, sub := range et.Subcalls {
tallyGasCharges(charges, sub)
}
}
var importAnalyzeCmd = &cli.Command{
@ -233,57 +455,143 @@ var importAnalyzeCmd = &cli.Command{
return nil
}
go func() {
http.ListenAndServe("localhost:6060", nil)
}()
fi, err := os.Open(cctx.Args().First())
if err != nil {
return err
}
var results []TipSetExec
for {
var tse TipSetExec
if err := json.NewDecoder(fi).Decode(&tse); err != nil {
if err != io.EOF {
return err
}
break
}
results = append(results, tse)
const nWorkers = 16
jsonIn := make(chan []byte, 2*nWorkers)
type result struct {
totalTime time.Duration
chargeStats map[string]*stats
expensiveInvocs []Invocation
}
chargeDeltas := make(map[string][]float64)
results := make(chan result, nWorkers)
for i := 0; i < nWorkers; i++ {
go func() {
chargeStats := make(map[string]*stats)
var totalTime time.Duration
const invocsKeep = 32
var expensiveInvocs = make([]Invocation, 0, 8*invocsKeep)
var leastExpensiveInvoc = time.Duration(0)
for {
b, ok := <-jsonIn
if !ok {
results <- result{
totalTime: totalTime,
chargeStats: chargeStats,
expensiveInvocs: expensiveInvocs,
}
return
}
var tse TipSetExec
err := json.Unmarshal(b, &tse)
if err != nil {
log.Warnf("error unmarshaling tipset: %+v", err)
continue
}
totalTime += tse.Duration
for _, inv := range tse.Trace {
if inv.Duration > leastExpensiveInvoc {
expensiveInvocs = append(expensiveInvocs, Invocation{
TipSet: tse.TipSet,
Invoc: inv,
})
}
tallyGasCharges(chargeStats, inv.ExecutionTrace)
}
if len(expensiveInvocs) > 4*invocsKeep {
sort.Slice(expensiveInvocs, func(i, j int) bool {
return expensiveInvocs[i].Invoc.Duration > expensiveInvocs[j].Invoc.Duration
})
leastExpensiveInvoc = expensiveInvocs[len(expensiveInvocs)-1].Invoc.Duration
n := 30
if len(expensiveInvocs) < n {
n = len(expensiveInvocs)
}
expensiveInvocs = expensiveInvocs[:n]
}
}
}()
}
var totalTipsets int64
reader := bufio.NewReader(fi)
for {
b, err := reader.ReadBytes('\n')
if err != nil && err != io.EOF {
if e, ok := err.(*json.SyntaxError); ok {
log.Warnf("syntax error at byte offset %d", e.Offset)
}
return err
}
totalTipsets++
jsonIn <- b
fmt.Fprintf(os.Stderr, "\rProcessed %d tipsets", totalTipsets)
if err == io.EOF {
break
}
}
close(jsonIn)
fmt.Fprintf(os.Stderr, "\n")
fmt.Fprintf(os.Stderr, "Collecting results\n")
var invocs []Invocation
var totalTime time.Duration
for i, r := range results {
_ = i
totalTime += r.Duration
for _, inv := range r.Trace {
invocs = append(invocs, Invocation{
TipSet: r.TipSet,
Invoc: inv,
})
cgas, vgas := countGasCosts(&inv.ExecutionTrace)
fmt.Printf("Invocation: %d %s: %s %d -> %0.2f\n", inv.Msg.Method, inv.Msg.To, inv.Duration, cgas+vgas, float64(GasPerNs*inv.Duration.Nanoseconds())/float64(cgas+vgas))
tallyGasCharges(chargeDeltas, &inv.ExecutionTrace)
var keys []string
var charges = make(map[string]*stats)
for i := 0; i < nWorkers; i++ {
fmt.Fprintf(os.Stderr, "\rProcessing results from worker %d/%d", i+1, nWorkers)
res := <-results
invocs = append(invocs, res.expensiveInvocs...)
for k, v := range res.chargeStats {
s := charges[k]
if s == nil {
s = new(stats)
charges[k] = s
}
s.timeTaken.Combine(&v.timeTaken)
s.gasRatio.Combine(&v.gasRatio)
if v.extraCovar != nil {
if s.extraCovar == nil {
s.extraCovar = &covar{}
}
s.extraCovar.Combine(v.extraCovar)
}
}
totalTime += res.totalTime
}
var keys []string
for k := range chargeDeltas {
fmt.Fprintf(os.Stderr, "\nCollecting gas keys\n")
for k := range charges {
keys = append(keys, k)
}
fmt.Println("Gas Price Deltas")
sort.Strings(keys)
for _, k := range keys {
vals := chargeDeltas[k]
av, stdev := compStats(vals)
fmt.Printf("%s: incr by %f (%f)\n", k, av, stdev)
s := charges[k]
fmt.Printf("%s: incr by %.4f~%.4f; tt %.4f~%.4f\n", k, s.gasRatio.Mean(), s.gasRatio.Stddev(),
s.timeTaken.Mean(), s.timeTaken.Stddev())
if s.extraCovar != nil {
fmt.Printf("\t correll: %.2f, tt = %.2f * extra + %.2f\n", s.extraCovar.Correl(),
s.extraCovar.A(), s.extraCovar.B())
fmt.Printf("\t covar: %.2f, extra: %.2f~%.2f, tt2: %.2f~%.2f, count %.0f\n",
s.extraCovar.Covariance(), s.extraCovar.meanX, s.extraCovar.StddevX(),
s.extraCovar.meanY, s.extraCovar.StddevY(), s.extraCovar.n)
}
}
sort.Slice(invocs, func(i, j int) bool {
@ -291,9 +599,20 @@ var importAnalyzeCmd = &cli.Command{
})
fmt.Println("Total time: ", totalTime)
fmt.Println("Average time per epoch: ", totalTime/time.Duration(len(results)))
fmt.Println("Average time per epoch: ", totalTime/time.Duration(totalTipsets))
if actorExec, ok := charges["OnActorExec"]; ok {
timeInActors := actorExec.timeTaken.Mean() * actorExec.timeTaken.n
fmt.Printf("Avarage time per epoch in actors: %s (%.1f%%)\n", time.Duration(timeInActors)/time.Duration(totalTipsets), timeInActors/float64(totalTime)*100)
}
if actorExecDone, ok := charges["OnActorExecDone"]; ok {
timeInActors := actorExecDone.timeTaken.Mean() * actorExecDone.timeTaken.n
fmt.Printf("Avarage time per epoch in OnActorExecDone %s (%.1f%%)\n", time.Duration(timeInActors)/time.Duration(totalTipsets), timeInActors/float64(totalTime)*100)
}
n := 30
if len(invocs) < n {
n = len(invocs)
}
fmt.Printf("Top %d most expensive calls:\n", n)
for i := 0; i < n; i++ {
inv := invocs[i].Invoc

47
cmd/lotus-bench/stats_test.go Normal file
View File

@ -0,0 +1,47 @@
package main
import (
"math/rand"
"testing"
)
func TestMeanVar(t *testing.T) {
N := 16
ss := make([]*meanVar, N)
rng := rand.New(rand.NewSource(1))
for i := 0; i < N; i++ {
ss[i] = &meanVar{}
maxJ := rng.Intn(1000)
for j := 0; j < maxJ; j++ {
ss[i].AddPoint(rng.NormFloat64()*5 + 500)
}
t.Logf("mean: %f, stddev: %f, count %f", ss[i].mean, ss[i].Stddev(), ss[i].n)
}
out := &meanVar{}
for i := 0; i < N; i++ {
out.Combine(ss[i])
t.Logf("combine: mean: %f, stddev: %f", out.mean, out.Stddev())
}
}
func TestCovar(t *testing.T) {
N := 16
ss := make([]*covar, N)
rng := rand.New(rand.NewSource(1))
for i := 0; i < N; i++ {
ss[i] = &covar{}
maxJ := rng.Intn(1000) + 500
for j := 0; j < maxJ; j++ {
x := rng.NormFloat64()*5 + 500
ss[i].AddPoint(x, x*2-1000)
}
t.Logf("corell: %f, y = %f*x+%f @%.0f", ss[i].Correl(), ss[i].A(), ss[i].B(), ss[i].n)
t.Logf("\txVar: %f yVar: %f covar: %f", ss[i].StddevX(), ss[i].StddevY(), ss[i].Covariance())
}
out := &covar{}
for i := 0; i < N; i++ {
out.Combine(ss[i])
t.Logf("combine: corell: %f, y = %f*x+%f", out.Correl(), out.A(), out.B())
t.Logf("\txVar: %f yVar: %f covar: %f", out.StddevX(), out.StddevY(), out.Covariance())
}
}

3
cmd/lotus-chainwatch/main.go
View File

@ -18,6 +18,9 @@ var log = logging.Logger("chainwatch")
func main() {
_ = logging.SetLogLevel("*", "INFO")
if err := logging.SetLogLevel("rpc", "error"); err != nil {
panic(err)
}
log.Info("Starting chainwatch")

2
cmd/lotus-chainwatch/mpool.go
View File

@ -46,7 +46,7 @@ func subMpool(ctx context.Context, api aapi.FullNode, st *storage) {
msgs[v.Message.Message.Cid()] = &v.Message.Message
}
log.Infof("Processing %d mpool updates", len(msgs))
log.Debugf("Processing %d mpool updates", len(msgs))
err := st.storeMessages(msgs)
if err != nil {

386
cmd/lotus-chainwatch/storage.go
View File

@ -1,11 +1,17 @@
package main
import (
"context"
"database/sql"
"fmt"
"github.com/filecoin-project/specs-actors/actors/util/adt"
"github.com/libp2p/go-libp2p-core/peer"
"sync"
"time"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/specs-actors/actors/abi"
miner_spec "github.com/filecoin-project/specs-actors/actors/builtin/miner"
"github.com/ipfs/go-cid"
_ "github.com/lib/pq"
"golang.org/x/xerrors"
@ -18,6 +24,9 @@ type storage struct {
db *sql.DB
headerLk sync.Mutex
// stateful miner data
minerSectors map[cid.Cid]struct{}
}
func openStorage(dbSource string) (*storage, error) {
@ -28,7 +37,10 @@ func openStorage(dbSource string) (*storage, error) {
db.SetMaxOpenConns(1350)
st := &storage{db: db}
ms := make(map[cid.Cid]struct{})
ms[cid.Undef] = struct{}{}
st := &storage{db: db, minerSectors: ms}
return st, st.setup()
}
@ -118,7 +130,7 @@ create unique index if not exists block_cid_uindex
create materialized view if not exists state_heights
as select distinct height, parentstateroot from blocks;
create unique index if not exists state_heights_uindex
create index if not exists state_heights_index
on state_heights (height);
create index if not exists state_heights_height_index
@ -252,31 +264,56 @@ create table if not exists receipts
create index if not exists receipts_msg_state_index
on receipts (msg, state);
/*
create table if not exists miner_heads
create table if not exists miner_sectors
(
head text not null,
addr text not null,
stateroot text not null,
sectorset text not null,
setsize decimal not null,
provingset text not null,
provingsize decimal not null,
owner text not null,
worker text not null,
peerid text not null,
sectorsize bigint not null,
power decimal not null,
active bool,
ppe bigint not null,
slashed_at bigint not null,
constraint miner_heads_pk
primary key (head, addr)
miner_id text not null,
sector_id bigint not null,
activation_epoch bigint not null,
expiration_epoch bigint not null,
termination_epoch bigint,
deal_weight text not null,
verified_deal_weight text not null,
seal_cid text not null,
seal_rand_epoch bigint not null,
constraint miner_sectors_pk
primary key (miner_id, sector_id)
);
create index if not exists miner_heads_stateroot_index
on miner_heads (stateroot);
create index if not exists miner_sectors_miner_sectorid_index
on miner_sectors (miner_id, sector_id);
create table if not exists miner_info
(
miner_id text not null,
owner_addr text not null,
worker_addr text not null,
peer_id text,
sector_size text not null,
precommit_deposits text not null,
locked_funds text not null,
next_deadline_process_faults bigint not null,
constraint miner_info_pk
primary key (miner_id)
);
/* used to tell when a miners sectors (proven-not-yet-expired) changed if the miner_sectors_cid's are different a new sector was added or removed (terminated/expired) */
create table if not exists miner_sectors_heads
(
miner_id text not null,
miner_sectors_cid text not null,
state_root text not null,
constraint miner_sectors_heads_pk
primary key (miner_id,miner_sectors_cid)
);
/*
create or replace function miner_tips(epoch bigint)
returns table (head text,
addr text,
@ -456,54 +493,261 @@ func (st *storage) storeActors(actors map[address.Address]map[types.Actor]actorI
return nil
}
func (st *storage) storeMiners(miners map[minerKey]*minerInfo) error {
/*tx, err := st.db.Begin()
if err != nil {
return err
}
type storeSectorsAPI interface {
StateMinerSectors(context.Context, address.Address, *abi.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error)
}
if _, err := tx.Exec(`
func (st *storage) storeSectors(minerTips map[types.TipSetKey][]*minerStateInfo, sectorApi storeSectorsAPI) error {
tx, err := st.db.Begin()
if err != nil {
return err
}
create temp table mh (like miner_heads excluding constraints) on commit drop;
if _, err := tx.Exec(`create temp table ms (like miner_sectors excluding constraints) on commit drop;`); err != nil {
return xerrors.Errorf("prep temp: %w", err)
}
stmt, err := tx.Prepare(`copy ms (miner_id, sector_id, activation_epoch, expiration_epoch, deal_weight, verified_deal_weight, seal_cid, seal_rand_epoch) from STDIN`)
if err != nil {
return err
}
`); err != nil {
return xerrors.Errorf("prep temp: %w", err)
}
for tipset, miners := range minerTips {
for _, miner := range miners {
sectors, err := sectorApi.StateMinerSectors(context.TODO(), miner.addr, nil, true, tipset)
if err != nil {
log.Debugw("Failed to load sectors", "tipset", tipset.String(), "miner", miner.addr.String(), "error", err)
}
stmt, err := tx.Prepare(`copy mh (head, addr, stateroot, sectorset, setsize, provingset, provingsize, owner, worker, peerid, sectorsize, power, ppe) from STDIN`)
if err != nil {
return err
}
for k, i := range miners {
if _, err := stmt.Exec(
k.act.Head.String(),
k.addr.String(),
k.stateroot.String(),
i.state.Sectors.String(),
fmt.Sprint(i.ssize),
i.state.ProvingSet.String(),
fmt.Sprint(i.psize),
i.info.Owner.String(),
i.info.Worker.String(),
i.info.PeerId.String(),
i.info.SectorSize,
i.power.String(), // TODO: SPA
i.state.PoStState.ProvingPeriodStart,
); err != nil {
return err
for _, sector := range sectors {
if _, err := stmt.Exec(
miner.addr.String(),
uint64(sector.ID),
int64(sector.Info.ActivationEpoch),
int64(sector.Info.Info.Expiration),
sector.Info.DealWeight.String(),
sector.Info.VerifiedDealWeight.String(),
sector.Info.Info.SealedCID.String(),
int64(sector.Info.Info.SealRandEpoch),
); err != nil {
return err
}
}
}
if err := stmt.Close(); err != nil {
}
if err := stmt.Close(); err != nil {
return err
}
if _, err := tx.Exec(`insert into miner_sectors select * from ms on conflict do nothing `); err != nil {
return xerrors.Errorf("actor put: %w", err)
}
return tx.Commit()
}
func (st *storage) storeMiners(minerTips map[types.TipSetKey][]*minerStateInfo) error {
tx, err := st.db.Begin()
if err != nil {
return err
}
if _, err := tx.Exec(`create temp table mi (like miner_info excluding constraints) on commit drop;`); err != nil {
return xerrors.Errorf("prep temp: %w", err)
}
stmt, err := tx.Prepare(`copy mi (miner_id, owner_addr, worker_addr, peer_id, sector_size, precommit_deposits, locked_funds, next_deadline_process_faults) from STDIN`)
if err != nil {
return err
}
for ts, miners := range minerTips {
for _, miner := range miners {
var pid string
if len(miner.info.PeerId) != 0 {
peerid, err := peer.IDFromBytes(miner.info.PeerId)
if err != nil {
// this should "never happen", but if it does we should still store info about the miner.
log.Warnw("failed to decode peerID", "peerID (bytes)", miner.info.PeerId, "miner", miner.addr, "tipset", ts.String())
} else {
pid = peerid.String()
}
}
if _, err := stmt.Exec(
miner.addr.String(),
miner.info.Owner.String(),
miner.info.Worker.String(),
pid,
miner.info.SectorSize.ShortString(),
miner.state.PreCommitDeposits.String(),
miner.state.LockedFunds.String(),
miner.state.NextDeadlineToProcessFaults,
); err != nil {
log.Errorw("failed to store miner state", "state", miner.state, "info", miner.info, "error", err)
return err
}
}
}
if err := stmt.Close(); err != nil {
return err
}
if _, err := tx.Exec(`insert into miner_info select * from mi on conflict do nothing `); err != nil {
return xerrors.Errorf("actor put: %w", err)
}
return tx.Commit()
}
type minerSectorUpdate struct {
minerState *minerStateInfo
tskey types.TipSetKey
oldSector cid.Cid
}
func (st *storage) storeMinerSectorsHeads(minerTips map[types.TipSetKey][]*minerStateInfo, api api.FullNode) error {
tx, err := st.db.Begin()
if err != nil {
return err
}
if _, err := tx.Exec(`create temp table msh (like miner_sectors_heads excluding constraints) on commit drop;`); err != nil {
return xerrors.Errorf("prep temp: %w", err)
}
stmt, err := tx.Prepare(`copy msh (miner_id, miner_sectors_cid, state_root) from STDIN`)
if err != nil {
return err
}
var updateMiners []*minerSectorUpdate
for tsk, miners := range minerTips {
for _, miner := range miners {
sectorCID, err := st.getLatestMinerSectorCID(context.TODO(), miner.addr)
if err != nil {
panic(err)
}
if sectorCID == cid.Undef {
continue
}
if _, found := st.minerSectors[sectorCID]; !found {
// schedule miner table update
updateMiners = append(updateMiners, &minerSectorUpdate{
minerState: miner,
tskey: tsk,
oldSector: sectorCID,
})
}
st.minerSectors[sectorCID] = struct{}{}
log.Debugw("got sector CID", "miner", miner.addr, "cid", sectorCID.String())
if _, err := stmt.Exec(
miner.addr.String(),
miner.state.Sectors.String(),
miner.stateroot.String(),
); err != nil {
log.Errorw("failed to store miners sectors head", "state", miner.state, "info", miner.info, "error", err)
return err
}
}
}
if err := stmt.Close(); err != nil {
return err
}
if _, err := tx.Exec(`insert into miner_sectors_heads select * from msh on conflict do nothing `); err != nil {
return xerrors.Errorf("actor put: %w", err)
}
if err := tx.Commit(); err != nil {
return err
}
return st.updateMinerSectors(updateMiners, api)
}
type deletedSector struct {
deletedSector miner_spec.SectorOnChainInfo
miner address.Address
tskey types.TipSetKey
}
func (st *storage) updateMinerSectors(miners []*minerSectorUpdate, api api.FullNode) error {
log.Info("updating miners constant sector table")
var deletedSectors []*deletedSector
for _, miner := range miners {
s := &apiIpldStore{context.TODO(), api}
newSectors, err := adt.AsArray(s, miner.minerState.state.Sectors)
if err != nil {
log.Warnw("new sectors as array", "error", err, "cid", miner.minerState.state.Sectors)
return err
}
if _, err := tx.Exec(`insert into miner_heads select * from mh on conflict do nothing `); err != nil {
return xerrors.Errorf("actor put: %w", err)
oldSectors, err := adt.AsArray(s, miner.oldSector)
if err != nil {
log.Warnw("old sectors as array", "error", err, "cid", miner.oldSector.String())
return err
}
return tx.Commit()*/
return nil
var oldSecInfo miner_spec.SectorOnChainInfo
var newSecInfo miner_spec.SectorOnChainInfo
// if we cannot find an old sector in the new list then it was removed.
if err := oldSectors.ForEach(&oldSecInfo, func(i int64) error {
found, err := newSectors.Get(uint64(oldSecInfo.Info.SectorNumber), &newSecInfo)
if err != nil {
log.Warnw("new sectors get", "error", err)
return err
}
if !found {
log.Infow("MINER DELETED SECTOR", "miner", miner.minerState.addr.String(), "sector", oldSecInfo.Info.SectorNumber, "tipset", miner.tskey.String())
deletedSectors = append(deletedSectors, &deletedSector{
deletedSector: oldSecInfo,
miner: miner.minerState.addr,
tskey: miner.tskey,
})
}
return nil
}); err != nil {
log.Warnw("old sectors foreach", "error", err)
return err
}
if len(deletedSectors) > 0 {
log.Infow("Calculated updates", "miner", miner.minerState.addr, "deleted sectors", len(deletedSectors))
}
}
// now we have all the sectors that were removed, update the database
tx, err := st.db.Begin()
if err != nil {
return err
}
stmt, err := tx.Prepare(`UPDATE miner_sectors SET termination_epoch=$1 WHERE miner_id=$2 AND sector_id=$3`)
if err != nil {
return err
}
for _, ds := range deletedSectors {
ts, err := api.ChainGetTipSet(context.TODO(), ds.tskey)
if err != nil {
log.Warnw("get tipset", "error", err)
return err
}
// TODO validate this shits right
if ts.Height() >= ds.deletedSector.Info.Expiration {
// means it expired, do nothing
log.Infow("expired sector", "miner", ds.miner.String(), "sector", ds.deletedSector.Info.SectorNumber)
continue
}
log.Infow("terminated sector", "miner", ds.miner.String(), "sector", ds.deletedSector.Info.SectorNumber)
// means it was terminated.
if _, err := stmt.Exec(int64(ts.Height()), ds.miner.String(), int64(ds.deletedSector.Info.SectorNumber)); err != nil {
return err
}
}
if err := stmt.Close(); err != nil {
return err
}
defer log.Info("update miner sectors complete")
return tx.Commit()
}
func (st *storage) storeHeaders(bhs map[cid.Cid]*types.BlockHeader, sync bool) error {
@ -1008,3 +1252,27 @@ func (st *storage) refreshViews() error {
func (st *storage) close() error {
return st.db.Close()
}
func (st *storage) getLatestMinerSectorCID(ctx context.Context, miner address.Address) (cid.Cid, error) {
queryStr := fmt.Sprintf(`
SELECT miner_sectors_cid
FROM miner_sectors_heads
LEFT JOIN blocks ON miner_sectors_heads.state_root = blocks.parentstateroot
WHERE miner_id = '%s'
ORDER BY blocks.height DESC
LIMIT 1;
`,
miner.String())
var cidstr string
err := st.db.QueryRowContext(ctx, queryStr).Scan(&cidstr)
switch {
case err == sql.ErrNoRows:
log.Warnf("no miner with miner_id: %s in table", miner)
return cid.Undef, nil
case err != nil:
return cid.Undef, err
default:
return cid.Decode(cidstr)
}
}

263
cmd/lotus-chainwatch/sync.go
View File

@ -5,17 +5,21 @@ import (
"container/list"
"context"
"encoding/json"
"fmt"
"math"
"sync"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
"time"
"github.com/filecoin-project/go-address"
"github.com/ipfs/go-cid"
cbg "github.com/whyrusleeping/cbor-gen"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
"github.com/filecoin-project/specs-actors/actors/builtin/power"
"github.com/filecoin-project/specs-actors/actors/util/adt"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/store"
@ -49,27 +53,30 @@ func runSyncer(ctx context.Context, api api.FullNode, st *storage, maxBatch int)
}()
}
type minerKey struct {
type minerStateInfo struct {
// common
addr address.Address
act types.Actor
stateroot cid.Cid
}
type minerInfo struct {
// miner specific
state miner.State
info miner.MinerInfo
power big.Int
ssize uint64
psize uint64
// tracked by power actor
rawPower big.Int
qalPower big.Int
ssize uint64
psize uint64
}
type actorInfo struct {
stateroot cid.Cid
tsKey types.TipSetKey
state string
}
func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipSet, maxBatch int) {
func syncHead(ctx context.Context, api api.FullNode, st *storage, headTs *types.TipSet, maxBatch int) {
var alk sync.Mutex
log.Infof("Getting synced block list")
@ -78,25 +85,28 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
log.Infof("Getting headers / actors")
// global list of all blocks that need to be synced
allToSync := map[cid.Cid]*types.BlockHeader{}
// a stack
toVisit := list.New()
for _, header := range ts.Blocks() {
for _, header := range headTs.Blocks() {
toVisit.PushBack(header)
}
// TODO consider making a db query to check where syncing left off at in the case of a restart and avoid reprocessing
// those entries, or write value to file on shutdown
// walk the entire chain starting from headTS
for toVisit.Len() > 0 {
bh := toVisit.Remove(toVisit.Back()).(*types.BlockHeader)
_, has := hazlist[bh.Cid()]
if _, seen := allToSync[bh.Cid()]; seen || has {
continue
}
allToSync[bh.Cid()] = bh
if len(allToSync)%500 == 10 {
log.Infof("todo: (%d) %s @%d", len(allToSync), bh.Cid(), bh.Height)
log.Debugf("to visit: (%d) %s @%d", len(allToSync), bh.Cid(), bh.Height)
}
if len(bh.Parents) == 0 {
@ -114,30 +124,42 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
}
}
// Main worker loop, this loop runs until all tipse from headTS to genesis have been processed.
for len(allToSync) > 0 {
// first map is addresses -> common actors states (head, code, balance, nonce)
// second map common actor states -> chain state (tipset, stateroot) & unique actor state (deserialization of their head CID) represented as json.
actors := map[address.Address]map[types.Actor]actorInfo{}
addresses := map[address.Address]address.Address{}
// map of actor public key address to ID address
addressToID := map[address.Address]address.Address{}
minH := abi.ChainEpoch(math.MaxInt64)
// find the blockheader with the lowest height
for _, header := range allToSync {
if header.Height < minH {
minH = header.Height
}
}
// toSync maps block cids to their headers and contains all block headers that will be synced in this batch
// `maxBatch` is a tunable parameter to control how many blocks we sync per iteration.
toSync := map[cid.Cid]*types.BlockHeader{}
for c, header := range allToSync {
if header.Height < minH+abi.ChainEpoch(maxBatch) {
toSync[c] = header
addresses[header.Miner] = address.Undef
addressToID[header.Miner] = address.Undef
}
}
// remove everything we are syncing this round from the global list of blocks to sync
for c := range toSync {
delete(allToSync, c)
}
log.Infof("Syncing %d blocks", len(toSync))
log.Infow("Starting Sync", "height", minH, "numBlocks", len(toSync), "maxBatch", maxBatch)
// map of addresses to changed actors
var changes map[string]types.Actor
// collect all actor state that has changes between block headers
paDone := 0
parmap.Par(50, parmap.MapArr(toSync), func(bh *types.BlockHeader) {
paDone++
@ -146,20 +168,23 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
}
if len(bh.Parents) == 0 { // genesis case
ts, _ := types.NewTipSet([]*types.BlockHeader{bh})
aadrs, err := api.StateListActors(ctx, ts.Key())
genesisTs, _ := types.NewTipSet([]*types.BlockHeader{bh})
aadrs, err := api.StateListActors(ctx, genesisTs.Key())
if err != nil {
log.Error(err)
return
}
// TODO suspicious there is not a lot to be gained by doing this in parallel since the genesis state
// is unlikely to contain a lot of actors, why not for loop here?
parmap.Par(50, aadrs, func(addr address.Address) {
act, err := api.StateGetActor(ctx, addr, ts.Key())
act, err := api.StateGetActor(ctx, addr, genesisTs.Key())
if err != nil {
log.Error(err)
return
}
ast, err := api.StateReadState(ctx, act, ts.Key())
ast, err := api.StateReadState(ctx, addr, genesisTs.Key())
if err != nil {
log.Error(err)
return
@ -177,9 +202,10 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
}
actors[addr][*act] = actorInfo{
stateroot: bh.ParentStateRoot,
tsKey: genesisTs.Key(),
state: string(state),
}
addresses[addr] = address.Undef
addressToID[addr] = address.Undef
alk.Unlock()
})
@ -192,12 +218,15 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
return
}
changes, err := api.StateChangedActors(ctx, pts.ParentState(), bh.ParentStateRoot)
// TODO Does this return actors that have been deleted between states?
// collect all actors that had state changes between the blockheader parent-state and its grandparent-state.
changes, err = api.StateChangedActors(ctx, pts.ParentState(), bh.ParentStateRoot)
if err != nil {
log.Error(err)
return
}
// record the state of all actors that have changed
for a, act := range changes {
act := act
@ -206,11 +235,14 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
log.Error(err)
return
}
ast, err := api.StateReadState(ctx, &act, pts.Key())
ast, err := api.StateReadState(ctx, addr, pts.Key())
if err != nil {
log.Error(err)
return
}
state, err := json.Marshal(ast.State)
if err != nil {
log.Error(err)
@ -222,15 +254,22 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
if !ok {
actors[addr] = map[types.Actor]actorInfo{}
}
// a change occurred for the actor with address `addr` and state `act` at tipset `pts`.
actors[addr][act] = actorInfo{
stateroot: bh.ParentStateRoot,
state: string(state),
tsKey: pts.Key(),
}
addresses[addr] = address.Undef
addressToID[addr] = address.Undef
alk.Unlock()
}
})
// map of tipset to all miners that had a head-change at that tipset.
minerTips := make(map[types.TipSetKey][]*minerStateInfo, len(changes))
// heads we've seen, im being paranoid
headsSeen := make(map[cid.Cid]struct{}, len(actors))
log.Infof("Getting messages")
msgs, incls := fetchMessages(ctx, api, toSync)
@ -238,70 +277,109 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
log.Infof("Resolving addresses")
for _, message := range msgs {
addresses[message.To] = address.Undef
addresses[message.From] = address.Undef
addressToID[message.To] = address.Undef
addressToID[message.From] = address.Undef
}
parmap.Par(50, parmap.KMapArr(addresses), func(addr address.Address) {
parmap.Par(50, parmap.KMapArr(addressToID), func(addr address.Address) {
// FIXME: cannot use EmptyTSK here since actorID's can change during reorgs, need to use the corresponding tipset.
// TODO: figure out a way to get the corresponding tipset...
raddr, err := api.StateLookupID(ctx, addr, types.EmptyTSK)
if err != nil {
log.Warn(err)
return
}
alk.Lock()
addresses[addr] = raddr
addressToID[addr] = raddr
alk.Unlock()
})
log.Infof("Getting miner info")
miners := map[minerKey]*minerInfo{}
minerChanges := 0
for addr, m := range actors {
for actor, c := range m {
if actor.Code != builtin.StorageMinerActorCodeID {
continue
}
miners[minerKey{
// only want miner actors with head change events
if _, found := headsSeen[actor.Head]; found {
continue
}
minerChanges++
minerTips[c.tsKey] = append(minerTips[c.tsKey], &minerStateInfo{
addr: addr,
act: actor,
stateroot: c.stateroot,
}] = &minerInfo{}
state: miner.State{},
info: miner.MinerInfo{},
rawPower: big.Zero(),
qalPower: big.Zero(),
})
headsSeen[actor.Head] = struct{}{}
}
}
parmap.Par(50, parmap.KVMapArr(miners), func(it func() (minerKey, *minerInfo)) {
k, info := it()
minerProcessingStartedAt := time.Now()
log.Infow("Processing miners", "numTips", len(minerTips), "numMinerChanges", minerChanges)
// extract the power actor state at each tipset, loop over all miners that changed at said tipset and extract their
// claims from the power actor state. This ensures we only fetch the power actors state once for each tipset.
parmap.Par(50, parmap.KVMapArr(minerTips), func(it func() (types.TipSetKey, []*minerStateInfo)) {
tsKey, minerInfo := it()
pow, err := api.StateMinerPower(ctx, k.addr, types.EmptyTSK)
if err != nil {
log.Error(err)
// Not sure why this would fail, but its probably worth continuing
}
info.power = pow.MinerPower.QualityAdjPower
sszs, err := api.StateMinerSectorCount(ctx, k.addr, types.EmptyTSK)
// get the power actors claims map
mp, err := getPowerActorClaimsMap(ctx, api, tsKey)
if err != nil {
log.Error(err)
return
}
info.psize = sszs.Pset
info.ssize = sszs.Sset
// Get miner raw and quality power
for _, mi := range minerInfo {
var claim power.Claim
// get miner claim from power actors claim map and store if found, else the miner had no claim at
// this tipset
found, err := mp.Get(adt.AddrKey(mi.addr), &claim)
if err != nil {
log.Error(err)
}
if found {
mi.qalPower = claim.QualityAdjPower
mi.rawPower = claim.RawBytePower
}
astb, err := api.ChainReadObj(ctx, k.act.Head)
if err != nil {
log.Error(err)
return
// Get the miner state info
astb, err := api.ChainReadObj(ctx, mi.act.Head)
if err != nil {
log.Error(err)
return
}
if err := mi.state.UnmarshalCBOR(bytes.NewReader(astb)); err != nil {
log.Error(err)
return
}
mi.info = mi.state.Info
}
if err := info.state.UnmarshalCBOR(bytes.NewReader(astb)); err != nil {
log.Error(err)
return
}
info.info = info.state.Info
// TODO Get the Sector Count
// FIXME this is returning a lot of "address not found" errors, which is strange given that StateChangedActors
// retruns all actors that had a state change at tipset `k.tsKey`, maybe its returning deleted miners too??
/*
sszs, err := api.StateMinerSectorCount(ctx, k.addr, k.tsKey)
if err != nil {
info.psize = 0
info.ssize = 0
} else {
info.psize = sszs.Pset
info.ssize = sszs.Sset
}
*/
})
log.Infow("Completed Miner Processing", "duration", time.Since(minerProcessingStartedAt).String(), "processed", minerChanges)
log.Info("Getting receipts")
@ -316,7 +394,7 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
log.Info("Storing address mapping")
if err := st.storeAddressMap(addresses); err != nil {
if err := st.storeAddressMap(addressToID); err != nil {
log.Error(err)
return
}
@ -329,8 +407,21 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
}
log.Info("Storing miners")
if err := st.storeMiners(minerTips); err != nil {
log.Error(err)
return
}
if err := st.storeMiners(miners); err != nil {
log.Info("Storing miner sectors")
sectorStart := time.Now()
if err := st.storeSectors(minerTips, api); err != nil {
log.Error(err)
return
}
log.Infow("Finished storing miner sectors", "duration", time.Since(sectorStart).String())
log.Info("Storing miner sectors heads")
if err := st.storeMinerSectorsHeads(minerTips, api); err != nil {
log.Error(err)
return
}
@ -361,7 +452,7 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
log.Infof("Get deals")
// TODO: incremental, gather expired
deals, err := api.StateMarketDeals(ctx, ts.Key())
deals, err := api.StateMarketDeals(ctx, headTs.Key())
if err != nil {
log.Error(err)
return
@ -451,3 +542,55 @@ func fetchParentReceipts(ctx context.Context, api api.FullNode, toSync map[cid.C
return out
}
// load the power actor state clam as an adt.Map at the tipset `ts`.
func getPowerActorClaimsMap(ctx context.Context, api api.FullNode, ts types.TipSetKey) (*adt.Map, error) {
powerActor, err := api.StateGetActor(ctx, builtin.StoragePowerActorAddr, ts)
if err != nil {
return nil, err
}
powerRaw, err := api.ChainReadObj(ctx, powerActor.Head)
if err != nil {
return nil, err
}
var powerActorState power.State
if err := powerActorState.UnmarshalCBOR(bytes.NewReader(powerRaw)); err != nil {
return nil, fmt.Errorf("failed to unmarshal power actor state: %w", err)
}
s := &apiIpldStore{ctx, api}
return adt.AsMap(s, powerActorState.Claims)
}
// require for AMT and HAMT access
// TODO extract this to a common location in lotus and reuse the code
type apiIpldStore struct {
ctx context.Context
api api.FullNode
}
func (ht *apiIpldStore) Context() context.Context {
return ht.ctx
}
func (ht *apiIpldStore) Get(ctx context.Context, c cid.Cid, out interface{}) error {
raw, err := ht.api.ChainReadObj(ctx, c)
if err != nil {
return err
}
cu, ok := out.(cbg.CBORUnmarshaler)
if ok {
if err := cu.UnmarshalCBOR(bytes.NewReader(raw)); err != nil {
return err
}
return nil
}
return fmt.Errorf("Object does not implement CBORUnmarshaler: %T", out)
}
func (ht *apiIpldStore) Put(ctx context.Context, v interface{}) (cid.Cid, error) {
return cid.Undef, fmt.Errorf("Put is not implemented on apiIpldStore")
}

9
cmd/lotus-health/main.go
View File

@ -63,7 +63,7 @@ var watchHeadCmd = &cli.Command{
},
&cli.IntFlag{
Name: "interval",
Value: build.BlockDelay,
Value: int(build.BlockDelaySecs),
Usage: "interval in seconds between chain head checks",
},
&cli.StringFlag{
@ -72,8 +72,9 @@ var watchHeadCmd = &cli.Command{
Usage: "systemd unit name to restart on health check failure",
},
&cli.IntFlag{
Name: "api-timeout",
Value: build.BlockDelay,
Name: "api-timeout",
// TODO: this default value seems spurious.
Value: int(build.BlockDelaySecs),
Usage: "timeout between API retries",
},
&cli.IntFlag{
@ -236,7 +237,7 @@ func waitForSyncComplete(ctx context.Context, a api.FullNode, r int, t time.Dura
return err
}
if time.Now().Unix()-int64(head.MinTimestamp()) < build.BlockDelay {
if time.Now().Unix()-int64(head.MinTimestamp()) < int64(build.BlockDelaySecs) {
return nil
}
}

2
cmd/lotus-seed/genesis.go
View File

@ -124,7 +124,7 @@ var genesisAddMinerCmd = &cli.Command{
log.Infof("Giving %s some initial balance", miner.Owner)
template.Accounts = append(template.Accounts, genesis.Actor{
Type: genesis.TAccount,
Balance: big.Mul(big.NewInt(50_000_000), big.NewInt(build.FilecoinPrecision)),
Balance: big.Mul(big.NewInt(50_000_000), big.NewInt(int64(build.FilecoinPrecision))),
Meta: (&genesis.AccountMeta{Owner: miner.Owner}).ActorMeta(),
})
}

349
cmd/lotus-shed/keyinfo.go
View File

@ -1,8 +1,10 @@
package main
import (
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
@ -11,77 +13,356 @@ import (
"github.com/urfave/cli/v2"
_ "github.com/filecoin-project/lotus/lib/sigs/bls"
_ "github.com/filecoin-project/lotus/lib/sigs/secp"
"github.com/libp2p/go-libp2p-core/crypto"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/node/modules/lp2p"
"github.com/filecoin-project/lotus/node/repo"
_ "github.com/filecoin-project/lotus/lib/sigs/bls"
_ "github.com/filecoin-project/lotus/lib/sigs/secp"
)
type walletInfo struct {
var validTypes = []string{wallet.KTBLS, wallet.KTSecp256k1, lp2p.KTLibp2pHost}
type keyInfoOutput struct {
Type string
Address string
PublicKey string
}
func (wi walletInfo) String() string {
bs, _ := json.Marshal(wi)
return string(bs)
var keyinfoCmd = &cli.Command{
Name: "keyinfo",
Usage: "work with lotus keyinfo files (wallets and libp2p host keys)",
Description: `The subcommands of keyinfo provide helpful tools for working with keyinfo files without
having to run the lotus daemon.`,
Subcommands: []*cli.Command{
keyinfoNewCmd,
keyinfoInfoCmd,
keyinfoImportCmd,
},
}
var keyinfoCmd = &cli.Command{
Name: "keyinfo",
Description: "decode a keyinfo",
var keyinfoImportCmd = &cli.Command{
Name: "import",
Usage: "import a keyinfo file into a lotus repository",
Description: `The import command provides a way to import keyfiles into a lotus repository
without running the daemon.
Note: The LOTUS_PATH directory must be created. This command will not create this directory for you.
Examples
env LOTUS_PATH=/var/lib/lotus lotus-shed keyinfo import libp2p-host.keyinfo`,
Action: func(cctx *cli.Context) error {
flagRepo := cctx.String("repo")
var input io.Reader
if cctx.Args().Len() == 0 {
input = os.Stdin
} else {
var err error
input, err = os.Open(cctx.Args().First())
if err != nil {
return err
}
}
encoded, err := ioutil.ReadAll(input)
if err != nil {
return err
}
decoded, err := hex.DecodeString(strings.TrimSpace(string(encoded)))
if err != nil {
return err
}
var keyInfo types.KeyInfo
if err := json.Unmarshal(decoded, &keyInfo); err != nil {
return err
}
fsrepo, err := repo.NewFS(flagRepo)
if err != nil {
return err
}
lkrepo, err := fsrepo.Lock(repo.FullNode)
if err != nil {
return err
}
defer lkrepo.Close()
keystore, err := lkrepo.KeyStore()
if err != nil {
return err
}
switch keyInfo.Type {
case lp2p.KTLibp2pHost:
if err := keystore.Put(lp2p.KLibp2pHost, keyInfo); err != nil {
return err
}
sk, err := crypto.UnmarshalPrivateKey(keyInfo.PrivateKey)
if err != nil {
return err
}
peerid, err := peer.IDFromPrivateKey(sk)
if err != nil {
return err
}
fmt.Printf("%s\n", peerid.String())
break
case wallet.KTSecp256k1:
case wallet.KTBLS:
w, err := wallet.NewWallet(keystore)
if err != nil {
return err
}
addr, err := w.Import(&keyInfo)
if err != nil {
return err
}
fmt.Printf("%s\n", addr.String())
}
return nil
},
}
var keyinfoInfoCmd = &cli.Command{
Name: "info",
Usage: "print information about a keyinfo file",
Description: `The info command prints additional information about a key which can't easily
be retrieved by inspecting the file itself.
The 'format' flag takes a golang text/template template as its value.
The following fields can be retrived through this command
Type
Address
PublicKey
The PublicKey value will be printed base64 encoded using golangs StdEncoding
Examples
Retreive the address of a lotus wallet
lotus-shed keyinfo info --format '{{ .Address }}' wallet.keyinfo
`,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "format",
Value: "{{.Address}}",
Usage: "Format to output",
Value: "{{ .Type }} {{ .Address }}",
Usage: "specify which output columns to print",
},
},
Action: func(cctx *cli.Context) error {
format := cctx.String("format")
var input io.Reader
if cctx.Args().Len() == 0 {
input = os.Stdin
} else {
input = strings.NewReader(cctx.Args().First())
var err error
input, err = os.Open(cctx.Args().First())
if err != nil {
return err
}
}
bytes, err := ioutil.ReadAll(input)
data, err := hex.DecodeString(strings.TrimSpace(string(bytes)))
encoded, err := ioutil.ReadAll(input)
if err != nil {
return err
}
var ki types.KeyInfo
if err := json.Unmarshal(data, &ki); err != nil {
return err
}
key, err := wallet.NewKey(ki)
decoded, err := hex.DecodeString(strings.TrimSpace(string(encoded)))
if err != nil {
return err
}
bs, err := json.Marshal(key)
var keyInfo types.KeyInfo
if err := json.Unmarshal(decoded, &keyInfo); err != nil {
return err
}
var kio keyInfoOutput
switch keyInfo.Type {
case lp2p.KTLibp2pHost:
kio.Type = keyInfo.Type
sk, err := crypto.UnmarshalPrivateKey(keyInfo.PrivateKey)
if err != nil {
return err
}
pk := sk.GetPublic()
peerid, err := peer.IDFromPrivateKey(sk)
if err != nil {
return err
}
pkBytes, err := pk.Raw()
if err != nil {
return err
}
kio.Address = peerid.String()
kio.PublicKey = base64.StdEncoding.EncodeToString(pkBytes)
break
case wallet.KTSecp256k1:
case wallet.KTBLS:
kio.Type = keyInfo.Type
key, err := wallet.NewKey(keyInfo)
if err != nil {
return err
}
kio.Address = key.Address.String()
kio.PublicKey = base64.StdEncoding.EncodeToString(key.PublicKey)
}
tmpl, err := template.New("output").Parse(format)
if err != nil {
return err
}
var wi walletInfo
if err := json.Unmarshal(bs, &wi); err != nil {
return err
}
tmpl, err := template.New("").Parse(format)
if err != nil {
return err
}
return tmpl.Execute(os.Stdout, wi)
return tmpl.Execute(os.Stdout, kio)
},
}
var keyinfoNewCmd = &cli.Command{
Name: "new",
Usage: "create a new keyinfo file of the provided type",
ArgsUsage: "[bls|secp256k1|libp2p-host]",
Description: `Keyinfo files are base16 encoded json structures containing a type
string value, and a base64 encoded private key.
Both the bls and secp256k1 keyfiles can be imported into a running lotus daemon using
the 'lotus wallet import' command. Or imported to a non-running / unitialized repo using
the 'lotus-shed keyinfo import' command. Libp2p host keys can only be imported using lotus-shed
as lotus itself does not provide this functionality at the moment.`,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "output",
Value: "<type>-<addr>.keyinfo",
Usage: "output file formt",
},
&cli.BoolFlag{
Name: "silent",
Value: false,
Usage: "do not print the address to stdout",
},
},
Action: func(cctx *cli.Context) error {
if !cctx.Args().Present() {
return fmt.Errorf("please specify a type to generate")
}
keyType := cctx.Args().First()
flagOutput := cctx.String("output")
if i := SliceIndex(len(validTypes), func(i int) bool {
if keyType == validTypes[i] {
return true
}
return false
}); i == -1 {
return fmt.Errorf("invalid key type argument provided '%s'", keyType)
}
keystore := wallet.NewMemKeyStore()
var keyAddr string
var keyInfo types.KeyInfo
switch keyType {
case lp2p.KTLibp2pHost:
sk, err := lp2p.PrivKey(keystore)
if err != nil {
return err
}
ki, err := keystore.Get(lp2p.KLibp2pHost)
if err != nil {
return err
}
peerid, err := peer.IDFromPrivateKey(sk)
if err != nil {
return err
}
keyAddr = peerid.String()
keyInfo = ki
break
case wallet.KTSecp256k1:
case wallet.KTBLS:
key, err := wallet.GenerateKey(wallet.ActSigType(keyType))
if err != nil {
return err
}
keyAddr = key.Address.String()
keyInfo = key.KeyInfo
break
}
filename := flagOutput
filename = strings.ReplaceAll(filename, "<addr>", keyAddr)
filename = strings.ReplaceAll(filename, "<type>", keyType)
file, err := os.Create(filename)
if err != nil {
return err
}
defer func() {
if err := file.Close(); err != nil {
log.Warnf("failed to close output file: %w", err)
}
}()
bytes, err := json.Marshal(keyInfo)
if err != nil {
return err
}
encoded := hex.EncodeToString(bytes)
if _, err := file.Write([]byte(encoded)); err != nil {
return err
}
if !cctx.Bool("silent") {
fmt.Println(keyAddr)
}
return nil
},
}
func SliceIndex(length int, fn func(i int) bool) int {
for i := 0; i < length; i++ {
if fn(i) {
return i
}
}
return -1
}

1
cmd/lotus-shed/main.go
View File

@ -19,7 +19,6 @@ func main() {
base16Cmd,
bitFieldCmd,
keyinfoCmd,
peerkeyCmd,
noncefix,
bigIntParseCmd,
staterootStatsCmd,

101
cmd/lotus-shed/peerkey.go
View File

@ -1,101 +0,0 @@
package main
import (
"encoding/json"
"fmt"
"os"
"strings"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/modules/lp2p"
"github.com/libp2p/go-libp2p-core/peer"
)
type keystore struct {
set bool
info types.KeyInfo
}
func (ks *keystore) Put(name string, info types.KeyInfo) error {
ks.info = info
ks.set = true
return nil
}
func (ks *keystore) Get(name string) (types.KeyInfo, error) {
if !ks.set {
return types.KeyInfo{}, types.ErrKeyInfoNotFound
}
return ks.info, nil
}
func (ks *keystore) Delete(name string) error {
panic("Implement me")
}
func (ks *keystore) List() ([]string, error) {
panic("Implement me")
}
var peerkeyCmd = &cli.Command{
Name: "peerkey",
Description: "create libp2p host key",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "output",
Value: "<peerid>.peerkey",
Usage: "Output file format",
},
&cli.BoolFlag{
Name: "silent",
Value: false,
Usage: "Do not print peerid at end",
},
},
Action: func(cctx *cli.Context) error {
output := cctx.String("output")
ks := keystore{}
sk, err := lp2p.PrivKey(&ks)
if err != nil {
return err
}
bs, err := json.Marshal(ks.info)
if err != nil {
return err
}
peerid, err := peer.IDFromPrivateKey(sk)
if err != nil {
return err
}
output = strings.ReplaceAll(output, "<peerid>", peerid.String())
f, err := os.Create(output)
if err != nil {
return err
}
defer func() {
if err := f.Close(); err != nil {
log.Warnf("failed to close output file: %w", err)
}
}()
if _, err := f.Write(bs); err != nil {
return err
}
if !cctx.Bool("silent") {
fmt.Println(peerid.String())
}
return nil
},
}

2
cmd/lotus-storage-miner/info.go
View File

@ -128,7 +128,7 @@ var infoCmd = &cli.Command{
if expWinChance > 1 {
expWinChance = 1
}
winRate := time.Duration(float64(time.Second*build.BlockDelay) / expWinChance)
winRate := time.Duration(float64(time.Second*time.Duration(build.BlockDelaySecs)) / expWinChance)
winPerDay := float64(time.Hour*24) / float64(winRate)
fmt.Print("Expected block win rate: ")

3
cmd/lotus-storage-miner/main.go
View File

@ -23,7 +23,8 @@ func main() {
local := []*cli.Command{
actorCmd,
dealsCmd,
storageDealsCmd,
retrievalDealsCmd,
infoCmd,
initCmd,
rewardsCmd,

116
cmd/lotus-storage-miner/market.go
View File

@ -50,33 +50,100 @@ func GetCidEncoder(cctx *cli.Context) (cidenc.Encoder, error) {
return e, nil
}
var enableCmd = &cli.Command{
Name: "enable",
Usage: "Configure the miner to consider storage deal proposals",
Flags: []cli.Flag{},
Action: func(cctx *cli.Context) error {
api, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
return api.DealsSetAcceptingStorageDeals(lcli.DaemonContext(cctx), true)
var storageDealSelectionCmd = &cli.Command{
Name: "selection",
Usage: "Configure acceptance criteria for storage deal proposals",
Subcommands: []*cli.Command{
storageDealSelectionShowCmd,
storageDealSelectionResetCmd,
storageDealSelectionRejectCmd,
},
}
var disableCmd = &cli.Command{
Name: "disable",
Usage: "Configure the miner to reject all storage deal proposals",
Flags: []cli.Flag{},
var storageDealSelectionShowCmd = &cli.Command{
Name: "list",
Usage: "List storage deal proposal selection criteria",
Action: func(cctx *cli.Context) error {
api, closer, err := lcli.GetStorageMinerAPI(cctx)
smapi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
return api.DealsSetAcceptingStorageDeals(lcli.DaemonContext(cctx), false)
onlineOk, err := smapi.DealsConsiderOnlineStorageDeals(lcli.DaemonContext(cctx))
if err != nil {
return err
}
offlineOk, err := smapi.DealsConsiderOfflineStorageDeals(lcli.DaemonContext(cctx))
if err != nil {
return err
}
fmt.Printf("considering online storage deals: %t\n", onlineOk)
fmt.Printf("considering offline storage deals: %t\n", offlineOk)
return nil
},
}
var storageDealSelectionResetCmd = &cli.Command{
Name: "reset",
Usage: "Reset storage deal proposal selection criteria to default values",
Action: func(cctx *cli.Context) error {
smapi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
err = smapi.DealsSetConsiderOnlineStorageDeals(lcli.DaemonContext(cctx), true)
if err != nil {
return err
}
err = smapi.DealsSetConsiderOfflineStorageDeals(lcli.DaemonContext(cctx), true)
if err != nil {
return err
}
return nil
},
}
var storageDealSelectionRejectCmd = &cli.Command{
Name: "reject",
Usage: "Configure criteria which necessitate automatic rejection",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "online",
},
&cli.BoolFlag{
Name: "offline",
},
},
Action: func(cctx *cli.Context) error {
smapi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
if cctx.Bool("online") {
err = smapi.DealsSetConsiderOnlineStorageDeals(lcli.DaemonContext(cctx), false)
if err != nil {
return err
}
}
if cctx.Bool("offline") {
err = smapi.DealsSetConsiderOfflineStorageDeals(lcli.DaemonContext(cctx), false)
if err != nil {
return err
}
}
return nil
},
}
@ -123,7 +190,7 @@ var setAskCmd = &cli.Command{
return xerrors.Errorf("cannot parse duration: %w", err)
}
qty := dur.Seconds() / build.BlockDelay
qty := dur.Seconds() / float64(build.BlockDelaySecs)
min, err := units.RAMInBytes(cctx.String("min-piece-size"))
if err != nil {
@ -208,7 +275,7 @@ var getAskCmd = &cli.Command{
dlt := ask.Expiry - head.Height()
rem := "<expired>"
if dlt > 0 {
rem = (time.Second * time.Duration(dlt*build.BlockDelay)).String()
rem = (time.Second * time.Duration(int64(dlt)*int64(build.BlockDelaySecs))).String()
}
fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%s\t%d\n", ask.Price, types.SizeStr(types.NewInt(uint64(ask.MinPieceSize))), types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize))), ask.Expiry, rem, ask.SeqNo)
@ -217,14 +284,13 @@ var getAskCmd = &cli.Command{
},
}
var dealsCmd = &cli.Command{
Name: "deals",
Usage: "interact with your deals",
var storageDealsCmd = &cli.Command{
Name: "storage-deals",
Usage: "Manage storage deals and related configuration",
Subcommands: []*cli.Command{
dealsImportDataCmd,
dealsListCmd,
enableCmd,
disableCmd,
storageDealSelectionCmd,
setAskCmd,
getAskCmd,
setBlocklistCmd,

73
cmd/lotus-storage-miner/proving.go
View File

@ -26,6 +26,75 @@ var provingCmd = &cli.Command{
Subcommands: []*cli.Command{
provingInfoCmd,
provingDeadlinesCmd,
provingFaultsCmd,
},
}
var provingFaultsCmd = &cli.Command{
Name: "faults",
Usage: "View the currently known proving faulty sectors information",
Action: func(cctx *cli.Context) error {
nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
api, acloser, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer acloser()
ctx := lcli.ReqContext(cctx)
maddr, err := nodeApi.ActorAddress(ctx)
if err != nil {
return xerrors.Errorf("getting actor address: %w", err)
}
var mas miner.State
{
mact, err := api.StateGetActor(ctx, maddr, types.EmptyTSK)
if err != nil {
return err
}
rmas, err := api.ChainReadObj(ctx, mact.Head)
if err != nil {
return err
}
if err := mas.UnmarshalCBOR(bytes.NewReader(rmas)); err != nil {
return err
}
}
faults, err := mas.Faults.All(100000000000)
if err != nil {
return err
}
if len(faults) == 0 {
fmt.Println("no faulty sectors")
}
head, err := api.ChainHead(ctx)
if err != nil {
return xerrors.Errorf("getting chain head: %w", err)
}
deadlines, err := api.StateMinerDeadlines(ctx, maddr, head.Key())
if err != nil {
return xerrors.Errorf("getting miner deadlines: %w", err)
}
tw := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0)
_, _ = fmt.Fprintln(tw, "deadline\tsectors")
for deadline, sectors := range deadlines.Due {
intersectSectors, _ := bitfield.IntersectBitField(sectors, mas.Faults)
if intersectSectors != nil {
allSectors, _ := intersectSectors.All(100000000000)
for _, num := range allSectors {
_, _ = fmt.Fprintf(tw, "%d\t%d\n", deadline, num)
}
}
}
return tw.Flush()
},
}
@ -142,11 +211,11 @@ var provingInfoCmd = &cli.Command{
func epochTime(curr, e abi.ChainEpoch) string {
switch {
case curr > e:
return fmt.Sprintf("%d (%s ago)", e, time.Second*time.Duration(build.BlockDelay*(curr-e)))
return fmt.Sprintf("%d (%s ago)", e, time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(curr-e)))
case curr == e:
return fmt.Sprintf("%d (now)", e)
case curr < e:
return fmt.Sprintf("%d (in %s)", e, time.Second*time.Duration(build.BlockDelay*(e-curr)))
return fmt.Sprintf("%d (in %s)", e, time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(e-curr)))
}
panic("math broke")

113
cmd/lotus-storage-miner/retrieval-deals.go Normal file
View File

@ -0,0 +1,113 @@
package main
import (
"fmt"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/urfave/cli/v2"
)
var retrievalDealsCmd = &cli.Command{
Name: "retrieval-deals",
Usage: "Manage retrieval deals and related configuration",
Subcommands: []*cli.Command{
retrievalDealSelectionCmd,
},
}
var retrievalDealSelectionCmd = &cli.Command{
Name: "selection",
Usage: "Configure acceptance criteria for retrieval deal proposals",
Subcommands: []*cli.Command{
retrievalDealSelectionShowCmd,
retrievalDealSelectionResetCmd,
retrievalDealSelectionRejectCmd,
},
}
var retrievalDealSelectionShowCmd = &cli.Command{
Name: "list",
Usage: "List retrieval deal proposal selection criteria",
Action: func(cctx *cli.Context) error {
smapi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
onlineOk, err := smapi.DealsConsiderOnlineRetrievalDeals(lcli.DaemonContext(cctx))
if err != nil {
return err
}
offlineOk, err := smapi.DealsConsiderOfflineRetrievalDeals(lcli.DaemonContext(cctx))
if err != nil {
return err
}
fmt.Printf("considering online retrieval deals: %t\n", onlineOk)
fmt.Printf("considering offline retrieval deals: %t\n", offlineOk)
return nil
},
}
var retrievalDealSelectionResetCmd = &cli.Command{
Name: "reset",
Usage: "Reset retrieval deal proposal selection criteria to default values",
Action: func(cctx *cli.Context) error {
smapi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
err = smapi.DealsSetConsiderOnlineRetrievalDeals(lcli.DaemonContext(cctx), true)
if err != nil {
return err
}
err = smapi.DealsSetConsiderOfflineRetrievalDeals(lcli.DaemonContext(cctx), true)
if err != nil {
return err
}
return nil
},
}
var retrievalDealSelectionRejectCmd = &cli.Command{
Name: "reject",
Usage: "Configure criteria which necessitate automatic rejection",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "online",
},
&cli.BoolFlag{
Name: "offline",
},
},
Action: func(cctx *cli.Context) error {
smapi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
if cctx.Bool("online") {
err = smapi.DealsSetConsiderOnlineRetrievalDeals(lcli.DaemonContext(cctx), false)
if err != nil {
return err
}
}
if cctx.Bool("offline") {
err = smapi.DealsSetConsiderOfflineRetrievalDeals(lcli.DaemonContext(cctx), false)
if err != nil {
return err
}
}
return nil
},
}

13
cmd/lotus/daemon.go
View File

@ -32,6 +32,7 @@ import (
"github.com/filecoin-project/lotus/chain/vm"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/lib/peermgr"
"github.com/filecoin-project/lotus/lib/ulimit"
"github.com/filecoin-project/lotus/metrics"
"github.com/filecoin-project/lotus/node"
"github.com/filecoin-project/lotus/node/modules"
@ -113,6 +114,11 @@ var DaemonCmd = &cli.Command{
Name: "profile",
Usage: "specify type of node",
},
&cli.BoolFlag{
Name: "manage-fdlimit",
Usage: "manage open file limit",
Value: true,
},
},
Action: func(cctx *cli.Context) error {
err := runmetrics.Enable(runmetrics.RunMetricOptions{
@ -122,6 +128,13 @@ var DaemonCmd = &cli.Command{
if err != nil {
return xerrors.Errorf("enabling runtime metrics: %w", err)
}
if cctx.Bool("manage-fdlimit") {
if _, _, err := ulimit.ManageFdLimit(); err != nil {
log.Errorf("setting file descriptor limit: %s", err)
}
}
if prof := cctx.String("pprof"); prof != "" {
profile, err := os.Create(prof)
if err != nil {

2
cmd/lotus/debug_advance.go
View File

@ -69,7 +69,7 @@ func init() {
}
// TODO: beacon
uts := head.MinTimestamp() + uint64(build.BlockDelay)
uts := head.MinTimestamp() + uint64(build.BlockDelaySecs)
nheight := head.Height() + 1
blk, err := api.MinerCreateBlock(ctx, &lapi.BlockTemplate{
addr, head.Key(), ticket, &types.ElectionProof{}, nil, msgs, nheight, uts, gen.ValidWpostForTesting,

137
documentation/en/block-validation.md Normal file
View File

@ -0,0 +1,137 @@
# Incoming block validations
This document reviews the code flow that takes place inside the full node after receiving a new block from the GossipSub `/fil/blocks` topic and traces all of its protocol-related validation logic. We do not include validation logic *inside* the VM, the analysis stops at `(*VM).Invoke()`. The `V:` tag explicitly signals validations throughout the text.
## `modules.HandleIncomingBlocks()`
We subscribe to the `/fil/blocks` PubSub topic to receive external blocks from peers in the network and register a block validator that operates at the PubSub (`libp2p` stack) level, validating each PubSub message containing a Filecoin block header.
`V:` PubSub message is a valid CBOR `BlockMsg`.
`V:` Total messages in block are under `BlockMessageLimit`.
`V:` Aggregate message CIDs, encapsulated in the `MsgMeta` structure, serialize to the `Messages` CID in the block header (`ValidateMsgMeta()`).
`V:` Miner `Address` in block header is present and corresponds to a public-key address in the current chain state.
`V:` Block signature (`BlockSig`) is present and belongs to the public-key address retrieved for the miner (`CheckBlockSignature()`).
## `sub.HandleIncomingBlocks()`
Assemble a `FullBlock` from the received block header retrieving its Filecoin messages.
`V:` Block messages CIDs can be retrieved from the network and decode into valid CBOR `Message`/`SignedMessage`.
## `(*Syncer).InformNewHead()`
Assemble a `FullTipSet` populated with the single block received earlier.
`V:` `ValidateMsgMeta()` (already done in the topic validator).
`V:` Block's `ParentWeight` is greater than the one from the (first block of the) heaviest tipset.
## `(*Syncer).Sync()`
`(*Syncer).collectHeaders()`: we retrieve all tipsets from the received block down to our chain. Validation now is expanded to *every* block inside these tipsets.
`V`: Beacon entires are ordered by their round number.
`V:` Tipset `Parents` CIDs match the fetched parent tipset through block sync. (This check is not enforced correctly at the moment, see [issue](https://github.com/filecoin-project/lotus/issues/1918).)
## `(*Syncer).ValidateBlock()`
This function contains most of the validation logic grouped in separate closures that run asynchronously, this list does not reflect validation order then.
`V:` Block `Timestamp`:
* Is not bigger than current time plus `AllowableClockDriftSecs`.
* Is not smaller than previous block's `Timestamp` plus `BlockDelay` (including null blocks).
### Messages
We check all the messages contained in one block at a time (`(*Syncer).checkBlockMessages()`).
`V:` The block's `BLSAggregate` matches the aggregate of BLS messages digests and public keys (extracted from the messages `From`).
`V:` Each `secp256k1` message `Signature` is signed with the public key extracted from the message `From`.
`V:` Aggregate message CIDs, encapsulated in the `MsgMeta` structure, serialize to the `Messages` CID in block header (similar to `ValidateMsgMeta()` call).
`V:` For each message, in `ValidForBlockInclusion()`:
* Message fields `Version`, `To`, `From`, `Value`, `GasPrice`, and `GasLimit` are correctly defined.
* Message `GasLimit` is under the message minimum gas cost (derived from chain height and message length).
`V:` Actor associated with message `From` exists and is an account actor, its `Nonce` matches the message `Nonce`.
### Miner
`V:` Miner address is registered in the `Claims` HAMT of the Power actor.
### Compute parent tipset state
`V:` Block's `ParentStateRoot` CID matches the state CID computed from the parent tipset.
`V:` Block's `ParentMessageReceipts` CID matches receipts CID computed from the parent tipset.
### Winner
Draw randomness for current epoch with minimum ticket from previous tipset, using `ElectionProofProduction`
domain separation tag.
`V`: `ElectionProof.VRFProof` is computed correctly by checking BLS signature using miner's key.
`V`: Miner is not slashed in `StoragePowerActor`.
`V`: Check if ticket is a winning ticket:
```
h := blake2b(VRFProof)
lhs := AsInt(h) * totalNetworkPower
rhs := minerPower * 2^256
if lhs < rhs { return "Winner" } else { return "Not a winner" }
```
### Block signature
`V:` `CheckBlockSignature()` (same signature validation as the one applied to the incoming block).
### Beacon values check
`V`: Validate that all `BeaconEntries` are valid. Check that every one of them is a signature of a message: `previousSignature || round` signed using drand's public key.
`V`: All entries between `MaxBeaconRoundForEpoch` down to `prevEntry` (from previous tipset) are included.
### Verify VRF Ticket chain
Draw randomness for current epoch with minimum ticket from previous tipset, using `TicketProduction`
domain separation tag.
`V`: `VerifyVRF` using drawn randomness and miner public key.
### Winning PoSt proof
Draw randomness for current epoch with `WinningPoSt` domain separation tag.
Get list of sectors challanged in this epoch for this miner, based on the randomness drawn.
`V`: Use filecoin proofs system to verify that miner prooved access to sealed versions of these sectors.
## `(*StateManager).TipSetState()`
Called throughout the validation process for the parent of each tipset being validated. The checks here then do not apply to the received new head itself that started the validation process.
### `(*StateManager).computeTipSetState()`
`V:` Every block in the tipset should belong to different a miner.
### `(*StateManager).ApplyBlocks()`
We create a new VM with the tipset's `ParentStateRoot` (this is then the parent state of the parent of the tipset currently being validated) on which to apply all messages from all blocks in the tipset. For each message independently we apply the validations listed next.
### `(*VM).ApplyMessage()`
`V:` Basic gas and value checks in `checkMessage()`:
* Message `GasLimit` is bigger than zero.
* Message `GasPrice` and `Value` are set.
`V:` Message storage gas cost is under the message's `GasLimit`.
`V:` Message's `Nonce` matches nonce in actor retrieved from message's `From`.
`V:` Message's maximum gas cost (derived from its `GasLimit`, `GasPrice`, and `Value`) is under the balance of the actor retrieved from message's `From`.
### `(*VM).send()`
`V:` Message's transfer `Value` is under the balance in actor retrieved from message's `From`.

2
documentation/en/install-lotus-arch.md
View File

@ -2,7 +2,7 @@
These steps will install the following dependencies:
- go (1.13 or higher)
- go (1.14 or higher)
- gcc (7.4.0 or higher)
- git (version 2 or higher)
- bzr (some go dependency needs this)

2
documentation/en/install-lotus-fedora.md
View File

@ -6,7 +6,7 @@
These steps will install the following dependencies:
- go (1.13 or higher)
- go (1.14 or higher)
- gcc (7.4.0 or higher)
- git (version 2 or higher)
- bzr (some go dependency needs this)

28
documentation/en/install-lotus-ubuntu.md
View File

@ -2,7 +2,7 @@
These steps will install the following dependencies:
- go (1.13 or higher)
- go (1.14 or higher)
- gcc (7.4.0 or higher)
- git (version 2 or higher)
- bzr (some go dependency needs this)
@ -15,29 +15,26 @@ These steps will install the following dependencies:
- llvm (proofs build)
- clang (proofs build)
Run
### Install dependencies
```sh
sudo apt update
sudo apt install mesa-opencl-icd ocl-icd-opencl-dev
sudo apt install mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl
sudo apt upgrade
```
Build
### Install Go 1.14
```sh
sudo add-apt-repository ppa:longsleep/golang-backports
sudo apt update
sudo apt install golang-go gcc git bzr jq pkg-config mesa-opencl-icd ocl-icd-opencl-dev
```
Install the latest version of Go by following [the docs on their website](https://golang.org/doc/install).
Clone
### Clone the Lotus repository
```sh
git clone https://github.com/filecoin-project/lotus.git
cd lotus/
```
Install
### Build the Lotus binaries from source and install
```sh
make clean && make all
@ -45,3 +42,12 @@ sudo make install
```
After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus Testnet](https://docs.lotu.sh/en+join-testnet).
### Interopnet
If you seek a smaller network to test, you can join the `interopnet`. Please note that this network is meant for developers - it resets much more often, and is much smaller. To join this network, checkout the branch `interopnet` instead of `master` before building and installing;
```
git checkout interopnet
```
Please also note that this documentation (if viewed on the website) might not be up to date with the interopnet. For the latest documentation on the interopnet branch, see the [Lotus Documentation Interopnet Branch on GitHub](https://github.com/filecoin-project/lotus/tree/interopnet/documentation/en)

5
documentation/en/mining-troubleshooting.md
View File

@ -38,7 +38,7 @@ If you see this, that means your computer is too slow and your blocks are not in
## Error: No space left on device
```sh
lotus-storage-miner pledge-sector
lotus-storage-miner sectors pledge
# No space left on device (os error 28)
```
@ -51,7 +51,8 @@ If you suspect that your GPU is not being used, first make sure it is properly c
First, to watch GPU utilization run `nvtop` in one terminal, then in a separate terminal, run:
```sh
lotus-bench --sector-size=2KiB
make bench
./bench sealing --sector-size=2KiB
```
This process uses a fair amount of GPU, and generally takes ~4 minutes to complete. If you do not see any activity in nvtop from lotus during the entire process, it is likely something is misconfigured with your GPU.

6
documentation/en/mining.md
View File

@ -82,12 +82,12 @@ lotus-storage-miner sectors pledge
Get **miner power** and **sector usage**:
```sh
lotus-storage-miner state power
lotus state power
# returns total power
lotus-storage-miner state power <miner>
lotus state power <miner>
lotus-storage-miner state sectors <miner>
lotus state sectors <miner>
```
## Performance tuning

2
extern/filecoin-ffi vendored

@ -1 +1 @@
Subproject commit ca281af0b6c00314382a75ae869e5cb22c83655b
Subproject commit 5342c7c97d1a1df4650629d14f2823d52889edd9

18
go.mod
View File

@ -1,6 +1,6 @@
module github.com/filecoin-project/lotus
go 1.13
go 1.14
require (
contrib.go.opencensus.io/exporter/jaeger v0.1.0
@ -18,21 +18,21 @@ require (
github.com/filecoin-project/filecoin-ffi v0.26.1-0.20200508175440-05b30afeb00d
github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef
github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2
github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e
github.com/filecoin-project/go-bitfield v0.0.2-0.20200629135455-587b27927d38
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
github.com/filecoin-project/go-data-transfer v0.3.0
github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5
github.com/filecoin-project/go-fil-markets v0.3.0
github.com/filecoin-project/go-fil-markets v0.3.2-0.20200702145639-4034a18364e4
github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24
github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6
github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca
github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261
github.com/filecoin-project/go-statestore v0.1.0
github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
github.com/filecoin-project/sector-storage v0.0.0-20200623224636-de544b531601
github.com/filecoin-project/specs-actors v0.6.2-0.20200617175406-de392ca14121
github.com/filecoin-project/sector-storage v0.0.0-20200630180318-4c1968f62a8f
github.com/filecoin-project/specs-actors v0.6.2-0.20200702170846-2cd72643a5cf
github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea
github.com/filecoin-project/storage-fsm v0.0.0-20200623213010-fe71d5b42de3
github.com/filecoin-project/storage-fsm v0.0.0-20200625160832-379a4655b044
github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
github.com/go-kit/kit v0.10.0
github.com/go-ole/go-ole v1.2.4 // indirect
@ -44,7 +44,7 @@ require (
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d
github.com/ipfs/go-bitswap v0.2.8
github.com/ipfs/go-block-format v0.0.2
github.com/ipfs/go-blockservice v0.1.3
github.com/ipfs/go-blockservice v0.1.4-0.20200624145336-a978cec6e834
github.com/ipfs/go-cid v0.0.6
github.com/ipfs/go-cidutil v0.0.2
github.com/ipfs/go-datastore v0.4.4
@ -66,7 +66,7 @@ require (
github.com/ipfs/go-ipld-cbor v0.0.5-0.20200428170625-a0bd04d3cbdf
github.com/ipfs/go-ipld-format v0.2.0
github.com/ipfs/go-log v1.0.4
github.com/ipfs/go-log/v2 v2.1.2-0.20200609205458-f8d20c392cb7
github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4
github.com/ipfs/go-merkledag v0.3.1
github.com/ipfs/go-path v0.0.7
github.com/ipfs/go-unixfs v0.2.4

33
go.sum
View File

@ -225,6 +225,8 @@ github.com/filecoin-project/go-bitfield v0.0.0-20200416002808-b3ee67ec9060/go.mo
github.com/filecoin-project/go-bitfield v0.0.1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY=
github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e h1:gkG/7G+iKy4He+IiQNeQn+nndFznb/vCoOR8iRQsm60=
github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY=
github.com/filecoin-project/go-bitfield v0.0.2-0.20200629135455-587b27927d38 h1:B2gUde2DlfCb5YMYNVems2orobxC3KhrX3migym1IOQ=
github.com/filecoin-project/go-bitfield v0.0.2-0.20200629135455-587b27927d38/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY=
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8=
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
@ -233,16 +235,16 @@ github.com/filecoin-project/go-data-transfer v0.3.0 h1:BwBrrXu9Unh9JjjX4GAc5FfzU
github.com/filecoin-project/go-data-transfer v0.3.0/go.mod h1:cONglGP4s/d+IUQw5mWZrQK+FQATQxr3AXzi4dRh0l4=
github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1:yvQJCW9mmi9zy+51xA01Ea2X7/dL7r8eKDPuGUjRmbo=
github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA=
github.com/filecoin-project/go-fil-markets v0.3.0 h1:7iCGiuTSia4f4DmOn3s96NWUwMNSOI0ZHel/XgeApAQ=
github.com/filecoin-project/go-fil-markets v0.3.0/go.mod h1:UXsXi43AyUQ5ieb4yIaLgk4PVt7TAbl1UCccuNw+7ds=
github.com/filecoin-project/go-fil-markets v0.3.2-0.20200702145639-4034a18364e4 h1:VqNmKGy4/ryzo/TqevSa1kancc3hSdws7sl/NCTZzT0=
github.com/filecoin-project/go-fil-markets v0.3.2-0.20200702145639-4034a18364e4/go.mod h1:UY+/zwNXHN73HcrN6HxNDpv6KKM6ehqfCuE9vK9khF8=
github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms=
github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM=
github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs=
github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6/go.mod h1:0HgYnrkeSU4lu1p+LEOeDpFsNBssa0OGGriWdA4hvaE=
github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc=
github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc=
github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca h1:OGykrCr6mSn/ckk2IFbIlkc76nsgEs7tSLhZXQt7+z4=
github.com/filecoin-project/go-paramfetch v0.0.2-0.20200605171344-fcac609550ca/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc=
github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 h1:A256QonvzRaknIIAuWhe/M2dpV2otzs3NBhi5TWa/UA=
github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc=
github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 h1:k9qVR9ItcziSB2rxtlkN/MDWNlbsI6yzec+zjUatLW0=
github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
github.com/filecoin-project/go-statemachine v0.0.0-20200612181802-4eb3d0c68eba h1:GEWb/6KQyNZt4jm8fgVcIFPH0ElAGXfHM59ZSiqPTvY=
@ -252,21 +254,22 @@ github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZO
github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg=
github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8=
github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM=
github.com/filecoin-project/sector-storage v0.0.0-20200623210524-47d93356586d h1:yJJqXCMEhvXJoOS6T1O46FXl+A3mlttXhgjcTCp+Tgo=
github.com/filecoin-project/sector-storage v0.0.0-20200623210524-47d93356586d/go.mod h1:8f0hWDzzIi1hKs4IVKH9RnDsO4LEHVz8BNat0okDOuY=
github.com/filecoin-project/sector-storage v0.0.0-20200623224636-de544b531601 h1:EgMmHLoJ4caLU8RzgKQux4TyX/ZploXGtIu5Q1SaxKw=
github.com/filecoin-project/sector-storage v0.0.0-20200623224636-de544b531601/go.mod h1:8f0hWDzzIi1hKs4IVKH9RnDsO4LEHVz8BNat0okDOuY=
github.com/filecoin-project/sector-storage v0.0.0-20200625154333-98ef8e4ef246 h1:NfYQRmVRe0LzlNbK5Ket3vbBOwFD5TvtcNtfo/Sd8mg=
github.com/filecoin-project/sector-storage v0.0.0-20200625154333-98ef8e4ef246/go.mod h1:8f0hWDzzIi1hKs4IVKH9RnDsO4LEHVz8BNat0okDOuY=
github.com/filecoin-project/sector-storage v0.0.0-20200630180318-4c1968f62a8f h1:EHKqNJNIcYggqfrd5nu7SV1KR93ReZygfdSV0w/jefQ=
github.com/filecoin-project/sector-storage v0.0.0-20200630180318-4c1968f62a8f/go.mod h1:r12d7tsmJKz8QDGoCvl65Ay2al6mOgDqxAGUxbyrgMs=
github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA=
github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y=
github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY=
github.com/filecoin-project/specs-actors v0.6.2-0.20200617175406-de392ca14121 h1:oRA+b4iN4H86xXDXbU3TOyvmBZp7//c5VqTc0oJ6nLg=
github.com/filecoin-project/specs-actors v0.6.2-0.20200617175406-de392ca14121/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY=
github.com/filecoin-project/specs-actors v0.6.1/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY=
github.com/filecoin-project/specs-actors v0.6.2-0.20200702170846-2cd72643a5cf h1:2ERozAZteHYef3tVLVJRepzYieLtJdxvfXNUel19CeU=
github.com/filecoin-project/specs-actors v0.6.2-0.20200702170846-2cd72643a5cf/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY=
github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sgOVdJbvFjOnD5w94=
github.com/filecoin-project/specs-storage v0.1.0/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k=
github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea h1:iixjULRQFPn7Q9KlIqfwLJnlAXO10bbkI+xy5GKGdLY=
github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k=
github.com/filecoin-project/storage-fsm v0.0.0-20200623213010-fe71d5b42de3 h1:nH3L7YVqrHINOmvZ+5jFjFNSi9/swXcm+uufXpkFJfo=
github.com/filecoin-project/storage-fsm v0.0.0-20200623213010-fe71d5b42de3/go.mod h1:Nl0JX9I3fIVtPEJ9HzGzO4D8LXehT9PqvUQUbNvcstc=
github.com/filecoin-project/storage-fsm v0.0.0-20200625160832-379a4655b044 h1:i4oMhv1kx/MAUxRN4EM5tag5fI1uagrwQwINgKrzUt4=
github.com/filecoin-project/storage-fsm v0.0.0-20200625160832-379a4655b044/go.mod h1:JD7fmV1BYADDcy4EYQnqFH/rUzXsh0Je0jXarCjZqSk=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
@ -467,6 +470,8 @@ github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbR
github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M=
github.com/ipfs/go-blockservice v0.1.3 h1:9XgsPMwwWJSC9uVr2pMDsW2qFTBSkxpGMhmna8mIjPM=
github.com/ipfs/go-blockservice v0.1.3/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU=
github.com/ipfs/go-blockservice v0.1.4-0.20200624145336-a978cec6e834 h1:hFJoI1D2a3MqiNkSb4nKwrdkhCngUxUTFNwVwovZX2s=
github.com/ipfs/go-blockservice v0.1.4-0.20200624145336-a978cec6e834/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU=
github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
@ -589,8 +594,8 @@ github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBW
github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0=
github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw=
github.com/ipfs/go-log/v2 v2.0.8/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw=
github.com/ipfs/go-log/v2 v2.1.2-0.20200609205458-f8d20c392cb7 h1:LtL/rvdfbKSthZGmAAD9o4KKg6HA6Qn8gXCCdgnj7lw=
github.com/ipfs/go-log/v2 v2.1.2-0.20200609205458-f8d20c392cb7/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM=
github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4 h1:3bijxqzQ1O9yg7gd7Aqk80oaEvsJ+uXw0zSvi2qR3Jw=
github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM=
github.com/ipfs/go-merkledag v0.0.3/go.mod h1:Oc5kIXLHokkE1hWGMBHw+oxehkAaTOqtEb7Zbh6BhLA=
github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto=
github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=

9
lib/bufbstore/buf_bstore.go
View File

@ -108,6 +108,15 @@ func (bs *BufferedBS) GetSize(c cid.Cid) (int, error) {
}
func (bs *BufferedBS) Put(blk block.Block) error {
has, err := bs.read.Has(blk.Cid())
if err != nil {
return err
}
if has {
return nil
}
return bs.write.Put(blk)
}

38
lib/sigs/bls/bls_bench_test.go Normal file
View File

@ -0,0 +1,38 @@
package bls
import (
"crypto/rand"
"github.com/filecoin-project/go-address"
"testing"
)
func BenchmarkBLSSign(b *testing.B) {
signer := blsSigner{}
for i := 0; i < b.N; i++ {
b.StopTimer()
pk, _ := signer.GenPrivate()
randMsg := make([]byte, 32)
rand.Read(randMsg)
b.StartTimer()
_, _ = signer.Sign(pk, randMsg)
}
}
func BenchmarkBLSVerify(b *testing.B) {
signer := blsSigner{}
for i := 0; i < b.N; i++ {
b.StopTimer()
randMsg := make([]byte, 32)
rand.Read(randMsg)
priv, _ := signer.GenPrivate()
pk, _ := signer.ToPublic(priv)
addr, _ := address.NewBLSAddress(pk)
sig, _ := signer.Sign(priv, randMsg)
b.StartTimer()
_ = signer.Verify(sig, addr, randMsg)
}
}

35
markets/storageadapter/client.go
View File

@ -19,7 +19,7 @@ import (
"github.com/filecoin-project/specs-actors/actors/crypto"
"github.com/filecoin-project/specs-actors/actors/runtime/exitcode"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/multiformats/go-multiaddr"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/events"
@ -80,7 +80,15 @@ func (n *ClientNodeAdapter) ListStorageProviders(ctx context.Context, encodedTs
return nil, err
}
storageProviderInfo := utils.NewStorageProviderInfo(addr, mi.Worker, mi.SectorSize, peer.ID(mi.PeerId))
multiaddrs := make([]multiaddr.Multiaddr, 0, len(mi.Multiaddrs))
for _, a := range mi.Multiaddrs {
maddr, err := multiaddr.NewMultiaddrBytes(a)
if err != nil {
return nil, err
}
multiaddrs = append(multiaddrs, maddr)
}
storageProviderInfo := utils.NewStorageProviderInfo(addr, mi.Worker, mi.SectorSize, mi.PeerId, multiaddrs)
out = append(out, &storageProviderInfo)
}
@ -322,7 +330,7 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider
}
}
if err := c.ev.Called(checkFunc, called, revert, build.MessageConfidence+1, build.SealRandomnessLookbackLimit, matchEvent); err != nil {
if err := c.ev.Called(checkFunc, called, revert, int(build.MessageConfidence+1), build.SealRandomnessLookbackLimit, matchEvent); err != nil {
return xerrors.Errorf("failed to set up called handler: %w", err)
}
@ -405,4 +413,25 @@ func (n *ClientNodeAdapter) WaitForMessage(ctx context.Context, mcid cid.Cid, cb
return cb(receipt.Receipt.ExitCode, receipt.Receipt.Return, nil)
}
func (n *ClientNodeAdapter) GetMinerInfo(ctx context.Context, addr address.Address, encodedTs shared.TipSetToken) (*storagemarket.StorageProviderInfo, error) {
tsk, err := types.TipSetKeyFromBytes(encodedTs)
if err != nil {
return nil, err
}
mi, err := n.StateMinerInfo(ctx, addr, tsk)
if err != nil {
return nil, err
}
multiaddrs := make([]multiaddr.Multiaddr, 0, len(mi.Multiaddrs))
for _, a := range mi.Multiaddrs {
maddr, err := multiaddr.NewMultiaddrBytes(a)
if err != nil {
return nil, err
}
multiaddrs = append(multiaddrs, maddr)
}
out := utils.NewStorageProviderInfo(addr, mi.Worker, mi.SectorSize, mi.PeerId, multiaddrs)
return &out, nil
}
var _ storagemarket.StorageClientNode = &ClientNodeAdapter{}

2
markets/storageadapter/provider.go
View File

@ -321,7 +321,7 @@ func (n *ProviderNodeAdapter) OnDealSectorCommitted(ctx context.Context, provide
}
if err := n.ev.Called(checkFunc, called, revert, build.MessageConfidence+1, build.SealRandomnessLookbackLimit, matchEvent); err != nil {
if err := n.ev.Called(checkFunc, called, revert, int(build.MessageConfidence+1), build.SealRandomnessLookbackLimit, matchEvent); err != nil {
return xerrors.Errorf("failed to set up called handler: %w", err)
}

5
markets/utils/converters.go
View File

@ -6,17 +6,20 @@ import (
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/filecoin-project/specs-actors/actors/builtin/market"
peer "github.com/libp2p/go-libp2p-core/peer"
"github.com/multiformats/go-multiaddr"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-fil-markets/storagemarket"
)
func NewStorageProviderInfo(address address.Address, miner address.Address, sectorSize abi.SectorSize, peer peer.ID) storagemarket.StorageProviderInfo {
func NewStorageProviderInfo(address address.Address, miner address.Address, sectorSize abi.SectorSize, peer peer.ID, addrs []multiaddr.Multiaddr) storagemarket.StorageProviderInfo {
return storagemarket.StorageProviderInfo{
Address: address,
Worker: miner,
SectorSize: uint64(sectorSize),
PeerID: peer,
Addrs: addrs,
}
}

25
miner/miner.go
View File

@ -42,7 +42,7 @@ func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address)
address: addr,
waitFunc: func(ctx context.Context, baseTime uint64) (func(bool), error) {
// Wait around for half the block time in case other parents come in
deadline := baseTime + build.PropagationDelay
deadline := baseTime + build.PropagationDelaySecs
time.Sleep(time.Until(time.Unix(int64(deadline), 0)))
return func(bool) {}, nil
@ -140,7 +140,7 @@ func (m *Miner) mine(ctx context.Context) {
onDone, err := m.waitFunc(ctx, prebase.TipSet.MinTimestamp())
if err != nil {
log.Error(err)
return
continue
}
base, err := m.GetBestMiningCandidate(ctx)
@ -150,7 +150,7 @@ func (m *Miner) mine(ctx context.Context) {
}
if base.TipSet.Equals(lastBase.TipSet) && lastBase.NullRounds == base.NullRounds {
log.Warnf("BestMiningCandidate from the previous round: %s (nulls:%d)", lastBase.TipSet.Cids(), lastBase.NullRounds)
m.niceSleep(build.BlockDelay * time.Second)
m.niceSleep(time.Duration(build.BlockDelaySecs) * time.Second)
continue
}
@ -194,7 +194,7 @@ func (m *Miner) mine(ctx context.Context) {
// has enough time to form.
//
// See: https://github.com/filecoin-project/lotus/issues/1845
nextRound := time.Unix(int64(base.TipSet.MinTimestamp()+uint64(build.BlockDelay*base.NullRounds))+int64(build.PropagationDelay), 0)
nextRound := time.Unix(int64(base.TipSet.MinTimestamp()+build.BlockDelaySecs*uint64(base.NullRounds))+int64(build.PropagationDelaySecs), 0)
select {
case <-time.After(time.Until(nextRound)):
@ -215,6 +215,9 @@ type MiningBase struct {
}
func (m *Miner) GetBestMiningCandidate(ctx context.Context) (*MiningBase, error) {
m.lk.Lock()
defer m.lk.Unlock()
bts, err := m.api.ChainHead(ctx)
if err != nil {
return nil, err
@ -252,12 +255,16 @@ func (m *Miner) hasPower(ctx context.Context, addr address.Address, ts *types.Ti
return mpower.MinerPower.QualityAdjPower.GreaterThanEqual(power.ConsensusMinerMinPower), nil
}
// mineOne mines a single block, and does so synchronously, if and only if we
// have won the current round.
// mineOne attempts to mine a single block, and does so synchronously, if and
// only if we are eligible to mine.
//
// {hint/landmark}: This method coordinates all the steps involved in mining a
// block, including the condition of whether mine or not at all depending on
// whether we win the round or not.
//
// This method does the following:
//
// 1.
func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*types.BlockMsg, error) {
log.Debugw("attempting to mine a block", "tipset", types.LogCids(base.TipSet.Cids()))
start := time.Now()
@ -349,7 +356,7 @@ func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*types.BlockMsg,
tCreateBlock := time.Now()
dur := tCreateBlock.Sub(start)
log.Infow("mined new block", "cid", b.Cid(), "height", b.Header.Height, "took", dur)
if dur > time.Second*build.BlockDelay {
if dur > time.Second*time.Duration(build.BlockDelaySecs) {
log.Warn("CAUTION: block production took longer than the block delay. Your computer may not be fast enough to keep up")
log.Warnw("tMinerBaseInfo ", "duration", tMBI.Sub(start))
@ -410,7 +417,7 @@ func (m *Miner) createBlock(base *MiningBase, addr address.Address, ticket *type
msgs = msgs[:build.BlockMessageLimit]
}
uts := base.TipSet.MinTimestamp() + uint64(build.BlockDelay*(base.NullRounds+1))
uts := base.TipSet.MinTimestamp() + build.BlockDelaySecs*(uint64(base.NullRounds)+1)
nheight := base.TipSet.Height() + base.NullRounds + 1
@ -554,7 +561,7 @@ func SelectMessages(ctx context.Context, al ActorLookup, ts *types.TipSet, msgs
}
if tooHighNonceMsgs > 0 {
log.Warnf("%d messages in mempool had too high nonce", tooLowFundMsgs)
log.Warnf("%d messages in mempool had too high nonce", tooHighNonceMsgs)
}
sm := time.Now()

10
node/builder.go
View File

@ -313,10 +313,16 @@ func Online() Option {
Override(new(gen.WinningPoStProver), storage.NewWinningPoStProver),
Override(new(*miner.Miner), modules.SetupBlockProducer),
Override(new(dtypes.AcceptingStorageDealsConfigFunc), modules.NewAcceptingStorageDealsConfigFunc),
Override(new(dtypes.SetAcceptingStorageDealsConfigFunc), modules.NewSetAcceptingStorageDealsConfigFunc),
Override(new(dtypes.ConsiderOnlineStorageDealsConfigFunc), modules.NewConsiderOnlineStorageDealsConfigFunc),
Override(new(dtypes.SetConsiderOnlineStorageDealsConfigFunc), modules.NewSetConsideringOnlineStorageDealsFunc),
Override(new(dtypes.ConsiderOnlineRetrievalDealsConfigFunc), modules.NewConsiderOnlineRetrievalDealsConfigFunc),
Override(new(dtypes.SetConsiderOnlineRetrievalDealsConfigFunc), modules.NewSetConsiderOnlineRetrievalDealsConfigFunc),
Override(new(dtypes.StorageDealPieceCidBlocklistConfigFunc), modules.NewStorageDealPieceCidBlocklistConfigFunc),
Override(new(dtypes.SetStorageDealPieceCidBlocklistConfigFunc), modules.NewSetStorageDealPieceCidBlocklistConfigFunc),
Override(new(dtypes.ConsiderOfflineStorageDealsConfigFunc), modules.NewConsiderOfflineStorageDealsConfigFunc),
Override(new(dtypes.SetConsiderOfflineStorageDealsConfigFunc), modules.NewSetConsideringOfflineStorageDealsFunc),
Override(new(dtypes.ConsiderOfflineRetrievalDealsConfigFunc), modules.NewConsiderOfflineRetrievalDealsConfigFunc),
Override(new(dtypes.SetConsiderOfflineRetrievalDealsConfigFunc), modules.NewSetConsiderOfflineRetrievalDealsConfigFunc),
),
)
}

14
node/config/def.go
View File

@ -34,8 +34,11 @@ type StorageMiner struct {
}
type DealmakingConfig struct {
AcceptingStorageDeals bool
PieceCidBlocklist []cid.Cid
ConsiderOnlineStorageDeals bool
ConsiderOfflineStorageDeals bool
ConsiderOnlineRetrievalDeals bool
ConsiderOfflineRetrievalDeals bool
PieceCidBlocklist []cid.Cid
}
// API contains configs for API endpoint
@ -123,8 +126,11 @@ func DefaultStorageMiner() *StorageMiner {
},
Dealmaking: DealmakingConfig{
AcceptingStorageDeals: true,
PieceCidBlocklist: []cid.Cid{},
ConsiderOnlineStorageDeals: true,
ConsiderOfflineStorageDeals: true,
ConsiderOnlineRetrievalDeals: true,
ConsiderOfflineRetrievalDeals: true,
PieceCidBlocklist: []cid.Cid{},
},
}
cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http"

14
node/impl/client/client.go
View File

@ -9,6 +9,7 @@ import (
basicnode "github.com/ipld/go-ipld-prime/node/basic"
"github.com/ipld/go-ipld-prime/traversal/selector"
"github.com/ipld/go-ipld-prime/traversal/selector/builder"
"github.com/multiformats/go-multiaddr"
"io"
"os"
@ -90,6 +91,15 @@ func (a *API) ClientStartDeal(ctx context.Context, params *api.StartDealParams)
return nil, xerrors.Errorf("failed getting peer ID: %w", err)
}
multiaddrs := make([]multiaddr.Multiaddr, 0, len(mi.Multiaddrs))
for _, a := range mi.Multiaddrs {
maddr, err := multiaddr.NewMultiaddrBytes(a)
if err != nil {
return nil, err
}
multiaddrs = append(multiaddrs, maddr)
}
md, err := a.StateMinerProvingDeadline(ctx, params.Miner, types.EmptyTSK)
if err != nil {
return nil, xerrors.Errorf("failed getting peer ID: %w", err)
@ -104,7 +114,7 @@ func (a *API) ClientStartDeal(ctx context.Context, params *api.StartDealParams)
return nil, xerrors.New("data doesn't fit in a sector")
}
providerInfo := utils.NewStorageProviderInfo(params.Miner, mi.Worker, mi.SectorSize, peer.ID(mi.PeerId))
providerInfo := utils.NewStorageProviderInfo(params.Miner, mi.Worker, mi.SectorSize, mi.PeerId, multiaddrs)
dealStart := params.DealStartEpoch
if dealStart <= 0 { // unset, or explicitly 'epoch undefined'
@ -431,7 +441,7 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
}
func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) {
info := utils.NewStorageProviderInfo(miner, address.Undef, 0, p)
info := utils.NewStorageProviderInfo(miner, address.Undef, 0, p, nil)
signedAsk, err := a.SMDealClient.GetAsk(ctx, info)
if err != nil {
return nil, err

2
node/impl/common/common.go
View File

@ -122,7 +122,7 @@ func (a *CommonAPI) Version(context.Context) (api.Version, error) {
Version: build.UserVersion(),
APIVersion: build.APIVersion,
BlockDelay: build.BlockDelay,
BlockDelay: build.BlockDelaySecs,
}, nil
}

24
node/impl/full/state.go
View File

@ -15,6 +15,15 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-amt-ipld/v2"
"github.com/filecoin-project/sector-storage/ffiwrapper"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/market"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
samsig "github.com/filecoin-project/specs-actors/actors/builtin/multisig"
"github.com/filecoin-project/specs-actors/actors/builtin/power"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/beacon"
@ -27,14 +36,6 @@ import (
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/lib/bufbstore"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/sector-storage/ffiwrapper"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/market"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
samsig "github.com/filecoin-project/specs-actors/actors/builtin/multisig"
"github.com/filecoin-project/specs-actors/actors/builtin/power"
)
type StateAPI struct {
@ -300,7 +301,7 @@ func (a *StateAPI) StateAccountKey(ctx context.Context, addr address.Address, ts
return a.StateManager.ResolveToKeyAddress(ctx, addr, ts)
}
func (a *StateAPI) StateReadState(ctx context.Context, act *types.Actor, tsk types.TipSetKey) (*api.ActorState, error) {
func (a *StateAPI) StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) {
ts, err := a.Chain.GetTipSetFromKey(tsk)
if err != nil {
return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
@ -310,6 +311,11 @@ func (a *StateAPI) StateReadState(ctx context.Context, act *types.Actor, tsk typ
return nil, err
}
act, err := state.GetActor(actor)
if err != nil {
return nil, err
}
blk, err := state.Store.(*cbor.BasicIpldStore).Blocks.Get(act.Head)
if err != nil {
return nil, err

45
node/impl/storminer.go
View File

@ -43,9 +43,16 @@ type StorageMinerAPI struct {
StorageMgr *sectorstorage.Manager `optional:"true"`
*stores.Index
SetAcceptingStorageDealsConfigFunc dtypes.SetAcceptingStorageDealsConfigFunc
StorageDealPieceCidBlocklistConfigFunc dtypes.StorageDealPieceCidBlocklistConfigFunc
SetStorageDealPieceCidBlocklistConfigFunc dtypes.SetStorageDealPieceCidBlocklistConfigFunc
ConsiderOnlineStorageDealsConfigFunc dtypes.ConsiderOnlineStorageDealsConfigFunc
SetConsiderOnlineStorageDealsConfigFunc dtypes.SetConsiderOnlineStorageDealsConfigFunc
ConsiderOnlineRetrievalDealsConfigFunc dtypes.ConsiderOnlineRetrievalDealsConfigFunc
SetConsiderOnlineRetrievalDealsConfigFunc dtypes.SetConsiderOnlineRetrievalDealsConfigFunc
StorageDealPieceCidBlocklistConfigFunc dtypes.StorageDealPieceCidBlocklistConfigFunc
SetStorageDealPieceCidBlocklistConfigFunc dtypes.SetStorageDealPieceCidBlocklistConfigFunc
ConsiderOfflineStorageDealsConfigFunc dtypes.ConsiderOfflineStorageDealsConfigFunc
SetConsiderOfflineStorageDealsConfigFunc dtypes.SetConsiderOfflineStorageDealsConfigFunc
ConsiderOfflineRetrievalDealsConfigFunc dtypes.ConsiderOfflineRetrievalDealsConfigFunc
SetConsiderOfflineRetrievalDealsConfigFunc dtypes.SetConsiderOfflineRetrievalDealsConfigFunc
}
func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) {
@ -224,8 +231,36 @@ func (sm *StorageMinerAPI) DealsList(ctx context.Context) ([]storagemarket.Stora
return sm.StorageProvider.ListDeals(ctx)
}
func (sm *StorageMinerAPI) DealsSetAcceptingStorageDeals(ctx context.Context, b bool) error {
return sm.SetAcceptingStorageDealsConfigFunc(b)
func (sm *StorageMinerAPI) DealsConsiderOnlineStorageDeals(ctx context.Context) (bool, error) {
return sm.ConsiderOnlineStorageDealsConfigFunc()
}
func (sm *StorageMinerAPI) DealsSetConsiderOnlineStorageDeals(ctx context.Context, b bool) error {
return sm.SetConsiderOnlineStorageDealsConfigFunc(b)
}
func (sm *StorageMinerAPI) DealsConsiderOnlineRetrievalDeals(ctx context.Context) (bool, error) {
return sm.ConsiderOnlineRetrievalDealsConfigFunc()
}
func (sm *StorageMinerAPI) DealsSetConsiderOnlineRetrievalDeals(ctx context.Context, b bool) error {
return sm.SetConsiderOnlineRetrievalDealsConfigFunc(b)
}
func (sm *StorageMinerAPI) DealsConsiderOfflineStorageDeals(ctx context.Context) (bool, error) {
return sm.ConsiderOfflineStorageDealsConfigFunc()
}
func (sm *StorageMinerAPI) DealsSetConsiderOfflineStorageDeals(ctx context.Context, b bool) error {
return sm.SetConsiderOfflineStorageDealsConfigFunc(b)
}
func (sm *StorageMinerAPI) DealsConsiderOfflineRetrievalDeals(ctx context.Context) (bool, error) {
return sm.ConsiderOfflineRetrievalDealsConfigFunc()
}
func (sm *StorageMinerAPI) DealsSetConsiderOfflineRetrievalDeals(ctx context.Context, b bool) error {
return sm.SetConsiderOfflineRetrievalDealsConfigFunc(b)
}
func (sm *StorageMinerAPI) DealsImportData(ctx context.Context, deal cid.Cid, fname string) error {

42
node/modules/dtypes/miner.go
View File

@ -10,19 +10,43 @@ import (
type MinerAddress address.Address
type MinerID abi.ActorID
// AcceptingStorageDealsFunc is a function which reads from miner config to
// determine if the user has disabled storage deals (or not).
type AcceptingStorageDealsConfigFunc func() (bool, error)
// ConsiderOnlineStorageDealsConfigFunc is a function which reads from miner
// config to determine if the user has disabled storage deals (or not).
type ConsiderOnlineStorageDealsConfigFunc func() (bool, error)
// SetAcceptingStorageDealsFunc is a function which is used to disable or enable
// storage deal acceptance.
type SetAcceptingStorageDealsConfigFunc func(bool) error
// SetConsiderOnlineStorageDealsConfigFunc is a function which is used to
// disable or enable storage deal acceptance.
type SetConsiderOnlineStorageDealsConfigFunc func(bool) error
// StorageDealPieceCidBlocklistConfigFunc is a function which reads from miner config
// to obtain a list of CIDs for which the storage miner will not accept storage
// proposals.
// ConsiderOnlineRetrievalDealsConfigFunc is a function which reads from miner
// config to determine if the user has disabled retrieval acceptance (or not).
type ConsiderOnlineRetrievalDealsConfigFunc func() (bool, error)
// SetConsiderOnlineRetrievalDealsConfigFunc is a function which is used to
// disable or enable retrieval deal acceptance.
type SetConsiderOnlineRetrievalDealsConfigFunc func(bool) error
// StorageDealPieceCidBlocklistConfigFunc is a function which reads from miner
// config to obtain a list of CIDs for which the storage miner will not accept
// storage proposals.
type StorageDealPieceCidBlocklistConfigFunc func() ([]cid.Cid, error)
// SetStorageDealPieceCidBlocklistConfigFunc is a function which is used to set a
// list of CIDs for which the storage miner will reject deal proposals.
type SetStorageDealPieceCidBlocklistConfigFunc func([]cid.Cid) error
// ConsiderOfflineStorageDealsConfigFunc is a function which reads from miner
// config to determine if the user has disabled storage deals (or not).
type ConsiderOfflineStorageDealsConfigFunc func() (bool, error)
// SetConsiderOfflineStorageDealsConfigFunc is a function which is used to
// disable or enable storage deal acceptance.
type SetConsiderOfflineStorageDealsConfigFunc func(bool) error
// ConsiderOfflineRetrievalDealsConfigFunc is a function which reads from miner
// config to determine if the user has disabled retrieval acceptance (or not).
type ConsiderOfflineRetrievalDealsConfigFunc func() (bool, error)
// SetConsiderOfflineRetrievalDealsConfigFunc is a function which is used to
// disable or enable retrieval deal acceptance.
type SetConsiderOfflineRetrievalDealsConfigFunc func(bool) error

11
node/modules/lp2p/libp2p.go
View File

@ -19,7 +19,10 @@ import (
var log = logging.Logger("p2pnode")
const kstorePrivkey = "libp2p-host"
const (
KLibp2pHost = "libp2p-host"
KTLibp2pHost = KLibp2pHost
)
type Libp2pOpts struct {
fx.Out
@ -28,7 +31,7 @@ type Libp2pOpts struct {
}
func PrivKey(ks types.KeyStore) (crypto.PrivKey, error) {
k, err := ks.Get(kstorePrivkey)
k, err := ks.Get(KLibp2pHost)
if err == nil {
return crypto.UnmarshalPrivateKey(k.PrivateKey)
}
@ -44,8 +47,8 @@ func PrivKey(ks types.KeyStore) (crypto.PrivKey, error) {
return nil, err
}
if err := ks.Put(kstorePrivkey, types.KeyInfo{
Type: kstorePrivkey,
if err := ks.Put(KLibp2pHost, types.KeyInfo{
Type: KTLibp2pHost,
PrivateKey: kbytes,
}); err != nil {
return nil, err

6
node/modules/services.go
View File

@ -41,7 +41,7 @@ func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello.
pic := evt.(event.EvtPeerIdentificationCompleted)
go func() {
if err := svc.SayHello(helpers.LifecycleCtx(mctx, lc), pic.Peer); err != nil {
log.Warnw("failed to say hello", "error", err)
log.Warnw("failed to say hello", "error", err, "peer", pic.Peer)
return
}
}()
@ -123,6 +123,6 @@ func RandomBeacon(p RandomBeaconParams, _ dtypes.AfterGenesisSet) (beacon.Random
return nil, err
}
//return beacon.NewMockBeacon(build.BlockDelay * time.Second)
return drand.NewDrandBeacon(gen.Timestamp, build.BlockDelay, p.PubSub, p.DrandConfig)
//return beacon.NewMockBeacon(build.BlockDelaySecs * time.Second)
return drand.NewDrandBeacon(gen.Timestamp, build.BlockDelaySecs, p.PubSub, p.DrandConfig)
}

116
node/modules/storageminer.go
View File

@ -315,7 +315,7 @@ func NewStorageAsk(ctx helpers.MetricsCtx, fapi lapi.FullNode, ds dtypes.Metadat
return storedAsk, nil
}
func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Config, storedAsk *storedask.StoredAsk, h host.Host, ds dtypes.MetadataDS, ibs dtypes.StagingBlockstore, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode, isAcceptingFunc dtypes.AcceptingStorageDealsConfigFunc, blocklistFunc dtypes.StorageDealPieceCidBlocklistConfigFunc) (storagemarket.StorageProvider, error) {
func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Config, storedAsk *storedask.StoredAsk, h host.Host, ds dtypes.MetadataDS, ibs dtypes.StagingBlockstore, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode, onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc, offlineOk dtypes.ConsiderOfflineStorageDealsConfigFunc, blocklistFunc dtypes.StorageDealPieceCidBlocklistConfigFunc) (storagemarket.StorageProvider, error) {
net := smnet.NewFromLibp2pHost(h)
store, err := piecefilestore.NewLocalFileStore(piecefilestore.OsPath(r.Path()))
if err != nil {
@ -323,14 +323,24 @@ func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Con
}
opt := storageimpl.CustomDealDecisionLogic(func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) {
b, err := isAcceptingFunc()
b, err := onlineOk()
if err != nil {
return false, "miner error", err
}
if !b {
log.Warnf("storage deal acceptance disabled; rejecting storage deal proposal from client: %s", deal.Client.String())
return false, "miner is not accepting storage deals", nil
if deal.Ref != nil && deal.Ref.TransferType != storagemarket.TTManual && !b {
log.Warnf("online storage deal consideration disabled; rejecting storage deal proposal from client: %s", deal.Client.String())
return false, "miner is not considering online storage deals", nil
}
b, err = offlineOk()
if err != nil {
return false, "miner error", err
}
if deal.Ref != nil && deal.Ref.TransferType == storagemarket.TTManual && !b {
log.Warnf("offline storage deal consideration disabled; rejecting storage deal proposal from client: %s", deal.Client.String())
return false, "miner is not accepting offline storage deals", nil
}
blocklist, err := blocklistFunc()
@ -357,14 +367,40 @@ func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Con
}
// RetrievalProvider creates a new retrieval provider attached to the provider blockstore
func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.SectorManager, full lapi.FullNode, ds dtypes.MetadataDS, pieceStore dtypes.ProviderPieceStore, ibs dtypes.StagingBlockstore) (retrievalmarket.RetrievalProvider, error) {
func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.SectorManager, full lapi.FullNode, ds dtypes.MetadataDS, pieceStore dtypes.ProviderPieceStore, ibs dtypes.StagingBlockstore, onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc, offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) (retrievalmarket.RetrievalProvider, error) {
adapter := retrievaladapter.NewRetrievalProviderNode(miner, sealer, full)
address, err := minerAddrFromDS(ds)
maddr, err := minerAddrFromDS(ds)
if err != nil {
return nil, err
}
network := rmnet.NewFromLibp2pHost(h)
return retrievalimpl.NewProvider(address, adapter, network, pieceStore, ibs, namespace.Wrap(ds, datastore.NewKey("/retrievals/provider")))
netwk := rmnet.NewFromLibp2pHost(h)
opt := retrievalimpl.DealDeciderOpt(func(ctx context.Context, state retrievalmarket.ProviderDealState) (bool, string, error) {
b, err := onlineOk()
if err != nil {
return false, "miner error", err
}
if !b {
log.Warn("online retrieval deal consideration disabled; rejecting retrieval deal proposal from client")
return false, "miner is not accepting online retrieval deals", nil
}
b, err = offlineOk()
if err != nil {
return false, "miner error", err
}
if !b {
log.Info("offline retrieval has not been implemented yet")
}
return true, "", nil
})
return retrievalimpl.NewProvider(maddr, adapter, netwk, pieceStore, ibs, namespace.Wrap(ds, datastore.NewKey("/retrievals/provider")), opt)
}
func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls stores.LocalStorage, si stores.SectorIndex, cfg *ffiwrapper.Config, sc sectorstorage.SealerConfig, urls sectorstorage.URLs, sa sectorstorage.StorageAuth) (*sectorstorage.Manager, error) {
@ -399,19 +435,37 @@ func StorageAuth(ctx helpers.MetricsCtx, ca lapi.Common) (sectorstorage.StorageA
return sectorstorage.StorageAuth(headers), nil
}
func NewAcceptingStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.AcceptingStorageDealsConfigFunc, error) {
func NewConsiderOnlineStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOnlineStorageDealsConfigFunc, error) {
return func() (out bool, err error) {
err = readCfg(r, func(cfg *config.StorageMiner) {
out = cfg.Dealmaking.AcceptingStorageDeals
out = cfg.Dealmaking.ConsiderOnlineStorageDeals
})
return
}, nil
}
func NewSetAcceptingStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.SetAcceptingStorageDealsConfigFunc, error) {
func NewSetConsideringOnlineStorageDealsFunc(r repo.LockedRepo) (dtypes.SetConsiderOnlineStorageDealsConfigFunc, error) {
return func(b bool) (err error) {
err = mutateCfg(r, func(cfg *config.StorageMiner) {
cfg.Dealmaking.AcceptingStorageDeals = b
cfg.Dealmaking.ConsiderOnlineStorageDeals = b
})
return
}, nil
}
func NewConsiderOnlineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOnlineRetrievalDealsConfigFunc, error) {
return func() (out bool, err error) {
err = readCfg(r, func(cfg *config.StorageMiner) {
out = cfg.Dealmaking.ConsiderOnlineRetrievalDeals
})
return
}, nil
}
func NewSetConsiderOnlineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.SetConsiderOnlineRetrievalDealsConfigFunc, error) {
return func(b bool) (err error) {
err = mutateCfg(r, func(cfg *config.StorageMiner) {
cfg.Dealmaking.ConsiderOnlineRetrievalDeals = b
})
return
}, nil
@ -435,6 +489,42 @@ func NewSetStorageDealPieceCidBlocklistConfigFunc(r repo.LockedRepo) (dtypes.Set
}, nil
}
func NewConsiderOfflineStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOfflineStorageDealsConfigFunc, error) {
return func() (out bool, err error) {
err = readCfg(r, func(cfg *config.StorageMiner) {
out = cfg.Dealmaking.ConsiderOfflineStorageDeals
})
return
}, nil
}
func NewSetConsideringOfflineStorageDealsFunc(r repo.LockedRepo) (dtypes.SetConsiderOfflineStorageDealsConfigFunc, error) {
return func(b bool) (err error) {
err = mutateCfg(r, func(cfg *config.StorageMiner) {
cfg.Dealmaking.ConsiderOfflineStorageDeals = b
})
return
}, nil
}
func NewConsiderOfflineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOfflineRetrievalDealsConfigFunc, error) {
return func() (out bool, err error) {
err = readCfg(r, func(cfg *config.StorageMiner) {
out = cfg.Dealmaking.ConsiderOfflineRetrievalDeals
})
return
}, nil
}
func NewSetConsiderOfflineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.SetConsiderOfflineRetrievalDealsConfigFunc, error) {
return func(b bool) (err error) {
err = mutateCfg(r, func(cfg *config.StorageMiner) {
cfg.Dealmaking.ConsiderOfflineRetrievalDeals = b
})
return
}, nil
}
func readCfg(r repo.LockedRepo, accessor func(*config.StorageMiner)) error {
raw, err := r.Config()
if err != nil {

2
node/modules/testing/beacon.go
View File

@ -8,5 +8,5 @@ import (
)
func RandomBeacon() (beacon.RandomBeacon, error) {
return beacon.NewMockBeacon(build.BlockDelay * time.Second), nil
return beacon.NewMockBeacon(time.Duration(build.BlockDelaySecs) * time.Second), nil
}

2
node/node_test.go
View File

@ -349,7 +349,7 @@ func mockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test
templ := &genesis.Template{
Accounts: genaccs,
Miners: genms,
Timestamp: uint64(time.Now().Unix() - (build.BlockDelay * 20000)),
Timestamp: uint64(time.Now().Unix()) - (build.BlockDelaySecs * 20000),
}
// END PRESEAL SECTION

4
paychmgr/paych.go
View File

@ -76,7 +76,7 @@ func (pm *Manager) TrackInboundChannel(ctx context.Context, ch address.Address)
return err
}
from := account.Address
_, err = pm.sm.LoadActorState(ctx, st.From, &account, nil)
_, err = pm.sm.LoadActorState(ctx, st.To, &account, nil)
if err != nil {
return err
}
@ -114,7 +114,7 @@ func (pm *Manager) loadOutboundChannelInfo(ctx context.Context, ch address.Addre
return nil, err
}
from := account.Address
_, err = pm.sm.LoadActorState(ctx, st.From, &account, nil)
_, err = pm.sm.LoadActorState(ctx, st.To, &account, nil)
if err != nil {
return nil, err
}

19
storage/wdpost_run.go
View File

@ -383,17 +383,14 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo
return nil, xerrors.Errorf("get need prove sectors: %w", err)
}
var skipped *abi.BitField
{
good, err := s.checkSectors(ctx, nps)
if err != nil {
return nil, xerrors.Errorf("checking sectors to skip: %w", err)
}
good, err := s.checkSectors(ctx, nps)
if err != nil {
return nil, xerrors.Errorf("checking sectors to skip: %w", err)
}
skipped, err = bitfield.SubtractBitField(nps, good)
if err != nil {
return nil, xerrors.Errorf("nps - good: %w", err)
}
skipped, err := bitfield.SubtractBitField(nps, good)
if err != nil {
return nil, xerrors.Errorf("nps - good: %w", err)
}
skipCount, err := skipped.Count()
@ -401,7 +398,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo
return nil, xerrors.Errorf("getting skipped sector count: %w", err)
}
ssi, err := s.sortedSectorInfo(ctx, nps, ts)
ssi, err := s.sortedSectorInfo(ctx, good, ts)
if err != nil {
return nil, xerrors.Errorf("getting sorted sector info: %w", err)
}

2
tools/stats/rpc.go
View File

@ -114,7 +114,7 @@ sync_complete:
// If we get within 20 blocks of the current exected block height we
// consider sync complete. Block propagation is not always great but we still
// want to be recording stats as soon as we can
if timestampDelta < build.BlockDelay*20 {
if timestampDelta < int64(build.BlockDelaySecs)*20 {
return nil
}
}