cerc-io/lotus
15
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge remote-tracking branch 'origin/master' into feat/async-restartable-workers

Browse Source
This commit is contained in:
Łukasz Magiera 2020-09-30 20:48:16 +02:00
commit 2cfe22d4e5
94 changed files with 5219 additions and 3134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/CODEOWNERS vendored
View File

@ -13,3 +13,4 @@
### Conformance testing.
conformance/ @raulk
extern/test-vectors @raulk
cmd/tvx @raulk

4
README.md
View File

@ -1,5 +1,5 @@
<p align="center">
<a href="https://lotu.sh/" title="Lotus Docs">
<a href="https://docs.filecoin.io/" title="Filecoin Docs">
<img src="documentation/images/lotus_logo_h.png" alt="Project Lotus Logo" width="244" />
</a>
</p>
@ -18,7 +18,7 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more
## Building & Documentation
For instructions on how to build lotus from source, please visit [Lotus build and setup instruction](https://docs.filecoin.io/get-started/lotus/installation/#minimal-requirements) or read the source [here](https://github.com/filecoin-project/lotus/tree/master/documentation).
For instructions on how to build, install and setup lotus, please visit [https://docs.filecoin.io/get-started/lotus](https://docs.filecoin.io/get-started/lotus/).
## Reporting a Vulnerability

5
api/api_full.go
View File

@ -176,6 +176,9 @@ type FullNode interface {
// the reason.
SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error)
// SyncValidateTipset indicates whether the provided tipset is valid or not
SyncValidateTipset(ctx context.Context, tsk types.TipSetKey) (bool, error)
// MethodGroup: Mpool
// The Mpool methods are for interacting with the message pool. The message pool
// manages all incoming and outgoing 'messages' going over the network.
@ -244,6 +247,8 @@ type FullNode interface {
WalletImport(context.Context, *types.KeyInfo) (address.Address, error)
// WalletDelete deletes an address from the wallet.
WalletDelete(context.Context, address.Address) error
// WalletValidateAddress validates whether a given string can be decoded as a well-formed address
WalletValidateAddress(context.Context, string) (address.Address, error)
// Other

34
api/apistruct/struct.go
View File

@ -112,6 +112,7 @@ type FullNodeStruct struct {
SyncMarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"admin"`
SyncUnmarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"admin"`
SyncCheckBad func(ctx context.Context, bcid cid.Cid) (string, error) `perm:"read"`
SyncValidateTipset func(ctx context.Context, tsk types.TipSetKey) (bool, error) `perm:"read"`
MpoolGetConfig func(context.Context) (*types.MpoolConfig, error) `perm:"read"`
MpoolSetConfig func(context.Context, *types.MpoolConfig) error `perm:"write"`
@ -129,18 +130,19 @@ type FullNodeStruct struct {
MinerGetBaseInfo func(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*api.MiningBaseInfo, error) `perm:"read"`
MinerCreateBlock func(context.Context, *api.BlockTemplate) (*types.BlockMsg, error) `perm:"write"`
WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"`
WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"`
WalletList func(context.Context) ([]address.Address, error) `perm:"write"`
WalletBalance func(context.Context, address.Address) (types.BigInt, error) `perm:"read"`
WalletSign func(context.Context, address.Address, []byte) (*crypto.Signature, error) `perm:"sign"`
WalletSignMessage func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"`
WalletVerify func(context.Context, address.Address, []byte, *crypto.Signature) (bool, error) `perm:"read"`
WalletDefaultAddress func(context.Context) (address.Address, error) `perm:"write"`
WalletSetDefault func(context.Context, address.Address) error `perm:"admin"`
WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"`
WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"`
WalletDelete func(context.Context, address.Address) error `perm:"write"`
WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"`
WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"`
WalletList func(context.Context) ([]address.Address, error) `perm:"write"`
WalletBalance func(context.Context, address.Address) (types.BigInt, error) `perm:"read"`
WalletSign func(context.Context, address.Address, []byte) (*crypto.Signature, error) `perm:"sign"`
WalletSignMessage func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"`
WalletVerify func(context.Context, address.Address, []byte, *crypto.Signature) (bool, error) `perm:"read"`
WalletDefaultAddress func(context.Context) (address.Address, error) `perm:"write"`
WalletSetDefault func(context.Context, address.Address) error `perm:"admin"`
WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"`
WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"`
WalletDelete func(context.Context, address.Address) error `perm:"write"`
WalletValidateAddress func(context.Context, string) (address.Address, error) `perm:"read"`
ClientImport func(ctx context.Context, ref api.FileRef) (*api.ImportRes, error) `perm:"admin"`
ClientListImports func(ctx context.Context) ([]api.Import, error) `perm:"write"`
@ -642,6 +644,10 @@ func (c *FullNodeStruct) WalletDelete(ctx context.Context, addr address.Address)
return c.Internal.WalletDelete(ctx, addr)
}
func (c *FullNodeStruct) WalletValidateAddress(ctx context.Context, str string) (address.Address, error) {
return c.Internal.WalletValidateAddress(ctx, str)
}
func (c *FullNodeStruct) MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) {
return c.Internal.MpoolGetNonce(ctx, addr)
}
@ -746,6 +752,10 @@ func (c *FullNodeStruct) SyncCheckBad(ctx context.Context, bcid cid.Cid) (string
return c.Internal.SyncCheckBad(ctx, bcid)
}
func (c *FullNodeStruct) SyncValidateTipset(ctx context.Context, tsk types.TipSetKey) (bool, error) {
return c.Internal.SyncValidateTipset(ctx, tsk)
}
func (c *FullNodeStruct) StateNetworkName(ctx context.Context) (dtypes.NetworkName, error) {
return c.Internal.StateNetworkName(ctx)
}

21
api/test/window_post.go
View File

@ -153,18 +153,16 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
require.NoError(t, err)
fmt.Printf("Running one proving period\n")
fmt.Printf("End for head.Height > %d\n", di.PeriodStart+di.WPoStProvingPeriod+2)
for {
head, err := client.ChainHead(ctx)
require.NoError(t, err)
if head.Height() > di.PeriodStart+(di.WPoStProvingPeriod)+2 {
fmt.Printf("Now head.Height = %d\n", head.Height())
break
}
if head.Height()%100 == 0 {
fmt.Printf("@%d\n", head.Height())
}
build.Clock.Sleep(blocktime)
}
@ -186,7 +184,6 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
require.Greater(t, len(parts), 0)
secs := parts[0].AllSectors
require.NoError(t, err)
n, err := secs.Count()
require.NoError(t, err)
require.Equal(t, uint64(2), n)
@ -210,7 +207,6 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
require.Greater(t, len(parts), 0)
secs := parts[0].AllSectors
require.NoError(t, err)
n, err := secs.Count()
require.NoError(t, err)
require.Equal(t, uint64(2), n)
@ -236,18 +232,17 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
require.NoError(t, err)
fmt.Printf("Go through another PP, wait for sectors to become faulty\n")
fmt.Printf("End for head.Height > %d\n", di.PeriodStart+di.WPoStProvingPeriod+2)
for {
head, err := client.ChainHead(ctx)
require.NoError(t, err)
if head.Height() > di.PeriodStart+(di.WPoStProvingPeriod)+2 {
fmt.Printf("Now head.Height = %d\n", head.Height())
break
}
if head.Height()%100 == 0 {
fmt.Printf("@%d\n", head.Height())
}
build.Clock.Sleep(blocktime)
}
@ -267,17 +262,17 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
di, err = client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
require.NoError(t, err)
fmt.Printf("End for head.Height > %d\n", di.PeriodStart+di.WPoStProvingPeriod+2)
for {
head, err := client.ChainHead(ctx)
require.NoError(t, err)
if head.Height() > di.PeriodStart+di.WPoStProvingPeriod+2 {
fmt.Printf("Now head.Height = %d\n", head.Height())
break
}
if head.Height()%100 == 0 {
fmt.Printf("@%d\n", head.Height())
}
build.Clock.Sleep(blocktime)
}
@ -300,12 +295,14 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
require.NoError(t, err)
waitUntil := head.Height() + 10
fmt.Printf("End for head.Height > %d\n", waitUntil)
for {
head, err := client.ChainHead(ctx)
require.NoError(t, err)
if head.Height() > waitUntil {
fmt.Printf("Now head.Height = %d\n", head.Height())
break
}
}

4
chain/actors/builtin/miner/miner.go
View File

@ -20,6 +20,10 @@ import (
// Unchanged between v0 and v1 actors
var WPoStProvingPeriod = miner0.WPoStProvingPeriod
var WPoStPeriodDeadlines = miner0.WPoStPeriodDeadlines
var WPoStChallengeWindow = miner0.WPoStChallengeWindow
var WPoStChallengeLookback = miner0.WPoStChallengeLookback
var FaultDeclarationCutoff = miner0.FaultDeclarationCutoff
const MinSectorExpiration = miner0.MinSectorExpiration

94
chain/messagepool/messagepool.go
View File

@ -75,8 +75,6 @@ var (
ErrRBFTooLowPremium = errors.New("replace by fee has too low GasPremium")
ErrTooManyPendingMessages = errors.New("too many pending messages for actor")
ErrNonceGap = errors.New("unfulfilled nonce gap")
ErrTryAgain = errors.New("state inconsistency while pushing message; please try again")
)
const (
@ -795,98 +793,6 @@ func (mp *MessagePool) getStateBalance(addr address.Address, ts *types.TipSet) (
return act.Balance, nil
}
func (mp *MessagePool) PushWithNonce(ctx context.Context, addr address.Address, cb func(address.Address, uint64) (*types.SignedMessage, error)) (*types.SignedMessage, error) {
// serialize push access to reduce lock contention
mp.addSema <- struct{}{}
defer func() {
<-mp.addSema
}()
mp.curTsLk.Lock()
mp.lk.Lock()
curTs := mp.curTs
fromKey := addr
if fromKey.Protocol() == address.ID {
var err error
fromKey, err = mp.api.StateAccountKey(ctx, fromKey, mp.curTs)
if err != nil {
mp.lk.Unlock()
mp.curTsLk.Unlock()
return nil, xerrors.Errorf("resolving sender key: %w", err)
}
}
nonce, err := mp.getNonceLocked(fromKey, mp.curTs)
if err != nil {
mp.lk.Unlock()
mp.curTsLk.Unlock()
return nil, xerrors.Errorf("get nonce locked failed: %w", err)
}
// release the locks for signing
mp.lk.Unlock()
mp.curTsLk.Unlock()
msg, err := cb(fromKey, nonce)
if err != nil {
return nil, err
}
err = mp.checkMessage(msg)
if err != nil {
return nil, err
}
msgb, err := msg.Serialize()
if err != nil {
return nil, err
}
// reacquire the locks and check state for consistency
mp.curTsLk.Lock()
defer mp.curTsLk.Unlock()
if mp.curTs != curTs {
return nil, ErrTryAgain
}
mp.lk.Lock()
defer mp.lk.Unlock()
nonce2, err := mp.getNonceLocked(fromKey, mp.curTs)
if err != nil {
return nil, xerrors.Errorf("get nonce locked failed: %w", err)
}
if nonce2 != nonce {
return nil, ErrTryAgain
}
publish, err := mp.verifyMsgBeforeAdd(msg, curTs, true)
if err != nil {
return nil, err
}
if err := mp.checkBalance(msg, curTs); err != nil {
return nil, err
}
if err := mp.addLocked(msg, false); err != nil {
return nil, xerrors.Errorf("add locked failed: %w", err)
}
if err := mp.addLocal(msg, msgb); err != nil {
log.Errorf("addLocal failed: %+v", err)
}
if publish {
err = mp.api.PubSubPublish(build.MessagesTopic(mp.netName), msgb)
}
return msg, err
}
func (mp *MessagePool) Remove(from address.Address, nonce uint64, applied bool) {
mp.lk.Lock()
defer mp.lk.Unlock()

124
chain/messagesigner/messagesigner.go Normal file
View File

@ -0,0 +1,124 @@
package messagesigner
import (
"bytes"
"context"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/chain/messagepool"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
logging "github.com/ipfs/go-log/v2"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
)
const dsKeyActorNonce = "ActorNonce"
var log = logging.Logger("messagesigner")
type mpoolAPI interface {
GetNonce(address.Address) (uint64, error)
}
// MessageSigner keeps track of nonces per address, and increments the nonce
// when signing a message
type MessageSigner struct {
wallet *wallet.Wallet
mpool mpoolAPI
ds datastore.Batching
}
func NewMessageSigner(wallet *wallet.Wallet, mpool *messagepool.MessagePool, ds dtypes.MetadataDS) *MessageSigner {
return newMessageSigner(wallet, mpool, ds)
}
func newMessageSigner(wallet *wallet.Wallet, mpool mpoolAPI, ds dtypes.MetadataDS) *MessageSigner {
ds = namespace.Wrap(ds, datastore.NewKey("/message-signer/"))
return &MessageSigner{
wallet: wallet,
mpool: mpool,
ds: ds,
}
}
// SignMessage increments the nonce for the message From address, and signs
// the message
func (ms *MessageSigner) SignMessage(ctx context.Context, msg *types.Message) (*types.SignedMessage, error) {
nonce, err := ms.nextNonce(msg.From)
if err != nil {
return nil, xerrors.Errorf("failed to create nonce: %w", err)
}
msg.Nonce = nonce
sig, err := ms.wallet.Sign(ctx, msg.From, msg.Cid().Bytes())
if err != nil {
return nil, xerrors.Errorf("failed to sign message: %w", err)
}
return &types.SignedMessage{
Message: *msg,
Signature: *sig,
}, nil
}
// nextNonce increments the nonce.
// If there is no nonce in the datastore, gets the nonce from the message pool.
func (ms *MessageSigner) nextNonce(addr address.Address) (uint64, error) {
// Nonces used to be created by the mempool and we need to support nodes
// that have mempool nonces, so first check the mempool for a nonce for
// this address. Note that the mempool returns the actor state's nonce
// by default.
nonce, err := ms.mpool.GetNonce(addr)
if err != nil {
return 0, xerrors.Errorf("failed to get nonce from mempool: %w", err)
}
// Get the nonce for this address from the datastore
addrNonceKey := datastore.KeyWithNamespaces([]string{dsKeyActorNonce, addr.String()})
dsNonceBytes, err := ms.ds.Get(addrNonceKey)
switch {
case xerrors.Is(err, datastore.ErrNotFound):
// If a nonce for this address hasn't yet been created in the
// datastore, just use the nonce from the mempool
case err != nil:
return 0, xerrors.Errorf("failed to get nonce from datastore: %w", err)
default:
// There is a nonce in the datastore, so unmarshall and increment it
maj, val, err := cbg.CborReadHeader(bytes.NewReader(dsNonceBytes))
if err != nil {
return 0, xerrors.Errorf("failed to parse nonce from datastore: %w", err)
}
if maj != cbg.MajUnsignedInt {
return 0, xerrors.Errorf("bad cbor type parsing nonce from datastore")
}
dsNonce := val + 1
// The message pool nonce should be <= than the datastore nonce
if nonce <= dsNonce {
nonce = dsNonce
} else {
log.Warnf("mempool nonce was larger than datastore nonce (%d > %d)", nonce, dsNonce)
}
}
// Write the nonce for this address to the datastore
buf := bytes.Buffer{}
_, err = buf.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, nonce))
if err != nil {
return 0, xerrors.Errorf("failed to marshall nonce: %w", err)
}
err = ms.ds.Put(addrNonceKey, buf.Bytes())
if err != nil {
return 0, xerrors.Errorf("failed to write nonce to datastore: %w", err)
}
return nonce, nil
}

158
chain/messagesigner/messagesigner_test.go Normal file
View File

@ -0,0 +1,158 @@
package messagesigner
import (
"context"
"sync"
"testing"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/stretchr/testify/require"
ds_sync "github.com/ipfs/go-datastore/sync"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/chain/types"
"github.com/ipfs/go-datastore"
)
type mockMpool struct {
lk sync.RWMutex
nonces map[address.Address]uint64
}
func newMockMpool() *mockMpool {
return &mockMpool{nonces: make(map[address.Address]uint64)}
}
func (mp *mockMpool) setNonce(addr address.Address, nonce uint64) {
mp.lk.Lock()
defer mp.lk.Unlock()
mp.nonces[addr] = nonce
}
func (mp *mockMpool) GetNonce(addr address.Address) (uint64, error) {
mp.lk.RLock()
defer mp.lk.RUnlock()
return mp.nonces[addr], nil
}
func TestMessageSignerSignMessage(t *testing.T) {
ctx := context.Background()
w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
from1, err := w.GenerateKey(crypto.SigTypeSecp256k1)
require.NoError(t, err)
from2, err := w.GenerateKey(crypto.SigTypeSecp256k1)
require.NoError(t, err)
to1, err := w.GenerateKey(crypto.SigTypeSecp256k1)
require.NoError(t, err)
to2, err := w.GenerateKey(crypto.SigTypeSecp256k1)
require.NoError(t, err)
type msgSpec struct {
msg *types.Message
mpoolNonce [1]uint64
expNonce uint64
}
tests := []struct {
name string
msgs []msgSpec
}{{
// No nonce yet in datastore
name: "no nonce yet",
msgs: []msgSpec{{
msg: &types.Message{
To: to1,
From: from1,
},
expNonce: 0,
}},
}, {
// Get nonce value of zero from mpool
name: "mpool nonce zero",
msgs: []msgSpec{{
msg: &types.Message{
To: to1,
From: from1,
},
mpoolNonce: [1]uint64{0},
expNonce: 0,
}},
}, {
// Get non-zero nonce value from mpool
name: "mpool nonce set",
msgs: []msgSpec{{
msg: &types.Message{
To: to1,
From: from1,
},
mpoolNonce: [1]uint64{5},
expNonce: 5,
}, {
msg: &types.Message{
To: to1,
From: from1,
},
// Should adjust datastore nonce because mpool nonce is higher
mpoolNonce: [1]uint64{10},
expNonce: 10,
}},
}, {
// Nonce should increment independently for each address
name: "nonce increments per address",
msgs: []msgSpec{{
msg: &types.Message{
To: to1,
From: from1,
},
expNonce: 0,
}, {
msg: &types.Message{
To: to1,
From: from1,
},
expNonce: 1,
}, {
msg: &types.Message{
To: to2,
From: from2,
},
mpoolNonce: [1]uint64{5},
expNonce: 5,
}, {
msg: &types.Message{
To: to2,
From: from2,
},
expNonce: 6,
}, {
msg: &types.Message{
To: to1,
From: from1,
},
expNonce: 2,
}},
}}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
mpool := newMockMpool()
ds := ds_sync.MutexWrap(datastore.NewMapDatastore())
ms := newMessageSigner(w, mpool, ds)
for _, m := range tt.msgs {
if len(m.mpoolNonce) == 1 {
mpool.setNonce(m.msg.From, m.mpoolNonce[0])
}
smsg, err := ms.SignMessage(ctx, m.msg)
require.NoError(t, err)
require.Equal(t, m.expNonce, smsg.Message.Nonce)
}
})
}
}

37
chain/store/store.go
View File

@ -10,6 +10,8 @@ import (
"strconv"
"sync"
"golang.org/x/sync/errgroup"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/minio/blake2b-simd"
@ -284,6 +286,16 @@ func (cs *ChainStore) MarkBlockAsValidated(ctx context.Context, blkid cid.Cid) e
return nil
}
func (cs *ChainStore) UnmarkBlockAsValidated(ctx context.Context, blkid cid.Cid) error {
key := blockValidationCacheKeyPrefix.Instance(blkid.String())
if err := cs.ds.Delete(key); err != nil {
return xerrors.Errorf("removing from valid block cache: %w", err)
}
return nil
}
func (cs *ChainStore) SetGenesis(b *types.BlockHeader) error {
ts, err := types.NewTipSet([]*types.BlockHeader{b})
if err != nil {
@ -467,14 +479,25 @@ func (cs *ChainStore) LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error) {
return v.(*types.TipSet), nil
}
var blks []*types.BlockHeader
for _, c := range tsk.Cids() {
b, err := cs.GetBlock(c)
if err != nil {
return nil, xerrors.Errorf("get block %s: %w", c, err)
}
// Fetch tipset block headers from blockstore in parallel
var eg errgroup.Group
cids := tsk.Cids()
blks := make([]*types.BlockHeader, len(cids))
for i, c := range cids {
i, c := i, c
eg.Go(func() error {
b, err := cs.GetBlock(c)
if err != nil {
return xerrors.Errorf("get block %s: %w", c, err)
}
blks = append(blks, b)
blks[i] = b
return nil
})
}
err := eg.Wait()
if err != nil {
return nil, err
}
ts, err := types.NewTipSet(blks)

73
chain/sync.go
View File

@ -597,7 +597,7 @@ func isPermanent(err error) bool {
return !errors.Is(err, ErrTemporal)
}
func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet) error {
func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet, useCache bool) error {
ctx, span := trace.StartSpan(ctx, "validateTipSet")
defer span.End()
@ -613,7 +613,7 @@ func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet)
b := b // rebind to a scoped variable
futures = append(futures, async.Err(func() error {
if err := syncer.ValidateBlock(ctx, b); err != nil {
if err := syncer.ValidateBlock(ctx, b, useCache); err != nil {
if isPermanent(err) {
syncer.bad.Add(b.Cid(), NewBadBlockReason([]cid.Cid{b.Cid()}, err.Error()))
}
@ -680,7 +680,7 @@ func blockSanityChecks(h *types.BlockHeader) error {
}
// ValidateBlock should match up with 'Semantical Validation' in validation.md in the spec
func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (err error) {
func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock, useCache bool) (err error) {
defer func() {
// b.Cid() could panic for empty blocks that are used in tests.
if rerr := recover(); rerr != nil {
@ -689,13 +689,15 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
}
}()
isValidated, err := syncer.store.IsBlockValidated(ctx, b.Cid())
if err != nil {
return xerrors.Errorf("check block validation cache %s: %w", b.Cid(), err)
}
if useCache {
isValidated, err := syncer.store.IsBlockValidated(ctx, b.Cid())
if err != nil {
return xerrors.Errorf("check block validation cache %s: %w", b.Cid(), err)
}
if isValidated {
return nil
if isValidated {
return nil
}
}
validationStart := build.Clock.Now()
@ -782,31 +784,35 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
b.Header.ParentWeight, pweight)
}
// Stuff that needs stateroot / worker address
stateroot, precp, err := syncer.sm.TipSetState(ctx, baseTs)
if err != nil {
return xerrors.Errorf("get tipsetstate(%d, %s) failed: %w", h.Height, h.Parents, err)
}
if stateroot != h.ParentStateRoot {
msgs, err := syncer.store.MessagesForTipset(baseTs)
stateRootCheck := async.Err(func() error {
stateroot, precp, err := syncer.sm.TipSetState(ctx, baseTs)
if err != nil {
log.Error("failed to load messages for tipset during tipset state mismatch error: ", err)
} else {
log.Warn("Messages for tipset with mismatching state:")
for i, m := range msgs {
mm := m.VMMessage()
log.Warnf("Message[%d]: from=%s to=%s method=%d params=%x", i, mm.From, mm.To, mm.Method, mm.Params)
}
return xerrors.Errorf("get tipsetstate(%d, %s) failed: %w", h.Height, h.Parents, err)
}
return xerrors.Errorf("parent state root did not match computed state (%s != %s)", stateroot, h.ParentStateRoot)
}
if stateroot != h.ParentStateRoot {
msgs, err := syncer.store.MessagesForTipset(baseTs)
if err != nil {
log.Error("failed to load messages for tipset during tipset state mismatch error: ", err)
} else {
log.Warn("Messages for tipset with mismatching state:")
for i, m := range msgs {
mm := m.VMMessage()
log.Warnf("Message[%d]: from=%s to=%s method=%d params=%x", i, mm.From, mm.To, mm.Method, mm.Params)
}
}
if precp != h.ParentMessageReceipts {
return xerrors.Errorf("parent receipts root did not match computed value (%s != %s)", precp, h.ParentMessageReceipts)
}
return xerrors.Errorf("parent state root did not match computed state (%s != %s)", stateroot, h.ParentStateRoot)
}
if precp != h.ParentMessageReceipts {
return xerrors.Errorf("parent receipts root did not match computed value (%s != %s)", precp, h.ParentMessageReceipts)
}
return nil
})
// Stuff that needs worker address
waddr, err := stmgr.GetMinerWorkerRaw(ctx, syncer.sm, lbst, h.Miner)
if err != nil {
return xerrors.Errorf("GetMinerWorkerRaw failed: %w", err)
@ -927,6 +933,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
winnerCheck,
msgsCheck,
baseFeeCheck,
stateRootCheck,
}
var merr error
@ -954,8 +961,10 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
return mulErr
}
if err := syncer.store.MarkBlockAsValidated(ctx, b.Cid()); err != nil {
return xerrors.Errorf("caching block validation %s: %w", b.Cid(), err)
if useCache {
if err := syncer.store.MarkBlockAsValidated(ctx, b.Cid()); err != nil {
return xerrors.Errorf("caching block validation %s: %w", b.Cid(), err)
}
}
return nil
@ -1457,7 +1466,7 @@ func (syncer *Syncer) syncMessagesAndCheckState(ctx context.Context, headers []*
return syncer.iterFullTipsets(ctx, headers, func(ctx context.Context, fts *store.FullTipSet) error {
log.Debugw("validating tipset", "height", fts.TipSet().Height(), "size", len(fts.TipSet().Cids()))
if err := syncer.ValidateTipSet(ctx, fts); err != nil {
if err := syncer.ValidateTipSet(ctx, fts, true); err != nil {
log.Errorf("failed to validate tipset: %+v", err)
return xerrors.Errorf("message processing failed: %w", err)
}

4
chain/sync_test.go
View File

@ -732,7 +732,7 @@ func TestSyncInputs(t *testing.T) {
err := s.ValidateBlock(context.TODO(), &types.FullBlock{
Header: &types.BlockHeader{},
})
}, false)
if err == nil {
t.Fatal("should error on empty block")
}
@ -741,7 +741,7 @@ func TestSyncInputs(t *testing.T) {
h.ElectionProof = nil
err = s.ValidateBlock(context.TODO(), &types.FullBlock{Header: h})
err = s.ValidateBlock(context.TODO(), &types.FullBlock{Header: h}, false)
if err == nil {
t.Fatal("should error on block with nil election proof")
}

10
chain/types/tipset.go
View File

@ -167,12 +167,16 @@ func (ts *TipSet) Equals(ots *TipSet) bool {
return false
}
if len(ts.blks) != len(ots.blks) {
if ts.height != ots.height {
return false
}
for i, b := range ts.blks {
if b.Cid() != ots.blks[i].Cid() {
if len(ts.cids) != len(ots.cids) {
return false
}
for i, cid := range ts.cids {
if cid != ots.cids[i] {
return false
}
}

58
chain/vm/runtime.go
View File

@ -5,7 +5,6 @@ import (
"context"
"encoding/binary"
"fmt"
gruntime "runtime"
"time"
"github.com/filecoin-project/go-address"
@ -460,8 +459,10 @@ func (rt *Runtime) stateCommit(oldh, newh cid.Cid) aerrors.ActorError {
}
func (rt *Runtime) finilizeGasTracing() {
if rt.lastGasCharge != nil {
rt.lastGasCharge.TimeTaken = time.Since(rt.lastGasChargeTime)
if enableTracing {
if rt.lastGasCharge != nil {
rt.lastGasCharge.TimeTaken = time.Since(rt.lastGasChargeTime)
}
}
}
@ -490,34 +491,39 @@ func (rt *Runtime) chargeGasFunc(skip int) func(GasCharge) {
}
var enableTracing = false
func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError {
toUse := gas.Total()
var callers [10]uintptr
cout := gruntime.Callers(2+skip, callers[:])
if enableTracing {
var callers [10]uintptr
now := build.Clock.Now()
if rt.lastGasCharge != nil {
rt.lastGasCharge.TimeTaken = now.Sub(rt.lastGasChargeTime)
cout := 0 //gruntime.Callers(2+skip, callers[:])
now := build.Clock.Now()
if rt.lastGasCharge != nil {
rt.lastGasCharge.TimeTaken = now.Sub(rt.lastGasChargeTime)
}
gasTrace := types.GasTrace{
Name: gas.Name,
Extra: gas.Extra,
TotalGas: toUse,
ComputeGas: gas.ComputeGas,
StorageGas: gas.StorageGas,
TotalVirtualGas: gas.VirtualCompute*GasComputeMulti + gas.VirtualStorage*GasStorageMulti,
VirtualComputeGas: gas.VirtualCompute,
VirtualStorageGas: gas.VirtualStorage,
Callers: callers[:cout],
}
rt.executionTrace.GasCharges = append(rt.executionTrace.GasCharges, &gasTrace)
rt.lastGasChargeTime = now
rt.lastGasCharge = &gasTrace
}
gasTrace := types.GasTrace{
Name: gas.Name,
Extra: gas.Extra,
TotalGas: toUse,
ComputeGas: gas.ComputeGas,
StorageGas: gas.StorageGas,
TotalVirtualGas: gas.VirtualCompute*GasComputeMulti + gas.VirtualStorage*GasStorageMulti,
VirtualComputeGas: gas.VirtualCompute,
VirtualStorageGas: gas.VirtualStorage,
Callers: callers[:cout],
}
rt.executionTrace.GasCharges = append(rt.executionTrace.GasCharges, &gasTrace)
rt.lastGasChargeTime = now
rt.lastGasCharge = &gasTrace
// overflow safe
if rt.gasUsed > rt.gasAvailable-toUse {
rt.gasUsed = rt.gasAvailable

17
chain/vm/vm.go
View File

@ -227,14 +227,21 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime,
}
rt := vm.makeRuntime(ctx, msg, origin, on, gasUsed, nac)
rt.lastGasChargeTime = start
if enableTracing {
rt.lastGasChargeTime = start
if parent != nil {
rt.lastGasChargeTime = parent.lastGasChargeTime
rt.lastGasCharge = parent.lastGasCharge
defer func() {
parent.lastGasChargeTime = rt.lastGasChargeTime
parent.lastGasCharge = rt.lastGasCharge
}()
}
}
if parent != nil {
rt.lastGasChargeTime = parent.lastGasChargeTime
rt.lastGasCharge = parent.lastGasCharge
defer func() {
parent.gasUsed = rt.gasUsed
parent.lastGasChargeTime = rt.lastGasChargeTime
parent.lastGasCharge = rt.lastGasCharge
}()
}
if gasCharge != nil {

65
cli/wallet.go
View File

@ -9,13 +9,16 @@ import (
"os"
"strings"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/crypto"
types "github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/crypto"
types "github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/lib/tablewriter"
)
var walletCmd = &cli.Command{
@ -66,6 +69,13 @@ var walletNew = &cli.Command{
var walletList = &cli.Command{
Name: "list",
Usage: "List wallet address",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "addr-only",
Usage: "Only print addresses",
Aliases: []string{"a"},
},
},
Action: func(cctx *cli.Context) error {
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
@ -79,9 +89,52 @@ var walletList = &cli.Command{
return err
}
// Assume an error means no default key is set
def, _ := api.WalletDefaultAddress(ctx)
tw := tablewriter.New(
tablewriter.Col("Address"),
tablewriter.Col("Balance"),
tablewriter.Col("Nonce"),
tablewriter.Col("Default"),
tablewriter.NewLineCol("Error"))
for _, addr := range addrs {
fmt.Println(addr.String())
if cctx.Bool("addr-only") {
fmt.Println(addr.String())
} else {
a, err := api.StateGetActor(ctx, addr, types.EmptyTSK)
if err != nil {
if !strings.Contains(err.Error(), "actor not found") {
tw.Write(map[string]interface{}{
"Address": addr,
"Error": err,
})
continue
}
a = &types.Actor{
Balance: big.Zero(),
}
}
row := map[string]interface{}{
"Address": addr,
"Balance": types.FIL(a.Balance),
"Nonce": a.Nonce,
}
if addr == def {
row["Default"] = "X"
}
tw.Write(row)
}
}
if !cctx.Bool("addr-only") {
return tw.Flush(os.Stdout)
}
return nil
},
}

98
cmd/lotus-bench/caching_verifier.go Normal file
View File

@ -0,0 +1,98 @@
package main
import (
"bufio"
"context"
"errors"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
"github.com/filecoin-project/specs-actors/actors/runtime/proof"
"github.com/ipfs/go-datastore"
"github.com/minio/blake2b-simd"
cbg "github.com/whyrusleeping/cbor-gen"
)
type cachingVerifier struct {
ds datastore.Datastore
backend ffiwrapper.Verifier
}
const bufsize = 128
func (cv cachingVerifier) withCache(execute func() (bool, error), param cbg.CBORMarshaler) (bool, error) {
hasher := blake2b.New256()
wr := bufio.NewWriterSize(hasher, bufsize)
err := param.MarshalCBOR(wr)
if err != nil {
log.Errorf("could not marshal call info: %+v", err)
return execute()
}
err = wr.Flush()
if err != nil {
log.Errorf("could not flush: %+v", err)
return execute()
}
hash := hasher.Sum(nil)
key := datastore.NewKey(string(hash))
fromDs, err := cv.ds.Get(key)
if err == nil {
switch fromDs[0] {
case 's':
return true, nil
case 'f':
return false, nil
case 'e':
return false, errors.New(string(fromDs[1:]))
default:
log.Errorf("bad cached result in cache %s(%x)", fromDs[0], fromDs[0])
return execute()
}
} else if errors.Is(err, datastore.ErrNotFound) {
// recalc
ok, err := execute()
var save []byte
if err != nil {
if ok {
log.Errorf("success with an error: %+v", err)
} else {
save = append([]byte{'e'}, []byte(err.Error())...)
}
} else if ok {
save = []byte{'s'}
} else {
save = []byte{'f'}
}
if len(save) != 0 {
errSave := cv.ds.Put(key, save)
if errSave != nil {
log.Errorf("error saving result: %+v", errSave)
}
}
return ok, err
} else {
log.Errorf("could not get data from cache: %+v", err)
return execute()
}
}
func (cv *cachingVerifier) VerifySeal(svi proof.SealVerifyInfo) (bool, error) {
return cv.withCache(func() (bool, error) {
return cv.backend.VerifySeal(svi)
}, &svi)
}
func (cv *cachingVerifier) VerifyWinningPoSt(ctx context.Context, info proof.WinningPoStVerifyInfo) (bool, error) {
return cv.backend.VerifyWinningPoSt(ctx, info)
}
func (cv *cachingVerifier) VerifyWindowPoSt(ctx context.Context, info proof.WindowPoStVerifyInfo) (bool, error) {
return cv.withCache(func() (bool, error) {
return cv.backend.VerifyWindowPoSt(ctx, info)
}, &info)
}
func (cv *cachingVerifier) GenerateWinningPoStSectorChallenge(ctx context.Context, proofType abi.RegisteredPoStProof, a abi.ActorID, rnd abi.PoStRandomness, u uint64) ([]uint64, error) {
return cv.backend.GenerateWinningPoStSectorChallenge(ctx, proofType, a, rnd, u)
}
var _ ffiwrapper.Verifier = (*cachingVerifier)(nil)

208
cmd/lotus-bench/import.go
View File

@ -16,6 +16,8 @@ import (
"sort"
"time"
"github.com/cockroachdb/pebble"
"github.com/cockroachdb/pebble/bloom"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
@ -24,12 +26,16 @@ import (
"github.com/filecoin-project/lotus/lib/blockstore"
_ "github.com/filecoin-project/lotus/lib/sigs/bls"
_ "github.com/filecoin-project/lotus/lib/sigs/secp"
"github.com/ipld/go-car"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
bdg "github.com/dgraph-io/badger/v2"
"github.com/ipfs/go-datastore"
badger "github.com/ipfs/go-ds-badger2"
pebbleds "github.com/ipfs/go-ds-pebble"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
)
@ -60,6 +66,29 @@ var importBenchCmd = &cli.Command{
Name: "repodir",
Usage: "set the repo directory for the lotus bench run (defaults to /tmp)",
},
&cli.StringFlag{
Name: "syscall-cache",
Usage: "read and write syscall results from datastore",
},
&cli.BoolFlag{
Name: "export-traces",
Usage: "should we export execution traces",
Value: true,
},
&cli.BoolFlag{
Name: "no-import",
Usage: "should we import the chain? if set to true chain has to be previously imported",
},
&cli.BoolFlag{
Name: "global-profile",
Value: true,
},
&cli.Int64Flag{
Name: "start-at",
},
&cli.BoolFlag{
Name: "only-import",
},
},
Action: func(cctx *cli.Context) error {
vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads")
@ -74,6 +103,10 @@ var importBenchCmd = &cli.Command{
}
defer cfi.Close() //nolint:errcheck // read only file
go func() {
http.ListenAndServe("localhost:6060", nil) //nolint:errcheck
}()
var tdir string
if rdir := cctx.String("repodir"); rdir != "" {
tdir = rdir
@ -85,33 +118,105 @@ var importBenchCmd = &cli.Command{
tdir = tmp
}
bds, err := badger.NewDatastore(tdir, nil)
bdgOpt := badger.DefaultOptions
bdgOpt.GcInterval = 0
bdgOpt.Options = bdg.DefaultOptions("")
bdgOpt.Options.SyncWrites = false
bdgOpt.Options.Truncate = true
bdgOpt.Options.DetectConflicts = false
var bds datastore.Batching
if false {
cache := 512
bds, err = pebbleds.NewDatastore(tdir, &pebble.Options{
// Pebble has a single combined cache area and the write
// buffers are taken from this too. Assign all available
// memory allowance for cache.
Cache: pebble.NewCache(int64(cache * 1024 * 1024)),
// The size of memory table(as well as the write buffer).
// Note, there may have more than two memory tables in the system.
// MemTableStopWritesThreshold can be configured to avoid the memory abuse.
MemTableSize: cache * 1024 * 1024 / 4,
// The default compaction concurrency(1 thread),
// Here use all available CPUs for faster compaction.
MaxConcurrentCompactions: runtime.NumCPU(),
// Per-level options. Options for at least one level must be specified. The
// options for the last level are used for all subsequent levels.
Levels: []pebble.LevelOptions{
{TargetFileSize: 16 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10), Compression: pebble.NoCompression},
},
Logger: log,
})
} else {
bds, err = badger.NewDatastore(tdir, &bdgOpt)
}
if err != nil {
return err
}
defer bds.Close() //nolint:errcheck
bs := blockstore.NewBlockstore(bds)
cbs, err := blockstore.CachedBlockstore(context.TODO(), bs, blockstore.DefaultCacheOpts())
cacheOpts := blockstore.DefaultCacheOpts()
cacheOpts.HasBloomFilterSize = 0
cbs, err := blockstore.CachedBlockstore(context.TODO(), bs, cacheOpts)
if err != nil {
return err
}
bs = cbs
ds := datastore.NewMapDatastore()
cs := store.NewChainStore(bs, ds, vm.Syscalls(ffiwrapper.ProofVerifier))
var verifier ffiwrapper.Verifier = ffiwrapper.ProofVerifier
if cctx.IsSet("syscall-cache") {
scds, err := badger.NewDatastore(cctx.String("syscall-cache"), &bdgOpt)
if err != nil {
return xerrors.Errorf("opening syscall-cache datastore: %w", err)
}
defer scds.Close() //nolint:errcheck
verifier = &cachingVerifier{
ds: scds,
backend: verifier,
}
}
if cctx.Bool("only-gc") {
return nil
}
cs := store.NewChainStore(bs, ds, vm.Syscalls(verifier))
stm := stmgr.NewStateManager(cs)
prof, err := os.Create("import-bench.prof")
if err != nil {
return err
}
defer prof.Close() //nolint:errcheck
if cctx.Bool("global-profile") {
prof, err := os.Create("import-bench.prof")
if err != nil {
return err
}
defer prof.Close() //nolint:errcheck
if err := pprof.StartCPUProfile(prof); err != nil {
return err
if err := pprof.StartCPUProfile(prof); err != nil {
return err
}
}
head, err := cs.Import(cfi)
if err != nil {
return err
var head *types.TipSet
if !cctx.Bool("no-import") {
head, err = cs.Import(cfi)
if err != nil {
return err
}
} else {
cr, err := car.NewCarReader(cfi)
if err != nil {
return err
}
head, err = cs.LoadTipSet(types.NewTipSetKey(cr.Header.Roots...))
if err != nil {
return err
}
}
if cctx.Bool("only-import") {
return nil
}
gb, err := cs.GetTipsetByHeight(context.TODO(), 0, head, true)
@ -124,6 +229,20 @@ var importBenchCmd = &cli.Command{
return err
}
startEpoch := abi.ChainEpoch(1)
if cctx.IsSet("start-at") {
startEpoch = abi.ChainEpoch(cctx.Int64("start-at"))
start, err := cs.GetTipsetByHeight(context.TODO(), abi.ChainEpoch(cctx.Int64("start-at")), head, true)
if err != nil {
return err
}
err = cs.SetHead(start)
if err != nil {
return err
}
}
if h := cctx.Int64("height"); h != 0 {
tsh, err := cs.GetTipsetByHeight(context.TODO(), abi.ChainEpoch(h), head, true)
if err != nil {
@ -134,7 +253,7 @@ var importBenchCmd = &cli.Command{
ts := head
tschain := []*types.TipSet{ts}
for ts.Height() != 0 {
for ts.Height() > startEpoch {
next, err := cs.LoadTipSet(ts.Parents())
if err != nil {
return err
@ -144,45 +263,48 @@ var importBenchCmd = &cli.Command{
ts = next
}
ibj, err := os.Create("import-bench.json")
if err != nil {
return err
var enc *json.Encoder
if cctx.Bool("export-traces") {
ibj, err := os.Create("import-bench.json")
if err != nil {
return err
}
defer ibj.Close() //nolint:errcheck
enc = json.NewEncoder(ibj)
}
defer ibj.Close() //nolint:errcheck
enc := json.NewEncoder(ibj)
var lastTse *TipSetExec
lastState := tschain[len(tschain)-1].ParentState()
for i := len(tschain) - 2; i >= 0; i-- {
for i := len(tschain) - 1; i >= 1; i-- {
cur := tschain[i]
start := time.Now()
log.Infof("computing state (height: %d, ts=%s)", cur.Height(), cur.Cids())
if cur.ParentState() != lastState {
lastTrace := lastTse.Trace
st, trace, err := stm.ExecutionTrace(context.TODO(), cur)
if err != nil {
return err
}
tse := &TipSetExec{
TipSet: cur.Key(),
Trace: trace,
Duration: time.Since(start),
}
if enc != nil {
stripCallers(tse.Trace)
if err := enc.Encode(tse); err != nil {
return xerrors.Errorf("failed to write out tipsetexec: %w", err)
}
}
if tschain[i-1].ParentState() != st {
stripCallers(tse.Trace)
lastTrace := tse.Trace
d, err := json.MarshalIndent(lastTrace, "", " ")
if err != nil {
panic(err)
}
fmt.Println("TRACE")
fmt.Println(string(d))
return xerrors.Errorf("tipset chain had state mismatch at height %d (%s != %s)", cur.Height(), cur.ParentState(), lastState)
}
start := time.Now()
st, trace, err := stm.ExecutionTrace(context.TODO(), cur)
if err != nil {
return err
}
stripCallers(trace)
lastTse = &TipSetExec{
TipSet: cur.Key(),
Trace: trace,
Duration: time.Since(start),
}
lastState = st
if err := enc.Encode(lastTse); err != nil {
return xerrors.Errorf("failed to write out tipsetexec: %w", err)
//fmt.Println(statediff.Diff(context.Background(), bs, tschain[i-1].ParentState(), st, statediff.ExpandActors))
return xerrors.Errorf("tipset chain had state mismatch at height %d (%s != %s)", cur.Height(), cur.ParentState(), st)
}
}

6
cmd/lotus-seal-worker/main.go
View File

@ -111,6 +111,11 @@ var runCmd = &cli.Command{
Name: "no-local-storage",
Usage: "don't use storageminer repo for sector storage",
},
&cli.BoolFlag{
Name: "no-swap",
Usage: "don't use swap",
Value: false,
},
&cli.BoolFlag{
Name: "addpiece",
Usage: "enable addpiece",
@ -355,6 +360,7 @@ var runCmd = &cli.Command{
LocalWorker: sectorstorage.NewLocalWorker(sectorstorage.WorkerConfig{
SealProof: spt,
TaskTypes: taskTypes,
NoSwap: cctx.Bool("no-swap"),
}, remote, localStore, nodeApi, nodeApi, wsts),
localStore: localStore,
ls: lr,

268
cmd/lotus-shed/dealtracker.go
View File

@ -5,10 +5,10 @@ import (
"encoding/json"
"net"
"net/http"
"os"
"strings"
"sync"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/api"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/ipfs/go-cid"
@ -19,58 +19,54 @@ type dealStatsServer struct {
api api.FullNode
}
var filteredClients map[address.Address]bool
// Requested by @jbenet
// How many epochs back to look at for dealstats
var epochLookback = abi.ChainEpoch(10)
// these lists grow continuously with the network
// TODO: need to switch this to an LRU of sorts, to ensure refreshes
var knownFiltered = new(sync.Map)
var resolvedWallets = new(sync.Map)
func init() {
fc := []string{"t0112", "t0113", "t0114", "t010089"}
filtered, set := os.LookupEnv("FILTERED_CLIENTS")
if set {
fc = strings.Split(filtered, ":")
}
filteredClients = make(map[address.Address]bool)
for _, a := range fc {
addr, err := address.NewFromString(a)
for _, a := range []string{
"t0100", // client for genesis miner
"t0101", // client for genesis miner
"t0102", // client for genesis miner
"t0112", // client for genesis miner
"t0113", // client for genesis miner
"t0114", // client for genesis miner
"t1nslxql4pck5pq7hddlzym3orxlx35wkepzjkm3i", // SR1 dealbot wallet
"t1stghxhdp2w53dym2nz2jtbpk6ccd4l2lxgmezlq", // SR1 dealbot wallet
"t1mcr5xkgv4jdl3rnz77outn6xbmygb55vdejgbfi", // SR1 dealbot wallet
"t1qiqdbbmrdalbntnuapriirduvxu5ltsc5mhy7si", // SR1 dealbot wallet
} {
a, err := address.NewFromString(a)
if err != nil {
panic(err)
}
filteredClients[addr] = true
knownFiltered.Store(a, true)
}
}
type dealCountResp struct {
Total int64 `json:"total"`
Epoch int64 `json:"epoch"`
Epoch int64 `json:"epoch"`
Endpoint string `json:"endpoint"`
Payload int64 `json:"payload"`
}
func (dss *dealStatsServer) handleStorageDealCount(w http.ResponseWriter, r *http.Request) {
ctx := context.Background()
head, err := dss.api.ChainHead(ctx)
if err != nil {
log.Warnf("failed to get chain head: %s", err)
epoch, deals := dss.filteredDealList()
if epoch == 0 {
w.WriteHeader(500)
return
}
deals, err := dss.api.StateMarketDeals(ctx, head.Key())
if err != nil {
log.Warnf("failed to get market deals: %s", err)
w.WriteHeader(500)
return
}
var count int64
for _, d := range deals {
if !filteredClients[d.Proposal.Client] {
count++
}
}
if err := json.NewEncoder(w).Encode(&dealCountResp{
Total: count,
Epoch: int64(head.Height()),
Endpoint: "COUNT_DEALS",
Payload: int64(len(deals)),
Epoch: epoch,
}); err != nil {
log.Warnf("failed to write back deal count response: %s", err)
return
@ -78,39 +74,28 @@ func (dss *dealStatsServer) handleStorageDealCount(w http.ResponseWriter, r *htt
}
type dealAverageResp struct {
AverageSize int64 `json:"average_size"`
Epoch int64 `json:"epoch"`
Epoch int64 `json:"epoch"`
Endpoint string `json:"endpoint"`
Payload int64 `json:"payload"`
}
func (dss *dealStatsServer) handleStorageDealAverageSize(w http.ResponseWriter, r *http.Request) {
ctx := context.Background()
head, err := dss.api.ChainHead(ctx)
if err != nil {
log.Warnf("failed to get chain head: %s", err)
epoch, deals := dss.filteredDealList()
if epoch == 0 {
w.WriteHeader(500)
return
}
deals, err := dss.api.StateMarketDeals(ctx, head.Key())
if err != nil {
log.Warnf("failed to get market deals: %s", err)
w.WriteHeader(500)
return
}
var count int64
var totalBytes int64
for _, d := range deals {
if !filteredClients[d.Proposal.Client] {
count++
totalBytes += int64(d.Proposal.PieceSize.Unpadded())
}
totalBytes += int64(d.deal.Proposal.PieceSize.Unpadded())
}
if err := json.NewEncoder(w).Encode(&dealAverageResp{
AverageSize: totalBytes / count,
Epoch: int64(head.Height()),
Endpoint: "AVERAGE_DEAL_SIZE",
Payload: totalBytes / int64(len(deals)),
Epoch: epoch,
}); err != nil {
log.Warnf("failed to write back deal average response: %s", err)
return
@ -118,37 +103,27 @@ func (dss *dealStatsServer) handleStorageDealAverageSize(w http.ResponseWriter,
}
type dealTotalResp struct {
TotalBytes int64 `json:"total_size"`
Epoch int64 `json:"epoch"`
Epoch int64 `json:"epoch"`
Endpoint string `json:"endpoint"`
Payload int64 `json:"payload"`
}
func (dss *dealStatsServer) handleStorageDealTotalReal(w http.ResponseWriter, r *http.Request) {
ctx := context.Background()
head, err := dss.api.ChainHead(ctx)
if err != nil {
log.Warnf("failed to get chain head: %s", err)
w.WriteHeader(500)
return
}
deals, err := dss.api.StateMarketDeals(ctx, head.Key())
if err != nil {
log.Warnf("failed to get market deals: %s", err)
epoch, deals := dss.filteredDealList()
if epoch == 0 {
w.WriteHeader(500)
return
}
var totalBytes int64
for _, d := range deals {
if !filteredClients[d.Proposal.Client] {
totalBytes += int64(d.Proposal.PieceSize.Unpadded())
}
totalBytes += int64(d.deal.Proposal.PieceSize.Unpadded())
}
if err := json.NewEncoder(w).Encode(&dealTotalResp{
TotalBytes: totalBytes,
Epoch: int64(head.Height()),
Endpoint: "DEAL_BYTES",
Payload: totalBytes,
Epoch: epoch,
}); err != nil {
log.Warnf("failed to write back deal average response: %s", err)
return
@ -157,6 +132,12 @@ func (dss *dealStatsServer) handleStorageDealTotalReal(w http.ResponseWriter, r
}
type clientStatsOutput struct {
Epoch int64 `json:"epoch"`
Endpoint string `json:"endpoint"`
Payload []*clientStats `json:"payload"`
}
type clientStats struct {
Client address.Address `json:"client"`
DataSize int64 `json:"data_size"`
NumCids int `json:"num_cids"`
@ -168,51 +149,41 @@ type clientStatsOutput struct {
}
func (dss *dealStatsServer) handleStorageClientStats(w http.ResponseWriter, r *http.Request) {
ctx := context.Background()
head, err := dss.api.ChainHead(ctx)
if err != nil {
log.Warnf("failed to get chain head: %s", err)
epoch, deals := dss.filteredDealList()
if epoch == 0 {
w.WriteHeader(500)
return
}
deals, err := dss.api.StateMarketDeals(ctx, head.Key())
if err != nil {
log.Warnf("failed to get market deals: %s", err)
w.WriteHeader(500)
return
}
stats := make(map[address.Address]*clientStatsOutput)
stats := make(map[address.Address]*clientStats)
for _, d := range deals {
if filteredClients[d.Proposal.Client] {
continue
}
st, ok := stats[d.Proposal.Client]
st, ok := stats[d.deal.Proposal.Client]
if !ok {
st = &clientStatsOutput{
Client: d.Proposal.Client,
st = &clientStats{
Client: d.resolvedWallet,
cids: make(map[cid.Cid]bool),
providers: make(map[address.Address]bool),
}
stats[d.Proposal.Client] = st
stats[d.deal.Proposal.Client] = st
}
st.DataSize += int64(d.Proposal.PieceSize.Unpadded())
st.cids[d.Proposal.PieceCID] = true
st.providers[d.Proposal.Provider] = true
st.DataSize += int64(d.deal.Proposal.PieceSize.Unpadded())
st.cids[d.deal.Proposal.PieceCID] = true
st.providers[d.deal.Proposal.Provider] = true
st.NumDeals++
}
out := make([]*clientStatsOutput, 0, len(stats))
for _, cso := range stats {
cso.NumCids = len(cso.cids)
cso.NumMiners = len(cso.providers)
out = append(out, cso)
out := clientStatsOutput{
Epoch: epoch,
Endpoint: "CLIENT_DEAL_STATS",
Payload: make([]*clientStats, 0, len(stats)),
}
for _, cs := range stats {
cs.NumCids = len(cs.cids)
cs.NumMiners = len(cs.providers)
out.Payload = append(out.Payload, cs)
}
if err := json.NewEncoder(w).Encode(out); err != nil {
@ -221,6 +192,93 @@ func (dss *dealStatsServer) handleStorageClientStats(w http.ResponseWriter, r *h
}
}
type dealInfo struct {
deal api.MarketDeal
resolvedWallet address.Address
}
// filteredDealList returns the current epoch and a list of filtered deals
// on error returns an epoch of 0
func (dss *dealStatsServer) filteredDealList() (int64, map[string]dealInfo) {
ctx := context.Background()
head, err := dss.api.ChainHead(ctx)
if err != nil {
log.Warnf("failed to get chain head: %s", err)
return 0, nil
}
head, err = dss.api.ChainGetTipSetByHeight(ctx, head.Height()-epochLookback, head.Key())
if err != nil {
log.Warnf("failed to walk back %s epochs: %s", epochLookback, err)
return 0, nil
}
// Disabled as per @pooja's request
//
// // Exclude any address associated with a miner
// miners, err := dss.api.StateListMiners(ctx, head.Key())
// if err != nil {
// log.Warnf("failed to get miner list: %s", err)
// return 0, nil
// }
// for _, m := range miners {
// info, err := dss.api.StateMinerInfo(ctx, m, head.Key())
// if err != nil {
// log.Warnf("failed to get info for known miner '%s': %s", m, err)
// continue
// }
// knownFiltered.Store(info.Owner, true)
// knownFiltered.Store(info.Worker, true)
// for _, a := range info.ControlAddresses {
// knownFiltered.Store(a, true)
// }
// }
deals, err := dss.api.StateMarketDeals(ctx, head.Key())
if err != nil {
log.Warnf("failed to get market deals: %s", err)
return 0, nil
}
ret := make(map[string]dealInfo, len(deals))
for dealKey, d := range deals {
// Counting no-longer-active deals as per Pooja's request
// // https://github.com/filecoin-project/specs-actors/blob/v0.9.9/actors/builtin/market/deal.go#L81-L85
// if d.State.SectorStartEpoch < 0 {
// continue
// }
if _, isFiltered := knownFiltered.Load(d.Proposal.Client); isFiltered {
continue
}
if _, wasSeen := resolvedWallets.Load(d.Proposal.Client); !wasSeen {
w, err := dss.api.StateAccountKey(ctx, d.Proposal.Client, head.Key())
if err != nil {
log.Warnf("failed to resolve id '%s' to wallet address: %s", d.Proposal.Client, err)
continue
} else {
resolvedWallets.Store(d.Proposal.Client, w)
}
}
w, _ := resolvedWallets.Load(d.Proposal.Client)
if _, isFiltered := knownFiltered.Load(w); isFiltered {
continue
}
ret[dealKey] = dealInfo{
deal: d,
resolvedWallet: w.(address.Address),
}
}
return int64(head.Height()), ret
}
var serveDealStatsCmd = &cli.Command{
Name: "serve-deal-stats",
Flags: []cli.Flag{},
@ -260,6 +318,8 @@ var serveDealStatsCmd = &cli.Command{
panic(err)
}
log.Warnf("deal-stat server listening on %s\n== NOTE: QUERIES ARE EXPENSIVE - YOU MUST FRONT-CACHE THIS SERVICE\n", list.Addr().String())
return s.Serve(list)
},
}

57
cmd/lotus-shed/import-car.go
View File

@ -1,10 +1,13 @@
package main
import (
"encoding/hex"
"fmt"
"io"
"os"
block "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"github.com/ipld/go-car"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
@ -81,3 +84,57 @@ var importCarCmd = &cli.Command{
}
},
}
var importObjectCmd = &cli.Command{
Name: "import-obj",
Usage: "import a raw ipld object into your datastore",
Action: func(cctx *cli.Context) error {
r, err := repo.NewFS(cctx.String("repo"))
if err != nil {
return xerrors.Errorf("opening fs repo: %w", err)
}
exists, err := r.Exists()
if err != nil {
return err
}
if !exists {
return xerrors.Errorf("lotus repo doesn't exist")
}
lr, err := r.Lock(repo.FullNode)
if err != nil {
return err
}
defer lr.Close() //nolint:errcheck
ds, err := lr.Datastore("/chain")
if err != nil {
return err
}
bs := blockstore.NewBlockstore(ds)
c, err := cid.Decode(cctx.Args().Get(0))
if err != nil {
return err
}
data, err := hex.DecodeString(cctx.Args().Get(1))
if err != nil {
return err
}
blk, err := block.NewBlockWithCid(data, c)
if err != nil {
return err
}
if err := bs.Put(blk); err != nil {
return err
}
return nil
},
}

2
cmd/lotus-shed/main.go
View File

@ -25,6 +25,7 @@ func main() {
staterootCmd,
auditsCmd,
importCarCmd,
importObjectCmd,
commpToCidCmd,
fetchParamCmd,
proofsCmd,
@ -37,6 +38,7 @@ func main() {
exportChainCmd,
consensusCmd,
serveDealStatsCmd,
syncCmd,
datastoreCmd,
}

64
cmd/lotus-shed/sync.go Normal file
View File

@ -0,0 +1,64 @@
package main
import (
"fmt"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/lotus/chain/types"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/urfave/cli/v2"
)
var syncCmd = &cli.Command{
Name: "sync",
Usage: "tools for diagnosing sync issues",
Flags: []cli.Flag{},
Subcommands: []*cli.Command{
syncValidateCmd,
},
}
var syncValidateCmd = &cli.Command{
Name: "validate",
Usage: "checks whether a provided tipset is valid",
Action: func(cctx *cli.Context) error {
api, closer, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
if cctx.Args().Len() < 1 {
fmt.Println("usage: <blockCid1> <blockCid2>...")
fmt.Println("At least one block cid must be provided")
return nil
}
args := cctx.Args().Slice()
var tscids []cid.Cid
for _, s := range args {
c, err := cid.Decode(s)
if err != nil {
return fmt.Errorf("block cid was invalid: %s", err)
}
tscids = append(tscids, c)
}
tsk := types.NewTipSetKey(tscids...)
valid, err := api.SyncValidateTipset(ctx, tsk)
if err != nil {
fmt.Println("Tipset is invalid: ", err)
}
if valid {
fmt.Println("Tipset is valid")
}
return nil
},
}

16
cmd/lotus-storage-miner/info.go
View File

@ -33,6 +33,12 @@ var infoCmd = &cli.Command{
Subcommands: []*cli.Command{
infoAllCmd,
},
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "hide-sectors-info",
Usage: "hide sectors info",
},
},
Action: infoCmdAct,
}
@ -199,10 +205,12 @@ func infoCmdAct(cctx *cli.Context) error {
fmt.Printf("Expected Seal Duration: %s\n\n", sealdur)
fmt.Println("Sectors:")
err = sectorsInfo(ctx, nodeApi)
if err != nil {
return err
if !cctx.Bool("hide-sectors-info") {
fmt.Println("Sectors:")
err = sectorsInfo(ctx, nodeApi)
if err != nil {
return err
}
}
// TODO: grab actr state / info

78
cmd/lotus/daemon.go
View File

@ -3,11 +3,14 @@
package main
import (
"bufio"
"context"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"runtime/pprof"
"strings"
@ -23,6 +26,7 @@ import (
"go.opencensus.io/stats/view"
"go.opencensus.io/tag"
"golang.org/x/xerrors"
"gopkg.in/cheggaaa/pb.v1"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
@ -100,11 +104,11 @@ var DaemonCmd = &cli.Command{
},
&cli.StringFlag{
Name: "import-chain",
Usage: "on first run, load chain from given file and validate",
Usage: "on first run, load chain from given file or url and validate",
},
&cli.StringFlag{
Name: "import-snapshot",
Usage: "import chain state from a given chain export file",
Usage: "import chain state from a given chain export file or url",
},
&cli.BoolFlag{
Name: "halt-after-import",
@ -123,6 +127,10 @@ var DaemonCmd = &cli.Command{
Usage: "manage open file limit",
Value: true,
},
&cli.StringFlag{
Name: "config",
Usage: "specify path of config file to use",
},
},
Action: func(cctx *cli.Context) error {
err := runmetrics.Enable(runmetrics.RunMetricOptions{
@ -176,6 +184,10 @@ var DaemonCmd = &cli.Command{
return xerrors.Errorf("opening fs repo: %w", err)
}
if cctx.String("config") != "" {
r.SetConfigPath(cctx.String("config"))
}
if err := r.Init(repo.FullNode); err != nil && err != repo.ErrRepoExists {
return xerrors.Errorf("repo init error: %w", err)
}
@ -206,11 +218,6 @@ var DaemonCmd = &cli.Command{
issnapshot = true
}
chainfile, err := homedir.Expand(chainfile)
if err != nil {
return err
}
if err := ImportChain(r, chainfile, issnapshot); err != nil {
return err
}
@ -326,12 +333,42 @@ func importKey(ctx context.Context, api api.FullNode, f string) error {
return nil
}
func ImportChain(r repo.Repo, fname string, snapshot bool) error {
fi, err := os.Open(fname)
if err != nil {
return err
func ImportChain(r repo.Repo, fname string, snapshot bool) (err error) {
var rd io.Reader
var l int64
if strings.HasPrefix(fname, "http://") || strings.HasPrefix(fname, "https://") {
resp, err := http.Get(fname) //nolint:gosec
if err != nil {
return err
}
defer resp.Body.Close() //nolint:errcheck
if resp.StatusCode != http.StatusOK {
return xerrors.Errorf("non-200 response: %d", resp.StatusCode)
}
rd = resp.Body
l = resp.ContentLength
} else {
fname, err = homedir.Expand(fname)
if err != nil {
return err
}
fi, err := os.Open(fname)
if err != nil {
return err
}
defer fi.Close() //nolint:errcheck
st, err := os.Stat(fname)
if err != nil {
return err
}
rd = fi
l = st.Size()
}
defer fi.Close() //nolint:errcheck
lr, err := r.Lock(repo.FullNode)
if err != nil {
@ -353,8 +390,21 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) error {
cst := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier))
log.Info("importing chain from file...")
ts, err := cst.Import(fi)
log.Infof("importing chain from %s...", fname)
bufr := bufio.NewReaderSize(rd, 1<<20)
bar := pb.New64(l)
br := bar.NewProxyReader(bufr)
bar.ShowTimeLeft = true
bar.ShowPercent = true
bar.ShowSpeed = true
bar.Units = pb.U_BYTES
bar.Start()
ts, err := cst.Import(br)
bar.Finish()
if err != nil {
return xerrors.Errorf("importing chain failed: %w", err)
}

44
cmd/tvx/actor_mapping.go Normal file
View File

@ -0,0 +1,44 @@
package main
import (
"reflect"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
)
var ActorMethodTable = make(map[string][]string, 64)
var Actors = map[cid.Cid]interface{}{
builtin.InitActorCodeID: builtin.MethodsInit,
builtin.CronActorCodeID: builtin.MethodsCron,
builtin.AccountActorCodeID: builtin.MethodsAccount,
builtin.StoragePowerActorCodeID: builtin.MethodsPower,
builtin.StorageMinerActorCodeID: builtin.MethodsMiner,
builtin.StorageMarketActorCodeID: builtin.MethodsMarket,
builtin.PaymentChannelActorCodeID: builtin.MethodsPaych,
builtin.MultisigActorCodeID: builtin.MethodsMultisig,
builtin.RewardActorCodeID: builtin.MethodsReward,
builtin.VerifiedRegistryActorCodeID: builtin.MethodsVerifiedRegistry,
}
func init() {
for code, methods := range Actors {
cmh, err := multihash.Decode(code.Hash()) // identity hash.
if err != nil {
panic(err)
}
var (
aname = string(cmh.Digest)
rt = reflect.TypeOf(methods)
nf = rt.NumField()
)
ActorMethodTable[aname] = append(ActorMethodTable[aname], "Send")
for i := 0; i < nf; i++ {
ActorMethodTable[aname] = append(ActorMethodTable[aname], rt.Field(i).Name)
}
}
}

92
cmd/tvx/exec.go Normal file
View File

@ -0,0 +1,92 @@
package main
import (
"encoding/json"
"fmt"
"io"
"log"
"os"
"github.com/fatih/color"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/lotus/conformance"
"github.com/filecoin-project/test-vectors/schema"
)
var execFlags struct {
file string
}
var execCmd = &cli.Command{
Name: "exec",
Description: "execute one or many test vectors against Lotus; supplied as a single JSON file, or a ndjson stdin stream",
Action: runExecLotus,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "file",
Usage: "input file; if not supplied, the vector will be read from stdin",
TakesFile: true,
Destination: &execFlags.file,
},
},
}
func runExecLotus(_ *cli.Context) error {
if file := execFlags.file; file != "" {
// we have a single test vector supplied as a file.
file, err := os.Open(file)
if err != nil {
return fmt.Errorf("failed to open test vector: %w", err)
}
var (
dec = json.NewDecoder(file)
tv schema.TestVector
)
if err = dec.Decode(&tv); err != nil {
return fmt.Errorf("failed to decode test vector: %w", err)
}
return executeTestVector(tv)
}
for dec := json.NewDecoder(os.Stdin); ; {
var tv schema.TestVector
switch err := dec.Decode(&tv); err {
case nil:
if err = executeTestVector(tv); err != nil {
return err
}
case io.EOF:
// we're done.
return nil
default:
// something bad happened.
return err
}
}
}
func executeTestVector(tv schema.TestVector) error {
log.Println("executing test vector:", tv.Meta.ID)
r := new(conformance.LogReporter)
switch class := tv.Class; class {
case "message":
conformance.ExecuteMessageVector(r, &tv)
case "tipset":
conformance.ExecuteTipsetVector(r, &tv)
default:
return fmt.Errorf("test vector class %s not supported", class)
}
if r.Failed() {
log.Println(color.HiRedString("❌ test vector failed"))
} else {
log.Println(color.GreenString("✅ test vector succeeded"))
}
return nil
}

503
cmd/tvx/extract.go Normal file
View File

@ -0,0 +1,503 @@
package main
import (
"bytes"
"compress/gzip"
"context"
"encoding/json"
"fmt"
"io"
"log"
"os"
"path/filepath"
"github.com/fatih/color"
"github.com/filecoin-project/lotus/api"
init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init"
"github.com/filecoin-project/lotus/chain/actors/builtin/reward"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/conformance"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/test-vectors/schema"
"github.com/ipfs/go-cid"
"github.com/urfave/cli/v2"
)
const (
PrecursorSelectAll = "all"
PrecursorSelectSender = "sender"
)
type extractOpts struct {
id string
block string
class string
cid string
file string
retain string
precursor string
}
var extractFlags extractOpts
var extractCmd = &cli.Command{
Name: "extract",
Description: "generate a test vector by extracting it from a live chain",
Action: runExtract,
Flags: []cli.Flag{
&repoFlag,
&cli.StringFlag{
Name: "class",
Usage: "class of vector to extract; other required flags depend on the; values: 'message'",
Value: "message",
Destination: &extractFlags.class,
},
&cli.StringFlag{
Name: "id",
Usage: "identifier to name this test vector with",
Value: "(undefined)",
Destination: &extractFlags.id,
},
&cli.StringFlag{
Name: "block",
Usage: "optionally, the block CID the message was included in, to avoid expensive chain scanning",
Destination: &extractFlags.block,
},
&cli.StringFlag{
Name: "cid",
Usage: "message CID to generate test vector from",
Required: true,
Destination: &extractFlags.cid,
},
&cli.StringFlag{
Name: "out",
Aliases: []string{"o"},
Usage: "file to write test vector to",
Destination: &extractFlags.file,
},
&cli.StringFlag{
Name: "state-retain",
Usage: "state retention policy; values: 'accessed-cids', 'accessed-actors'",
Value: "accessed-cids",
Destination: &extractFlags.retain,
},
&cli.StringFlag{
Name: "precursor-select",
Usage: "precursors to apply; values: 'all', 'sender'; 'all' selects all preceding" +
"messages in the canonicalised tipset, 'sender' selects only preceding messages from the same" +
"sender. Usually, 'sender' is a good tradeoff and gives you sufficient accuracy. If the receipt sanity" +
"check fails due to gas reasons, switch to 'all', as previous messages in the tipset may have" +
"affected state in a disruptive way",
Value: "sender",
Destination: &extractFlags.precursor,
},
},
}
func runExtract(c *cli.Context) error {
// LOTUS_DISABLE_VM_BUF disables what's called "VM state tree buffering",
// which stashes write operations in a BufferedBlockstore
// (https://github.com/filecoin-project/lotus/blob/b7a4dbb07fd8332b4492313a617e3458f8003b2a/lib/bufbstore/buf_bstore.go#L21)
// such that they're not written until the VM is actually flushed.
//
// For some reason, the standard behaviour was not working for me (raulk),
// and disabling it (such that the state transformations are written immediately
// to the blockstore) worked.
_ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea")
ctx := context.Background()
// Make the API client.
fapi, closer, err := lcli.GetFullNodeAPI(c)
if err != nil {
return err
}
defer closer()
return doExtract(ctx, fapi, extractFlags)
}
func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
mcid, err := cid.Decode(opts.cid)
if err != nil {
return err
}
msg, execTs, incTs, err := resolveFromChain(ctx, fapi, mcid, opts.block)
if err != nil {
return fmt.Errorf("failed to resolve message and tipsets from chain: %w", err)
}
// get the circulating supply before the message was executed.
circSupplyDetail, err := fapi.StateCirculatingSupply(ctx, incTs.Key())
if err != nil {
return fmt.Errorf("failed while fetching circulating supply: %w", err)
}
circSupply := circSupplyDetail.FilCirculating
log.Printf("message was executed in tipset: %s", execTs.Key())
log.Printf("message was included in tipset: %s", incTs.Key())
log.Printf("circulating supply at inclusion tipset: %d", circSupply)
log.Printf("finding precursor messages using mode: %s", opts.precursor)
// Fetch messages in canonical order from inclusion tipset.
msgs, err := fapi.ChainGetParentMessages(ctx, execTs.Blocks()[0].Cid())
if err != nil {
return fmt.Errorf("failed to fetch messages in canonical order from inclusion tipset: %w", err)
}
related, found, err := findMsgAndPrecursors(opts.precursor, msg, msgs)
if err != nil {
return fmt.Errorf("failed while finding message and precursors: %w", err)
}
if !found {
return fmt.Errorf("message not found; precursors found: %d", len(related))
}
var (
precursors = related[:len(related)-1]
precursorsCids []cid.Cid
)
for _, p := range precursors {
precursorsCids = append(precursorsCids, p.Cid())
}
log.Println(color.GreenString("found message; precursors (count: %d): %v", len(precursors), precursorsCids))
var (
// create a read-through store that uses ChainGetObject to fetch unknown CIDs.
pst = NewProxyingStores(ctx, fapi)
g = NewSurgeon(ctx, fapi, pst)
)
driver := conformance.NewDriver(ctx, schema.Selector{}, conformance.DriverOpts{
DisableVMFlush: true,
})
// this is the root of the state tree we start with.
root := incTs.ParentState()
log.Printf("base state tree root CID: %s", root)
basefee := incTs.Blocks()[0].ParentBaseFee
log.Printf("basefee: %s", basefee)
// on top of that state tree, we apply all precursors.
log.Printf("number of precursors to apply: %d", len(precursors))
for i, m := range precursors {
log.Printf("applying precursor %d, cid: %s", i, m.Cid())
_, root, err = driver.ExecuteMessage(pst.Blockstore, conformance.ExecuteMessageParams{
Preroot: root,
Epoch: execTs.Height(),
Message: m,
CircSupply: &circSupplyDetail.FilCirculating,
BaseFee: &basefee,
})
if err != nil {
return fmt.Errorf("failed to execute precursor message: %w", err)
}
}
var (
preroot cid.Cid
postroot cid.Cid
applyret *vm.ApplyRet
carWriter func(w io.Writer) error
retention = opts.retain
)
log.Printf("using state retention strategy: %s", retention)
switch retention {
case "accessed-cids":
tbs, ok := pst.Blockstore.(TracingBlockstore)
if !ok {
return fmt.Errorf("requested 'accessed-cids' state retention, but no tracing blockstore was present")
}
tbs.StartTracing()
preroot = root
applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, conformance.ExecuteMessageParams{
Preroot: preroot,
Epoch: execTs.Height(),
Message: msg,
CircSupply: &circSupplyDetail.FilCirculating,
BaseFee: &basefee,
})
if err != nil {
return fmt.Errorf("failed to execute message: %w", err)
}
accessed := tbs.FinishTracing()
carWriter = func(w io.Writer) error {
return g.WriteCARIncluding(w, accessed, preroot, postroot)
}
case "accessed-actors":
log.Printf("calculating accessed actors")
// get actors accessed by message.
retain, err := g.GetAccessedActors(ctx, fapi, mcid)
if err != nil {
return fmt.Errorf("failed to calculate accessed actors: %w", err)
}
// also append the reward actor and the burnt funds actor.
retain = append(retain, reward.Address, builtin.BurntFundsActorAddr, init_.Address)
log.Printf("calculated accessed actors: %v", retain)
// get the masked state tree from the root,
preroot, err = g.GetMaskedStateTree(root, retain)
if err != nil {
return err
}
applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, conformance.ExecuteMessageParams{
Preroot: preroot,
Epoch: execTs.Height(),
Message: msg,
CircSupply: &circSupplyDetail.FilCirculating,
BaseFee: &basefee,
})
if err != nil {
return fmt.Errorf("failed to execute message: %w", err)
}
carWriter = func(w io.Writer) error {
return g.WriteCAR(w, preroot, postroot)
}
default:
return fmt.Errorf("unknown state retention option: %s", retention)
}
log.Printf("message applied; preroot: %s, postroot: %s", preroot, postroot)
log.Println("performing sanity check on receipt")
// TODO sometimes this returns a nil receipt and no error ¯\_(ツ)_/¯
// ex: https://filfox.info/en/message/bafy2bzacebpxw3yiaxzy2bako62akig46x3imji7fewszen6fryiz6nymu2b2
// This code is lenient and skips receipt comparison in case of a nil receipt.
rec, err := fapi.StateGetReceipt(ctx, mcid, execTs.Key())
if err != nil {
return fmt.Errorf("failed to find receipt on chain: %w", err)
}
log.Printf("found receipt: %+v", rec)
// generate the schema receipt; if we got
var receipt *schema.Receipt
if rec != nil {
receipt = &schema.Receipt{
ExitCode: int64(rec.ExitCode),
ReturnValue: rec.Return,
GasUsed: rec.GasUsed,
}
reporter := new(conformance.LogReporter)
conformance.AssertMsgResult(reporter, receipt, applyret, "as locally executed")
if reporter.Failed() {
log.Println(color.RedString("receipt sanity check failed; aborting"))
return fmt.Errorf("vector generation aborted")
}
log.Println(color.GreenString("receipt sanity check succeeded"))
} else {
receipt = &schema.Receipt{
ExitCode: int64(applyret.ExitCode),
ReturnValue: applyret.Return,
GasUsed: applyret.GasUsed,
}
log.Println(color.YellowString("skipping receipts comparison; we got back a nil receipt from lotus"))
}
log.Println("generating vector")
msgBytes, err := msg.Serialize()
if err != nil {
return err
}
var (
out = new(bytes.Buffer)
gw = gzip.NewWriter(out)
)
if err := carWriter(gw); err != nil {
return err
}
if err = gw.Flush(); err != nil {
return err
}
if err = gw.Close(); err != nil {
return err
}
version, err := fapi.Version(ctx)
if err != nil {
return err
}
ntwkName, err := fapi.StateNetworkName(ctx)
if err != nil {
return err
}
// Write out the test vector.
vector := schema.TestVector{
Class: schema.ClassMessage,
Meta: &schema.Metadata{
ID: opts.id,
// TODO need to replace schema.GenerationData with a more flexible
// data structure that makes no assumption about the traceability
// data that's being recorded; a flexible map[string]string
// would do.
Gen: []schema.GenerationData{
{Source: fmt.Sprintf("network:%s", ntwkName)},
{Source: fmt.Sprintf("message:%s", msg.Cid().String())},
{Source: fmt.Sprintf("inclusion_tipset:%s", incTs.Key().String())},
{Source: fmt.Sprintf("execution_tipset:%s", execTs.Key().String())},
{Source: "github.com/filecoin-project/lotus", Version: version.String()}},
},
CAR: out.Bytes(),
Pre: &schema.Preconditions{
Epoch: int64(execTs.Height()),
CircSupply: circSupply.Int,
BaseFee: basefee.Int,
StateTree: &schema.StateTree{
RootCID: preroot,
},
},
ApplyMessages: []schema.Message{{Bytes: msgBytes}},
Post: &schema.Postconditions{
StateTree: &schema.StateTree{
RootCID: postroot,
},
Receipts: []*schema.Receipt{
{
ExitCode: int64(applyret.ExitCode),
ReturnValue: applyret.Return,
GasUsed: applyret.GasUsed,
},
},
},
}
output := io.WriteCloser(os.Stdout)
if file := opts.file; file != "" {
dir := filepath.Dir(file)
if err := os.MkdirAll(dir, 0755); err != nil {
return fmt.Errorf("unable to create directory %s: %w", dir, err)
}
output, err = os.Create(file)
if err != nil {
return err
}
defer output.Close() //nolint:errcheck
defer log.Printf("wrote test vector to file: %s", file)
}
enc := json.NewEncoder(output)
enc.SetIndent("", " ")
if err := enc.Encode(&vector); err != nil {
return err
}
return nil
}
// resolveFromChain queries the chain for the provided message, using the block CID to
// speed up the query, if provided
func resolveFromChain(ctx context.Context, api api.FullNode, mcid cid.Cid, block string) (msg *types.Message, execTs *types.TipSet, incTs *types.TipSet, err error) {
// Extract the full message.
msg, err = api.ChainGetMessage(ctx, mcid)
if err != nil {
return nil, nil, nil, err
}
log.Printf("found message with CID %s: %+v", mcid, msg)
if block == "" {
log.Printf("locating message in blockchain")
// Locate the message.
msgInfo, err := api.StateSearchMsg(ctx, mcid)
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to locate message: %w", err)
}
log.Printf("located message at tipset %s (height: %d) with exit code: %s", msgInfo.TipSet, msgInfo.Height, msgInfo.Receipt.ExitCode)
execTs, incTs, err = fetchThisAndPrevTipset(ctx, api, msgInfo.TipSet)
return msg, execTs, incTs, err
}
bcid, err := cid.Decode(block)
if err != nil {
return nil, nil, nil, err
}
log.Printf("message inclusion block CID was provided; scanning around it: %s", bcid)
blk, err := api.ChainGetBlock(ctx, bcid)
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to get block: %w", err)
}
// types.EmptyTSK hints to use the HEAD.
execTs, err = api.ChainGetTipSetByHeight(ctx, blk.Height+1, types.EmptyTSK)
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to get message execution tipset: %w", err)
}
// walk back from the execTs instead of HEAD, to save time.
incTs, err = api.ChainGetTipSetByHeight(ctx, blk.Height, execTs.Key())
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to get message inclusion tipset: %w", err)
}
return msg, execTs, incTs, nil
}
// fetchThisAndPrevTipset returns the full tipset identified by the key, as well
// as the previous tipset. In the context of vector generation, the target
// tipset is the one where a message was executed, and the previous tipset is
// the one where the message was included.
func fetchThisAndPrevTipset(ctx context.Context, api api.FullNode, target types.TipSetKey) (targetTs *types.TipSet, prevTs *types.TipSet, err error) {
// get the tipset on which this message was "executed" on.
// https://github.com/filecoin-project/lotus/issues/2847
targetTs, err = api.ChainGetTipSet(ctx, target)
if err != nil {
return nil, nil, err
}
// get the previous tipset, on which this message was mined,
// i.e. included on-chain.
prevTs, err = api.ChainGetTipSet(ctx, targetTs.Parents())
if err != nil {
return nil, nil, err
}
return targetTs, prevTs, nil
}
// findMsgAndPrecursors ranges through the canonical messages slice, locating
// the target message and returning precursors in accordance to the supplied
// mode.
func findMsgAndPrecursors(mode string, target *types.Message, msgs []api.Message) (related []*types.Message, found bool, err error) {
// Range through canonicalised messages, selecting only the precursors based
// on selection mode.
for _, other := range msgs {
switch {
case mode == PrecursorSelectAll:
fallthrough
case mode == PrecursorSelectSender && other.Message.From == target.From:
related = append(related, other.Message)
}
// this message is the target; we're done.
if other.Cid == target.Cid() {
return related, true, nil
}
}
// this could happen because a block contained related messages, but not
// the target (that is, messages with a lower nonce, but ultimately not the
// target).
return related, false, nil
}

232
cmd/tvx/extract_many.go Normal file
View File

@ -0,0 +1,232 @@
package main
import (
"context"
"encoding/csv"
"fmt"
"io"
"log"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/fatih/color"
"github.com/filecoin-project/go-state-types/exitcode"
"github.com/hashicorp/go-multierror"
"github.com/urfave/cli/v2"
lcli "github.com/filecoin-project/lotus/cli"
)
var extractManyFlags struct {
in string
outdir string
batchId string
}
var extractManyCmd = &cli.Command{
Name: "extract-many",
Description: `generate many test vectors by repeatedly calling tvx extract, using a csv file as input.
The CSV file must have a format just like the following:
message_cid,receiver_code,method_num,exit_code,height,block_cid,seq
bafy2bzacedvuvgpsnwq7i7kltfap6hnp7fdmzf6lr4w34zycjrthb3v7k6zi6,fil/1/account,0,0,67972,bafy2bzacebthpxzlk7zhlkz3jfzl4qw7mdoswcxlf3rkof3b4mbxfj3qzfk7w,1
bafy2bzacedwicofymn4imgny2hhbmcm4o5bikwnv3qqgohyx73fbtopiqlro6,fil/1/account,0,0,67860,bafy2bzacebj7beoxyzll522o6o76mt7von4psn3tlvunokhv4zhpwmfpipgti,2
...
The first row MUST be a header row. At the bare minimum, those seven fields
must appear, in the order specified. Extra fields are accepted, but always
after these compulsory seven.
`,
Action: runExtractMany,
Flags: []cli.Flag{
&repoFlag,
&cli.StringFlag{
Name: "batch-id",
Usage: "batch id; a four-digit left-zero-padded sequential number (e.g. 0041)",
Required: true,
Destination: &extractManyFlags.batchId,
},
&cli.StringFlag{
Name: "in",
Usage: "path to input file (csv)",
Destination: &extractManyFlags.in,
},
&cli.StringFlag{
Name: "outdir",
Usage: "output directory",
Destination: &extractManyFlags.outdir,
},
},
}
func runExtractMany(c *cli.Context) error {
// LOTUS_DISABLE_VM_BUF disables what's called "VM state tree buffering",
// which stashes write operations in a BufferedBlockstore
// (https://github.com/filecoin-project/lotus/blob/b7a4dbb07fd8332b4492313a617e3458f8003b2a/lib/bufbstore/buf_bstore.go#L21)
// such that they're not written until the VM is actually flushed.
//
// For some reason, the standard behaviour was not working for me (raulk),
// and disabling it (such that the state transformations are written immediately
// to the blockstore) worked.
_ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea")
ctx := context.Background()
// Make the API client.
fapi, closer, err := lcli.GetFullNodeAPI(c)
if err != nil {
return err
}
defer closer()
var (
in = extractManyFlags.in
outdir = extractManyFlags.outdir
)
if in == "" {
return fmt.Errorf("input file not provided")
}
if outdir == "" {
return fmt.Errorf("output dir not provided")
}
// Open the CSV file for reading.
f, err := os.Open(in)
if err != nil {
return fmt.Errorf("could not open file %s: %w", in, err)
}
// Ensure the output directory exists.
if err := os.MkdirAll(outdir, 0755); err != nil {
return fmt.Errorf("could not create output dir %s: %w", outdir, err)
}
// Create a CSV reader and validate the header row.
reader := csv.NewReader(f)
if header, err := reader.Read(); err != nil {
return fmt.Errorf("failed to read header from csv: %w", err)
} else if l := len(header); l < 7 {
return fmt.Errorf("insufficient number of fields: %d", l)
} else if f := header[0]; f != "message_cid" {
return fmt.Errorf("csv sanity check failed: expected first field in header to be 'message_cid'; was: %s", f)
} else {
log.Println(color.GreenString("csv sanity check succeeded; header contains fields: %v", header))
}
var (
generated []string
merr = new(multierror.Error)
retry []extractOpts // to retry with 'canonical' precursor selection mode
)
// Read each row and extract the requested message.
for {
row, err := reader.Read()
if err == io.EOF {
break
} else if err != nil {
return fmt.Errorf("failed to read row: %w", err)
}
var (
cid = row[0]
actorcode = row[1]
methodnumstr = row[2]
exitcodestr = row[3]
_ = row[4]
block = row[5]
seq = row[6]
exit int
methodnum int
methodname string
)
// Parse the exit code.
if exit, err = strconv.Atoi(exitcodestr); err != nil {
return fmt.Errorf("invalid exitcode number: %d", exit)
}
// Parse the method number.
if methodnum, err = strconv.Atoi(methodnumstr); err != nil {
return fmt.Errorf("invalid method number: %s", methodnumstr)
}
// Lookup the method in actor method table.
if m, ok := ActorMethodTable[actorcode]; !ok {
return fmt.Errorf("unrecognized actor: %s", actorcode)
} else if methodnum >= len(m) {
return fmt.Errorf("unrecognized method number for actor %s: %d", actorcode, methodnum)
} else {
methodname = m[methodnum]
}
// exitcode string representations are of kind ErrType(0); strip out
// the number portion.
exitcodename := strings.Split(exitcode.ExitCode(exit).String(), "(")[0]
// replace the slashes in the actor code name with underscores.
actorcodename := strings.ReplaceAll(actorcode, "/", "_")
// Compute the ID of the vector.
id := fmt.Sprintf("ext-%s-%s-%s-%s-%s", extractManyFlags.batchId, actorcodename, methodname, exitcodename, seq)
// Vector filename, using a base of outdir.
file := filepath.Join(outdir, actorcodename, methodname, exitcodename, id) + ".json"
log.Println(color.YellowString("processing message cid with 'sender' precursor mode: %s", id))
opts := extractOpts{
id: id,
block: block,
class: "message",
cid: cid,
file: file,
retain: "accessed-cids",
precursor: PrecursorSelectSender,
}
if err := doExtract(ctx, fapi, opts); err != nil {
log.Println(color.RedString("failed to extract vector for message %s: %s; queuing for 'canonical' precursor selection", cid, err))
retry = append(retry, opts)
continue
}
log.Println(color.MagentaString("generated file: %s", file))
generated = append(generated, file)
}
log.Printf("extractions to try with canonical precursor selection mode: %d", len(retry))
for _, r := range retry {
log.Printf("retrying %s: %s", r.cid, r.id)
r.precursor = PrecursorSelectAll
if err := doExtract(ctx, fapi, r); err != nil {
merr = multierror.Append(merr, fmt.Errorf("failed to extract vector for message %s: %w", r.cid, err))
continue
}
log.Println(color.MagentaString("generated file: %s", r.file))
generated = append(generated, r.file)
}
if len(generated) == 0 {
log.Println("no files generated")
} else {
log.Println("files generated:")
for _, g := range generated {
log.Println(g)
}
}
if merr.ErrorOrNil() != nil {
log.Println(color.YellowString("done processing with errors: %v", merr))
} else {
log.Println(color.GreenString("done processing with no errors"))
}
return merr.ErrorOrNil()
}

71
cmd/tvx/main.go Normal file
View File

@ -0,0 +1,71 @@
package main
import (
"log"
"os"
"sort"
"github.com/urfave/cli/v2"
)
// DefaultLotusRepoPath is where the fallback path where to look for a Lotus
// client repo. It is expanded with mitchellh/go-homedir, so it'll work with all
// OSes despite the Unix twiddle notation.
const DefaultLotusRepoPath = "~/.lotus"
var repoFlag = cli.StringFlag{
Name: "repo",
EnvVars: []string{"LOTUS_PATH"},
Value: DefaultLotusRepoPath,
TakesFile: true,
}
func main() {
app := &cli.App{
Name: "tvx",
Description: `tvx is a tool for extracting and executing test vectors. It has three subcommands.
tvx extract extracts a test vector from a live network. It requires access to
a Filecoin client that exposes the standard JSON-RPC API endpoint. Only
message class test vectors are supported at this time.
tvx exec executes test vectors against Lotus. Either you can supply one in a
file, or many as an ndjson stdin stream.
tvx extract-many performs a batch extraction of many messages, supplied in a
CSV file. Refer to the help of that subcommand for more info.
SETTING THE JSON-RPC API ENDPOINT
You can set the JSON-RPC API endpoint through one of the following methods.
1. Directly set the API endpoint on the FULLNODE_API_INFO env variable.
The format is [token]:multiaddr, where token is optional for commands not
accessing privileged operations.
2. If you're running tvx against a local Lotus client, you can set the REPO
env variable to have the API endpoint and token extracted from the repo.
Alternatively, you can pass the --repo CLI flag.
3. Rely on the default fallback, which inspects ~/.lotus and extracts the
API endpoint string if the location is a Lotus repo.
tvx will apply these methods in the same order of precedence they're listed.
`,
Usage: "tvx is a tool for extracting and executing test vectors",
Commands: []*cli.Command{
extractCmd,
execCmd,
extractManyCmd,
},
}
sort.Sort(cli.CommandsByName(app.Commands))
for _, c := range app.Commands {
sort.Sort(cli.FlagsByName(c.Flags))
}
if err := app.Run(os.Args); err != nil {
log.Fatal(err)
}
}

293
cmd/tvx/state.go Normal file
View File

@ -0,0 +1,293 @@
package main
import (
"context"
"fmt"
"io"
"log"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/ipfs/go-cid"
format "github.com/ipfs/go-ipld-format"
"github.com/ipld/go-car"
cbg "github.com/whyrusleeping/cbor-gen"
"github.com/filecoin-project/lotus/api"
init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init"
"github.com/filecoin-project/lotus/chain/state"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
)
// StateSurgeon is an object used to fetch and manipulate state.
type StateSurgeon struct {
ctx context.Context
api api.FullNode
stores *Stores
}
// NewSurgeon returns a state surgeon, an object used to fetch and manipulate
// state.
func NewSurgeon(ctx context.Context, api api.FullNode, stores *Stores) *StateSurgeon {
return &StateSurgeon{
ctx: ctx,
api: api,
stores: stores,
}
}
// GetMaskedStateTree trims the state tree at the supplied tipset to contain
// only the state of the actors in the retain set. It also "dives" into some
// singleton system actors, like the init actor, to trim the state so as to
// compute a minimal state tree. In the future, thid method will dive into
// other system actors like the power actor and the market actor.
func (sg *StateSurgeon) GetMaskedStateTree(previousRoot cid.Cid, retain []address.Address) (cid.Cid, error) {
// TODO: this will need to be parameterized on network version.
st, err := state.LoadStateTree(sg.stores.CBORStore, previousRoot)
if err != nil {
return cid.Undef, err
}
initActor, initState, err := sg.loadInitActor(st)
if err != nil {
return cid.Undef, err
}
err = sg.retainInitEntries(initState, retain)
if err != nil {
return cid.Undef, err
}
err = sg.saveInitActor(initActor, initState, st)
if err != nil {
return cid.Undef, err
}
// resolve all addresses to ID addresses.
resolved, err := sg.resolveAddresses(retain, initState)
if err != nil {
return cid.Undef, err
}
st, err = sg.transplantActors(st, resolved)
if err != nil {
return cid.Undef, err
}
root, err := st.Flush(sg.ctx)
if err != nil {
return cid.Undef, err
}
return root, nil
}
// GetAccessedActors identifies the actors that were accessed during the
// execution of a message.
func (sg *StateSurgeon) GetAccessedActors(ctx context.Context, a api.FullNode, mid cid.Cid) ([]address.Address, error) {
log.Printf("calculating accessed actors during execution of message: %s", mid)
msgInfo, err := a.StateSearchMsg(ctx, mid)
if err != nil {
return nil, err
}
if msgInfo == nil {
return nil, fmt.Errorf("message info is nil")
}
msgObj, err := a.ChainGetMessage(ctx, mid)
if err != nil {
return nil, err
}
ts, err := a.ChainGetTipSet(ctx, msgInfo.TipSet)
if err != nil {
return nil, err
}
trace, err := a.StateCall(ctx, msgObj, ts.Parents())
if err != nil {
return nil, fmt.Errorf("could not replay msg: %w", err)
}
accessed := make(map[address.Address]struct{})
var recur func(trace *types.ExecutionTrace)
recur = func(trace *types.ExecutionTrace) {
accessed[trace.Msg.To] = struct{}{}
accessed[trace.Msg.From] = struct{}{}
for i := range trace.Subcalls {
recur(&trace.Subcalls[i])
}
}
recur(&trace.ExecutionTrace)
ret := make([]address.Address, 0, len(accessed))
for k := range accessed {
ret = append(ret, k)
}
return ret, nil
}
// WriteCAR recursively writes the tree referenced by the root as a CAR into the
// supplied io.Writer.
func (sg *StateSurgeon) WriteCAR(w io.Writer, roots ...cid.Cid) error {
carWalkFn := func(nd format.Node) (out []*format.Link, err error) {
for _, link := range nd.Links() {
if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
continue
}
out = append(out, link)
}
return out, nil
}
return car.WriteCarWithWalker(sg.ctx, sg.stores.DAGService, roots, w, carWalkFn)
}
// WriteCARIncluding writes a CAR including only the CIDs that are listed in
// the include set. This leads to an intentially sparse tree with dangling links.
func (sg *StateSurgeon) WriteCARIncluding(w io.Writer, include map[cid.Cid]struct{}, roots ...cid.Cid) error {
carWalkFn := func(nd format.Node) (out []*format.Link, err error) {
for _, link := range nd.Links() {
if _, ok := include[link.Cid]; !ok {
continue
}
if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
continue
}
out = append(out, link)
}
return out, nil
}
return car.WriteCarWithWalker(sg.ctx, sg.stores.DAGService, roots, w, carWalkFn)
}
// transplantActors plucks the state from the supplied actors at the given
// tipset, and places it into the supplied state map.
func (sg *StateSurgeon) transplantActors(src *state.StateTree, pluck []address.Address) (*state.StateTree, error) {
log.Printf("transplanting actor states: %v", pluck)
dst, err := state.NewStateTree(sg.stores.CBORStore, src.Version())
if err != nil {
return nil, err
}
for _, a := range pluck {
actor, err := src.GetActor(a)
if err != nil {
return nil, fmt.Errorf("get actor %s failed: %w", a, err)
}
err = dst.SetActor(a, actor)
if err != nil {
return nil, err
}
// recursive copy of the actor state.
err = vm.Copy(context.TODO(), sg.stores.Blockstore, sg.stores.Blockstore, actor.Head)
if err != nil {
return nil, err
}
actorState, err := sg.api.ChainReadObj(sg.ctx, actor.Head)
if err != nil {
return nil, err
}
cid, err := sg.stores.CBORStore.Put(sg.ctx, &cbg.Deferred{Raw: actorState})
if err != nil {
return nil, err
}
if cid != actor.Head {
panic("mismatched cids")
}
}
return dst, nil
}
// saveInitActor saves the state of the init actor to the provided state map.
func (sg *StateSurgeon) saveInitActor(initActor *types.Actor, initState init_.State, st *state.StateTree) error {
log.Printf("saving init actor into state tree")
// Store the state of the init actor.
cid, err := sg.stores.CBORStore.Put(sg.ctx, initState)
if err != nil {
return err
}
actor := *initActor
actor.Head = cid
err = st.SetActor(init_.Address, &actor)
if err != nil {
return err
}
cid, _ = st.Flush(sg.ctx)
log.Printf("saved init actor into state tree; new root: %s", cid)
return nil
}
// retainInitEntries takes an old init actor state, and retains only the
// entries in the retain set, returning a new init actor state.
func (sg *StateSurgeon) retainInitEntries(state init_.State, retain []address.Address) error {
log.Printf("retaining init actor entries for addresses: %v", retain)
m := make(map[address.Address]struct{}, len(retain))
for _, a := range retain {
m[a] = struct{}{}
}
var remove []address.Address
_ = state.ForEachActor(func(id abi.ActorID, address address.Address) error {
if _, ok := m[address]; !ok {
remove = append(remove, address)
}
return nil
})
err := state.Remove(remove...)
log.Printf("new init actor state: %+v", state)
return err
}
// resolveAddresses resolved the requested addresses from the provided
// InitActor state, returning a slice of length len(orig), where each index
// contains the resolved address.
func (sg *StateSurgeon) resolveAddresses(orig []address.Address, ist init_.State) (ret []address.Address, err error) {
log.Printf("resolving addresses: %v", orig)
ret = make([]address.Address, len(orig))
for i, addr := range orig {
resolved, found, err := ist.ResolveAddress(addr)
if err != nil {
return nil, err
}
if !found {
return nil, fmt.Errorf("address not found: %s", addr)
}
ret[i] = resolved
}
log.Printf("resolved addresses: %v", ret)
return ret, nil
}
// loadInitActor loads the init actor state from a given tipset.
func (sg *StateSurgeon) loadInitActor(st *state.StateTree) (*types.Actor, init_.State, error) {
actor, err := st.GetActor(init_.Address)
if err != nil {
return nil, nil, err
}
initState, err := init_.Load(sg.stores.ADTStore, actor)
if err != nil {
return nil, nil, err
}
log.Printf("loaded init actor state: %+v", initState)
return actor, initState, nil
}

142
cmd/tvx/stores.go Normal file
View File

@ -0,0 +1,142 @@
package main
import (
"context"
"log"
"sync"
"github.com/fatih/color"
dssync "github.com/ipfs/go-datastore/sync"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/lotus/chain/actors/adt"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
ds "github.com/ipfs/go-datastore"
exchange "github.com/ipfs/go-ipfs-exchange-interface"
offline "github.com/ipfs/go-ipfs-exchange-offline"
cbor "github.com/ipfs/go-ipld-cbor"
format "github.com/ipfs/go-ipld-format"
"github.com/ipfs/go-merkledag"
)
// Stores is a collection of the different stores and services that are needed
// to deal with the data layer of Filecoin, conveniently interlinked with one
// another.
type Stores struct {
CBORStore cbor.IpldStore
ADTStore adt.Store
Datastore ds.Batching
Blockstore blockstore.Blockstore
BlockService blockservice.BlockService
Exchange exchange.Interface
DAGService format.DAGService
}
// NewProxyingStores is a set of Stores backed by a proxying Blockstore that
// proxies Get requests for unknown CIDs to a Filecoin node, via the
// ChainReadObj RPC.
func NewProxyingStores(ctx context.Context, api api.FullNode) *Stores {
ds := dssync.MutexWrap(ds.NewMapDatastore())
bs := &proxyingBlockstore{
ctx: ctx,
api: api,
Blockstore: blockstore.NewBlockstore(ds),
}
return NewStores(ctx, ds, bs)
}
// NewStores creates a non-proxying set of Stores.
func NewStores(ctx context.Context, ds ds.Batching, bs blockstore.Blockstore) *Stores {
var (
cborstore = cbor.NewCborStore(bs)
offl = offline.Exchange(bs)
blkserv = blockservice.New(bs, offl)
dserv = merkledag.NewDAGService(blkserv)
)
return &Stores{
CBORStore: cborstore,
ADTStore: adt.WrapStore(ctx, cborstore),
Datastore: ds,
Blockstore: bs,
Exchange: offl,
BlockService: blkserv,
DAGService: dserv,
}
}
// TracingBlockstore is a Blockstore trait that records CIDs that were accessed
// through Get.
type TracingBlockstore interface {
// StartTracing starts tracing CIDs accessed through the this Blockstore.
StartTracing()
// FinishTracing finishes tracing accessed CIDs, and returns a map of the
// CIDs that were traced.
FinishTracing() map[cid.Cid]struct{}
}
// proxyingBlockstore is a Blockstore wrapper that fetches unknown CIDs from
// a Filecoin node via JSON-RPC.
type proxyingBlockstore struct {
ctx context.Context
api api.FullNode
lk sync.RWMutex
tracing bool
traced map[cid.Cid]struct{}
blockstore.Blockstore
}
var _ TracingBlockstore = (*proxyingBlockstore)(nil)
func (pb *proxyingBlockstore) StartTracing() {
pb.lk.Lock()
pb.tracing = true
pb.traced = map[cid.Cid]struct{}{}
pb.lk.Unlock()
}
func (pb *proxyingBlockstore) FinishTracing() map[cid.Cid]struct{} {
pb.lk.Lock()
ret := pb.traced
pb.tracing = false
pb.traced = map[cid.Cid]struct{}{}
pb.lk.Unlock()
return ret
}
func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
pb.lk.RLock()
if pb.tracing {
pb.traced[cid] = struct{}{}
}
pb.lk.RUnlock()
if block, err := pb.Blockstore.Get(cid); err == nil {
return block, err
}
log.Println(color.CyanString("fetching cid via rpc: %v", cid))
item, err := pb.api.ChainReadObj(pb.ctx, cid)
if err != nil {
return nil, err
}
block, err := blocks.NewBlockWithCid(item, cid)
if err != nil {
return nil, err
}
err = pb.Blockstore.Put(block)
if err != nil {
return nil, err
}
return block, nil
}

133
conformance/corpus_test.go Normal file
View File

@ -0,0 +1,133 @@
package conformance
import (
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"github.com/filecoin-project/test-vectors/schema"
)
const (
// EnvSkipConformance, if 1, skips the conformance test suite.
EnvSkipConformance = "SKIP_CONFORMANCE"
// EnvCorpusRootDir is the name of the environment variable where the path
// to an alternative corpus location can be provided.
//
// The default is defaultCorpusRoot.
EnvCorpusRootDir = "CORPUS_DIR"
// defaultCorpusRoot is the directory where the test vector corpus is hosted.
// It is mounted on the Lotus repo as a git submodule.
//
// When running this test, the corpus root can be overridden through the
// -conformance.corpus CLI flag to run an alternate corpus.
defaultCorpusRoot = "../extern/test-vectors/corpus"
)
// ignore is a set of paths relative to root to skip.
var ignore = map[string]struct{}{
".git": {},
"schema.json": {},
}
// TestConformance is the entrypoint test that runs all test vectors found
// in the corpus root directory.
//
// It locates all json files via a recursive walk, skipping over the ignore set,
// as well as files beginning with _. It parses each file as a test vector, and
// runs it via the Driver.
func TestConformance(t *testing.T) {
if skip := strings.TrimSpace(os.Getenv(EnvSkipConformance)); skip == "1" {
t.SkipNow()
}
// corpusRoot is the effective corpus root path, taken from the `-conformance.corpus` CLI flag,
// falling back to defaultCorpusRoot if not provided.
corpusRoot := defaultCorpusRoot
if dir := strings.TrimSpace(os.Getenv(EnvCorpusRootDir)); dir != "" {
corpusRoot = dir
}
var vectors []string
err := filepath.Walk(corpusRoot+"/", func(path string, info os.FileInfo, err error) error {
if err != nil {
t.Fatal(err)
}
filename := filepath.Base(path)
rel, err := filepath.Rel(corpusRoot, path)
if err != nil {
t.Fatal(err)
}
if _, ok := ignore[rel]; ok {
// skip over using the right error.
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
if info.IsDir() {
// dive into directories.
return nil
}
if filepath.Ext(path) != ".json" {
// skip if not .json.
return nil
}
if ignored := strings.HasPrefix(filename, "_"); ignored {
// ignore files starting with _.
t.Logf("ignoring: %s", rel)
return nil
}
vectors = append(vectors, rel)
return nil
})
if err != nil {
t.Fatal(err)
}
if len(vectors) == 0 {
t.Fatalf("no test vectors found")
}
// Run a test for each vector.
for _, v := range vectors {
path := filepath.Join(corpusRoot, v)
raw, err := ioutil.ReadFile(path)
if err != nil {
t.Fatalf("failed to read test raw file: %s", path)
}
var vector schema.TestVector
err = json.Unmarshal(raw, &vector)
if err != nil {
t.Errorf("failed to parse test vector %s: %s; skipping", path, err)
continue
}
t.Run(v, func(t *testing.T) {
for _, h := range vector.Hints {
if h == schema.HintIncorrect {
t.Logf("skipping vector marked as incorrect: %s", vector.Meta.ID)
t.SkipNow()
}
}
// dispatch the execution depending on the vector class.
switch vector.Class {
case "message":
ExecuteMessageVector(t, &vector)
case "tipset":
ExecuteTipsetVector(t, &vector)
default:
t.Fatalf("test vector class not supported: %s", vector.Class)
}
})
}
}

93
conformance/driver.go
View File

@ -2,9 +2,9 @@ package conformance
import (
"context"
"os"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/lotus/chain/state"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
@ -14,6 +14,7 @@ import (
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/test-vectors/schema"
@ -24,18 +25,36 @@ import (
)
var (
// BaseFee to use in the VM.
// TODO make parametrisable through vector.
BaseFee = abi.NewTokenAmount(100)
// DefaultCirculatingSupply is the fallback circulating supply returned by
// the driver's CircSupplyCalculator function, used if the vector specifies
// no circulating supply.
DefaultCirculatingSupply = types.TotalFilecoinInt
// DefaultBaseFee to use in the VM, if one is not supplied in the vector.
DefaultBaseFee = abi.NewTokenAmount(100)
)
type Driver struct {
ctx context.Context
selector schema.Selector
vmFlush bool
}
func NewDriver(ctx context.Context, selector schema.Selector) *Driver {
return &Driver{ctx: ctx, selector: selector}
type DriverOpts struct {
// DisableVMFlush, when true, avoids calling VM.Flush(), forces a blockstore
// recursive copy, from the temporary buffer blockstore, to the real
// system's blockstore. Disabling VM flushing is useful when extracting test
// vectors and trimming state, as we don't want to force an accidental
// deep copy of the state tree.
//
// Disabling VM flushing almost always should go hand-in-hand with
// LOTUS_DISABLE_VM_BUF=iknowitsabadidea. That way, state tree writes are
// immediately committed to the blockstore.
DisableVMFlush bool
}
func NewDriver(ctx context.Context, selector schema.Selector, opts DriverOpts) *Driver {
return &Driver{ctx: ctx, selector: selector, vmFlush: !opts.DisableVMFlush}
}
type ExecuteTipsetResult struct {
@ -120,19 +139,47 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot
return ret, nil
}
type ExecuteMessageParams struct {
Preroot cid.Cid
Epoch abi.ChainEpoch
Message *types.Message
CircSupply *abi.TokenAmount
BaseFee *abi.TokenAmount
}
// ExecuteMessage executes a conformance test vector message in a temporary VM.
func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch abi.ChainEpoch, msg *types.Message) (*vm.ApplyRet, cid.Cid, error) {
// dummy state manager; only to reference the GetNetworkVersion method, which does not depend on state.
func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, params ExecuteMessageParams) (*vm.ApplyRet, cid.Cid, error) {
if !d.vmFlush {
// do not flush the VM, just the state tree; this should be used with
// LOTUS_DISABLE_VM_BUF enabled, so writes will anyway be visible.
_ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea")
}
basefee := DefaultBaseFee
if params.BaseFee != nil {
basefee = *params.BaseFee
}
circSupply := DefaultCirculatingSupply
if params.CircSupply != nil {
circSupply = *params.CircSupply
}
// dummy state manager; only to reference the GetNetworkVersion method,
// which does not depend on state.
sm := new(stmgr.StateManager)
vmOpts := &vm.VMOpts{
StateBase: preroot,
Epoch: epoch,
Rand: &testRand{}, // TODO always succeeds; need more flexibility.
Bstore: bs,
Syscalls: mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility.
CircSupplyCalc: nil,
BaseFee: BaseFee,
NtwkVersion: sm.GetNtwkVersion,
StateBase: params.Preroot,
Epoch: params.Epoch,
Rand: &testRand{}, // TODO always succeeds; need more flexibility.
Bstore: bs,
Syscalls: mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility.
CircSupplyCalc: func(_ context.Context, _ abi.ChainEpoch, _ *state.StateTree) (abi.TokenAmount, error) {
return circSupply, nil
},
BaseFee: basefee,
NtwkVersion: sm.GetNtwkVersion,
}
lvm, err := vm.NewVM(context.TODO(), vmOpts)
@ -149,12 +196,20 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch
lvm.SetInvoker(invoker)
ret, err := lvm.ApplyMessage(d.ctx, toChainMsg(msg))
ret, err := lvm.ApplyMessage(d.ctx, toChainMsg(params.Message))
if err != nil {
return nil, cid.Undef, err
}
root, err := lvm.Flush(d.ctx)
var root cid.Cid
if d.vmFlush {
// flush the VM, committing the state tree changes and forcing a
// recursive copoy from the temporary blcokstore to the real blockstore.
root, err = lvm.Flush(d.ctx)
} else {
root, err = lvm.StateTree().(*state.StateTree).Flush(d.ctx)
}
return ret, root, err
}

62
conformance/reporter.go Normal file
View File

@ -0,0 +1,62 @@
package conformance
import (
"log"
"os"
"sync/atomic"
"testing"
"github.com/fatih/color"
)
// Reporter is a contains a subset of the testing.T methods, so that the
// Execute* functions in this package can be used inside or outside of
// go test runs.
type Reporter interface {
Helper()
Log(args ...interface{})
Errorf(format string, args ...interface{})
Fatalf(format string, args ...interface{})
Logf(format string, args ...interface{})
FailNow()
Failed() bool
}
var _ Reporter = (*testing.T)(nil)
// LogReporter wires the Reporter methods to the log package. It is appropriate
// to use when calling the Execute* functions from a standalone CLI program.
type LogReporter struct {
failed int32
}
var _ Reporter = (*LogReporter)(nil)
func (*LogReporter) Helper() {}
func (*LogReporter) Log(args ...interface{}) {
log.Println(args...)
}
func (*LogReporter) Logf(format string, args ...interface{}) {
log.Printf(format, args...)
}
func (*LogReporter) FailNow() {
os.Exit(1)
}
func (l *LogReporter) Failed() bool {
return atomic.LoadInt32(&l.failed) == 1
}
func (l *LogReporter) Errorf(format string, args ...interface{}) {
atomic.StoreInt32(&l.failed, 1)
log.Println(color.HiRedString("❌ "+format, args...))
}
func (l *LogReporter) Fatalf(format string, args ...interface{}) {
atomic.StoreInt32(&l.failed, 1)
log.Fatal(color.HiRedString("❌ "+format, args...))
}

272
conformance/runner.go Normal file
View File

@ -0,0 +1,272 @@
package conformance
import (
"bytes"
"compress/gzip"
"context"
"encoding/base64"
"fmt"
"io/ioutil"
"os"
"os/exec"
"strconv"
"github.com/fatih/color"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/exitcode"
"github.com/filecoin-project/test-vectors/schema"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
ds "github.com/ipfs/go-datastore"
offline "github.com/ipfs/go-ipfs-exchange-offline"
format "github.com/ipfs/go-ipld-format"
"github.com/ipfs/go-merkledag"
"github.com/ipld/go-car"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/lib/blockstore"
)
// ExecuteMessageVector executes a message-class test vector.
func ExecuteMessageVector(r Reporter, vector *schema.TestVector) {
var (
ctx = context.Background()
epoch = vector.Pre.Epoch
root = vector.Pre.StateTree.RootCID
)
// Load the CAR into a new temporary Blockstore.
bs, err := LoadVectorCAR(vector.CAR)
if err != nil {
r.Fatalf("failed to load the vector CAR: %w", err)
}
// Create a new Driver.
driver := NewDriver(ctx, vector.Selector, DriverOpts{DisableVMFlush: true})
var circSupply *abi.TokenAmount
if cs := vector.Pre.CircSupply; cs != nil {
ta := big.NewFromGo(cs)
circSupply = &ta
}
var basefee *abi.TokenAmount
if bf := vector.Pre.BaseFee; bf != nil {
ta := big.NewFromGo(bf)
basefee = &ta
}
// Apply every message.
for i, m := range vector.ApplyMessages {
msg, err := types.DecodeMessage(m.Bytes)
if err != nil {
r.Fatalf("failed to deserialize message: %s", err)
}
// add an epoch if one's set.
if m.Epoch != nil {
epoch = *m.Epoch
}
// Execute the message.
var ret *vm.ApplyRet
ret, root, err = driver.ExecuteMessage(bs, ExecuteMessageParams{
Preroot: root,
Epoch: abi.ChainEpoch(epoch),
Message: msg,
CircSupply: circSupply,
BaseFee: basefee,
})
if err != nil {
r.Fatalf("fatal failure when executing message: %s", err)
}
// Assert that the receipt matches what the test vector expects.
AssertMsgResult(r, vector.Post.Receipts[i], ret, strconv.Itoa(i))
}
// Once all messages are applied, assert that the final state root matches
// the expected postcondition root.
if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual {
r.Errorf("wrong post root cid; expected %v, but got %v", expected, actual)
dumpThreeWayStateDiff(r, vector, bs, root)
r.FailNow()
}
}
// ExecuteTipsetVector executes a tipset-class test vector.
func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) {
var (
ctx = context.Background()
prevEpoch = vector.Pre.Epoch
root = vector.Pre.StateTree.RootCID
tmpds = ds.NewMapDatastore()
)
// Load the vector CAR into a new temporary Blockstore.
bs, err := LoadVectorCAR(vector.CAR)
if err != nil {
r.Fatalf("failed to load the vector CAR: %w", err)
}
// Create a new Driver.
driver := NewDriver(ctx, vector.Selector, DriverOpts{})
// Apply every tipset.
var receiptsIdx int
for i, ts := range vector.ApplyTipsets {
ts := ts // capture
ret, err := driver.ExecuteTipset(bs, tmpds, root, abi.ChainEpoch(prevEpoch), &ts)
if err != nil {
r.Fatalf("failed to apply tipset %d message: %s", i, err)
}
for j, v := range ret.AppliedResults {
AssertMsgResult(r, vector.Post.Receipts[receiptsIdx], v, fmt.Sprintf("%d of tipset %d", j, i))
receiptsIdx++
}
// Compare the receipts root.
if expected, actual := vector.Post.ReceiptsRoots[i], ret.ReceiptsRoot; expected != actual {
r.Errorf("post receipts root doesn't match; expected: %s, was: %s", expected, actual)
}
prevEpoch = ts.Epoch
root = ret.PostStateRoot
}
// Once all messages are applied, assert that the final state root matches
// the expected postcondition root.
if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual {
r.Errorf("wrong post root cid; expected %v, but got %v", expected, actual)
dumpThreeWayStateDiff(r, vector, bs, root)
r.FailNow()
}
}
// AssertMsgResult compares a message result. It takes the expected receipt
// encoded in the vector, the actual receipt returned by Lotus, and a message
// label to log in the assertion failure message to facilitate debugging.
func AssertMsgResult(r Reporter, expected *schema.Receipt, actual *vm.ApplyRet, label string) {
r.Helper()
if expected, actual := exitcode.ExitCode(expected.ExitCode), actual.ExitCode; expected != actual {
r.Errorf("exit code of msg %s did not match; expected: %s, got: %s", label, expected, actual)
}
if expected, actual := expected.GasUsed, actual.GasUsed; expected != actual {
r.Errorf("gas used of msg %s did not match; expected: %d, got: %d", label, expected, actual)
}
if expected, actual := []byte(expected.ReturnValue), actual.Return; !bytes.Equal(expected, actual) {
r.Errorf("return value of msg %s did not match; expected: %s, got: %s", label, base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(actual))
}
}
func dumpThreeWayStateDiff(r Reporter, vector *schema.TestVector, bs blockstore.Blockstore, actual cid.Cid) {
// check if statediff exists; if not, skip.
if err := exec.Command("statediff", "--help").Run(); err != nil {
r.Log("could not dump 3-way state tree diff upon test failure: statediff command not found")
r.Log("install statediff with:")
r.Log("$ git clone https://github.com/filecoin-project/statediff.git")
r.Log("$ cd statediff")
r.Log("$ go generate ./...")
r.Log("$ go install ./cmd/statediff")
return
}
tmpCar, err := writeStateToTempCAR(bs,
vector.Pre.StateTree.RootCID,
vector.Post.StateTree.RootCID,
actual,
)
if err != nil {
r.Fatalf("failed to write temporary state CAR: %s", err)
}
defer os.RemoveAll(tmpCar) //nolint:errcheck
color.NoColor = false // enable colouring.
var (
a = color.New(color.FgMagenta, color.Bold).Sprint("(A) expected final state")
b = color.New(color.FgYellow, color.Bold).Sprint("(B) actual final state")
c = color.New(color.FgCyan, color.Bold).Sprint("(C) initial state")
d1 = color.New(color.FgGreen, color.Bold).Sprint("[Δ1]")
d2 = color.New(color.FgGreen, color.Bold).Sprint("[Δ2]")
d3 = color.New(color.FgGreen, color.Bold).Sprint("[Δ3]")
)
printDiff := func(left, right cid.Cid) {
cmd := exec.Command("statediff", "car", "--file", tmpCar, left.String(), right.String())
b, err := cmd.CombinedOutput()
if err != nil {
r.Fatalf("statediff failed: %s", err)
}
r.Log(string(b))
}
bold := color.New(color.Bold).SprintfFunc()
// run state diffs.
r.Log(bold("=== dumping 3-way diffs between %s, %s, %s ===", a, b, c))
r.Log(bold("--- %s left: %s; right: %s ---", d1, a, b))
printDiff(vector.Post.StateTree.RootCID, actual)
r.Log(bold("--- %s left: %s; right: %s ---", d2, c, b))
printDiff(vector.Pre.StateTree.RootCID, actual)
r.Log(bold("--- %s left: %s; right: %s ---", d3, c, a))
printDiff(vector.Pre.StateTree.RootCID, vector.Post.StateTree.RootCID)
}
// writeStateToTempCAR writes the provided roots to a temporary CAR that'll be
// cleaned up via t.Cleanup(). It returns the full path of the temp file.
func writeStateToTempCAR(bs blockstore.Blockstore, roots ...cid.Cid) (string, error) {
tmp, err := ioutil.TempFile("", "lotus-tests-*.car")
if err != nil {
return "", fmt.Errorf("failed to create temp file to dump CAR for diffing: %w", err)
}
carWalkFn := func(nd format.Node) (out []*format.Link, err error) {
for _, link := range nd.Links() {
if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
continue
}
out = append(out, link)
}
return out, nil
}
var (
offl = offline.Exchange(bs)
blkserv = blockservice.New(bs, offl)
dserv = merkledag.NewDAGService(blkserv)
)
err = car.WriteCarWithWalker(context.Background(), dserv, roots, tmp, carWalkFn)
if err != nil {
return "", fmt.Errorf("failed to dump CAR for diffing: %w", err)
}
_ = tmp.Close()
return tmp.Name(), nil
}
func LoadVectorCAR(vectorCAR schema.Base64EncodedBytes) (blockstore.Blockstore, error) {
bs := blockstore.NewTemporary()
// Read the base64-encoded CAR from the vector, and inflate the gzip.
buf := bytes.NewReader(vectorCAR)
r, err := gzip.NewReader(buf)
if err != nil {
return nil, fmt.Errorf("failed to inflate gzipped CAR: %s", err)
}
defer r.Close() // nolint
// Load the CAR embedded in the test vector into the Blockstore.
_, err = car.LoadCar(bs, r)
if err != nil {
return nil, fmt.Errorf("failed to load state tree car from test vector: %s", err)
}
return bs, nil
}

376
conformance/runner_test.go
View File

@ -1,376 +0,0 @@
package conformance
import (
"bytes"
"compress/gzip"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"testing"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/exitcode"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
ds "github.com/ipfs/go-datastore"
offline "github.com/ipfs/go-ipfs-exchange-offline"
format "github.com/ipfs/go-ipld-format"
"github.com/ipfs/go-merkledag"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/test-vectors/schema"
"github.com/fatih/color"
"github.com/ipld/go-car"
)
const (
// EnvSkipConformance, if 1, skips the conformance test suite.
EnvSkipConformance = "SKIP_CONFORMANCE"
// EnvCorpusRootDir is the name of the environment variable where the path
// to an alternative corpus location can be provided.
//
// The default is defaultCorpusRoot.
EnvCorpusRootDir = "CORPUS_DIR"
// defaultCorpusRoot is the directory where the test vector corpus is hosted.
// It is mounted on the Lotus repo as a git submodule.
//
// When running this test, the corpus root can be overridden through the
// -conformance.corpus CLI flag to run an alternate corpus.
defaultCorpusRoot = "../extern/test-vectors/corpus"
)
// ignore is a set of paths relative to root to skip.
var ignore = map[string]struct{}{
".git": {},
"schema.json": {},
}
// TestConformance is the entrypoint test that runs all test vectors found
// in the corpus root directory.
//
// It locates all json files via a recursive walk, skipping over the ignore set,
// as well as files beginning with _. It parses each file as a test vector, and
// runs it via the Driver.
func TestConformance(t *testing.T) {
if skip := strings.TrimSpace(os.Getenv(EnvSkipConformance)); skip == "1" {
t.SkipNow()
}
// corpusRoot is the effective corpus root path, taken from the `-conformance.corpus` CLI flag,
// falling back to defaultCorpusRoot if not provided.
corpusRoot := defaultCorpusRoot
if dir := strings.TrimSpace(os.Getenv(EnvCorpusRootDir)); dir != "" {
corpusRoot = dir
}
var vectors []string
err := filepath.Walk(corpusRoot+"/", func(path string, info os.FileInfo, err error) error {
if err != nil {
t.Fatal(err)
}
filename := filepath.Base(path)
rel, err := filepath.Rel(corpusRoot, path)
if err != nil {
t.Fatal(err)
}
if _, ok := ignore[rel]; ok {
// skip over using the right error.
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
if info.IsDir() {
// dive into directories.
return nil
}
if filepath.Ext(path) != ".json" {
// skip if not .json.
return nil
}
if ignored := strings.HasPrefix(filename, "_"); ignored {
// ignore files starting with _.
t.Logf("ignoring: %s", rel)
return nil
}
vectors = append(vectors, rel)
return nil
})
if err != nil {
t.Fatal(err)
}
if len(vectors) == 0 {
t.Fatalf("no test vectors found")
}
// Run a test for each vector.
for _, v := range vectors {
path := filepath.Join(corpusRoot, v)
raw, err := ioutil.ReadFile(path)
if err != nil {
t.Fatalf("failed to read test raw file: %s", path)
}
var vector schema.TestVector
err = json.Unmarshal(raw, &vector)
if err != nil {
t.Errorf("failed to parse test vector %s: %s; skipping", path, err)
continue
}
t.Run(v, func(t *testing.T) {
for _, h := range vector.Hints {
if h == schema.HintIncorrect {
t.Logf("skipping vector marked as incorrect: %s", vector.Meta.ID)
t.SkipNow()
}
}
// dispatch the execution depending on the vector class.
switch vector.Class {
case "message":
executeMessageVector(t, &vector)
case "tipset":
executeTipsetVector(t, &vector)
default:
t.Fatalf("test vector class not supported: %s", vector.Class)
}
})
}
}
// executeMessageVector executes a message-class test vector.
func executeMessageVector(t *testing.T, vector *schema.TestVector) {
var (
ctx = context.Background()
epoch = vector.Pre.Epoch
root = vector.Pre.StateTree.RootCID
)
// Load the CAR into a new temporary Blockstore.
bs := loadCAR(t, vector.CAR)
// Create a new Driver.
driver := NewDriver(ctx, vector.Selector)
// Apply every message.
for i, m := range vector.ApplyMessages {
msg, err := types.DecodeMessage(m.Bytes)
if err != nil {
t.Fatalf("failed to deserialize message: %s", err)
}
// add an epoch if one's set.
if m.Epoch != nil {
epoch = *m.Epoch
}
// Execute the message.
var ret *vm.ApplyRet
ret, root, err = driver.ExecuteMessage(bs, root, abi.ChainEpoch(epoch), msg)
if err != nil {
t.Fatalf("fatal failure when executing message: %s", err)
}
// Assert that the receipt matches what the test vector expects.
assertMsgResult(t, vector.Post.Receipts[i], ret, strconv.Itoa(i))
}
// Once all messages are applied, assert that the final state root matches
// the expected postcondition root.
if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual {
t.Logf("actual state root CID doesn't match expected one; expected: %s, actual: %s", expected, actual)
dumpThreeWayStateDiff(t, vector, bs, root)
t.FailNow()
}
}
// executeTipsetVector executes a tipset-class test vector.
func executeTipsetVector(t *testing.T, vector *schema.TestVector) {
var (
ctx = context.Background()
prevEpoch = vector.Pre.Epoch
root = vector.Pre.StateTree.RootCID
tmpds = ds.NewMapDatastore()
)
// Load the CAR into a new temporary Blockstore.
bs := loadCAR(t, vector.CAR)
// Create a new Driver.
driver := NewDriver(ctx, vector.Selector)
// Apply every tipset.
var receiptsIdx int
for i, ts := range vector.ApplyTipsets {
ts := ts // capture
ret, err := driver.ExecuteTipset(bs, tmpds, root, abi.ChainEpoch(prevEpoch), &ts)
if err != nil {
t.Fatalf("failed to apply tipset %d message: %s", i, err)
}
for j, v := range ret.AppliedResults {
assertMsgResult(t, vector.Post.Receipts[receiptsIdx], v, fmt.Sprintf("%d of tipset %d", j, i))
receiptsIdx++
}
// Compare the receipts root.
if expected, actual := vector.Post.ReceiptsRoots[i], ret.ReceiptsRoot; expected != actual {
t.Errorf("post receipts root doesn't match; expected: %s, was: %s", expected, actual)
}
prevEpoch = ts.Epoch
root = ret.PostStateRoot
}
// Once all messages are applied, assert that the final state root matches
// the expected postcondition root.
if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual {
t.Logf("actual state root CID doesn't match expected one; expected: %s, actual: %s", expected, actual)
dumpThreeWayStateDiff(t, vector, bs, root)
t.FailNow()
}
}
// assertMsgResult compares a message result. It takes the expected receipt
// encoded in the vector, the actual receipt returned by Lotus, and a message
// label to log in the assertion failure message to facilitate debugging.
func assertMsgResult(t *testing.T, expected *schema.Receipt, actual *vm.ApplyRet, label string) {
t.Helper()
if expected, actual := exitcode.ExitCode(expected.ExitCode), actual.ExitCode; expected != actual {
t.Errorf("exit code of msg %s did not match; expected: %s, got: %s", label, expected, actual)
}
if expected, actual := expected.GasUsed, actual.GasUsed; expected != actual {
t.Errorf("gas used of msg %s did not match; expected: %d, got: %d", label, expected, actual)
}
if expected, actual := []byte(expected.ReturnValue), actual.Return; !bytes.Equal(expected, actual) {
t.Errorf("return value of msg %s did not match; expected: %s, got: %s", label, base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(actual))
}
}
func dumpThreeWayStateDiff(t *testing.T, vector *schema.TestVector, bs blockstore.Blockstore, actual cid.Cid) {
// check if statediff exists; if not, skip.
if err := exec.Command("statediff", "--help").Run(); err != nil {
t.Log("could not dump 3-way state tree diff upon test failure: statediff command not found")
t.Log("install statediff with:")
t.Log("$ git clone https://github.com/filecoin-project/statediff.git")
t.Log("$ cd statediff")
t.Log("$ go generate ./...")
t.Log("$ go install ./cmd/statediff")
return
}
tmpCar := writeStateToTempCAR(t, bs,
vector.Pre.StateTree.RootCID,
vector.Post.StateTree.RootCID,
actual,
)
color.NoColor = false // enable colouring.
t.Errorf("wrong post root cid; expected %v, but got %v", vector.Post.StateTree.RootCID, actual)
var (
a = color.New(color.FgMagenta, color.Bold).Sprint("(A) expected final state")
b = color.New(color.FgYellow, color.Bold).Sprint("(B) actual final state")
c = color.New(color.FgCyan, color.Bold).Sprint("(C) initial state")
d1 = color.New(color.FgGreen, color.Bold).Sprint("[Δ1]")
d2 = color.New(color.FgGreen, color.Bold).Sprint("[Δ2]")
d3 = color.New(color.FgGreen, color.Bold).Sprint("[Δ3]")
)
printDiff := func(left, right cid.Cid) {
cmd := exec.Command("statediff", "car", "--file", tmpCar, left.String(), right.String())
b, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("statediff failed: %s", err)
}
t.Log(string(b))
}
bold := color.New(color.Bold).SprintfFunc()
// run state diffs.
t.Log(bold("=== dumping 3-way diffs between %s, %s, %s ===", a, b, c))
t.Log(bold("--- %s left: %s; right: %s ---", d1, a, b))
printDiff(vector.Post.StateTree.RootCID, actual)
t.Log(bold("--- %s left: %s; right: %s ---", d2, c, b))
printDiff(vector.Pre.StateTree.RootCID, actual)
t.Log(bold("--- %s left: %s; right: %s ---", d3, c, a))
printDiff(vector.Pre.StateTree.RootCID, vector.Post.StateTree.RootCID)
}
// writeStateToTempCAR writes the provided roots to a temporary CAR that'll be
// cleaned up via t.Cleanup(). It returns the full path of the temp file.
func writeStateToTempCAR(t *testing.T, bs blockstore.Blockstore, roots ...cid.Cid) string {
tmp, err := ioutil.TempFile("", "lotus-tests-*.car")
if err != nil {
t.Fatalf("failed to create temp file to dump CAR for diffing: %s", err)
}
// register a cleanup function to delete the CAR.
t.Cleanup(func() {
_ = os.Remove(tmp.Name())
})
carWalkFn := func(nd format.Node) (out []*format.Link, err error) {
for _, link := range nd.Links() {
if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
continue
}
out = append(out, link)
}
return out, nil
}
var (
offl = offline.Exchange(bs)
blkserv = blockservice.New(bs, offl)
dserv = merkledag.NewDAGService(blkserv)
)
err = car.WriteCarWithWalker(context.Background(), dserv, roots, tmp, carWalkFn)
if err != nil {
t.Fatalf("failed to dump CAR for diffing: %s", err)
}
_ = tmp.Close()
return tmp.Name()
}
func loadCAR(t *testing.T, vectorCAR schema.Base64EncodedBytes) blockstore.Blockstore {
bs := blockstore.NewTemporary()
// Read the base64-encoded CAR from the vector, and inflate the gzip.
buf := bytes.NewReader(vectorCAR)
r, err := gzip.NewReader(buf)
if err != nil {
t.Fatalf("failed to inflate gzipped CAR: %s", err)
}
defer r.Close() // nolint
// Load the CAR embedded in the test vector into the Blockstore.
_, err = car.LoadCar(bs, r)
if err != nil {
t.Fatalf("failed to load state tree car from test vector: %s", err)
}
return bs
}

147
documentation/en/.glossary.json
View File

@ -1,146 +1 @@
{
"bellman": {
"title": "Bellman",
"value": "Bellman is a rust crate for building zk-SNARK circuits. It provides circuit traits and primitive structures, as well as basic gadget implementations such as booleans and number abstractions."
},
"nvme": {
"title": "NVMe",
"value": "(non-volatile memory express) is a host controller interface and storage protocol created to accelerate the transfer of data between enterprise and client systems and solid-state drives (SSDs) over a computer's high-speed Peripheral Component Interconnect Express (PCIe) bus."
},
"multiaddr": {
"title": "Multiaddr",
"value": "Multiaddr is a format for encoding addresses from various well-established network protocols. It is useful to write applications that future-proof their use of addresses, and allow multiple transport protocols and addresses to coexist."
},
"attofil": {
"title": "attoFIL",
"value": "AttoFIL is a word used to describe 10^-18 FIL. The word atto comes from the Norwegian and Danish term: atten eighteen."
},
"fil": {
"title": "FIL",
"value": "A ticker symbol is an abbreviation used to uniquely identify Filecoin when it is used in a wallet exchange or a cryptocurrency exchange."
},
"epost": {
"title": "Election Proof-of-Spacetime",
"value": "Election Proof-of-Spacetime couples the Proof-of-Spacetime process with block production, meaning that in order to produce a block, the miner must produce a valid Proof-of-Spacetime proof (snark output)."
},
"jwt": {
"title": "JWT",
"value": "JSON Web Tokens are an open, industry standard RFC 7519 method for representing claims securely between two parties."
},
"json-rpc": {
"title": "JSON-RPC",
"value": "JSON-RPC is a remote procedure call protocol encoded in JSON. It is a very simple protocol (and very similar to XML-RPC), defining only a few data types and commands."
},
"bls-address": {
"title": "BLS Signature (Address)",
"value": "A Boneh–Lynn–Shacham (BLS) signature is a digital signature scheme that allows a user to determine the authenticity of a signer, and is a commonly used signature scheme in the Filecoin Distributed Storage Network."
},
"faucet": {
"title": "Filecoin Test Faucet",
"value": "A webpage where you can get free test Filecoin to participate in the Testnet."
},
"chain": {
"title": "Chain",
"value": "The Filecoin Blockchain is a distributed virtual machine that achieves consensus, processes messages, accounts for storage, and maintains security in the Filecoin Protocol. It is the main interface linking various actors in the Filecoin system."
},
"miner-power": {
"title": "Miner Power",
"value": "Miner storage in relation to network storage, tracked in the power table."
},
"sector": {
"title": "Sector",
"value": "A fixed-size block of data of SECTOR_SIZE bytes which generally contains client's data."
},
"sealing": {
"title": "Sealing",
"value": "A slow encoding process that returns commitments and proofs for data being stored in a sector."
},
"seal": {
"title": "Seal",
"value": "A slow encoding process that returns commitments and proofs for data being stored in a sector."
},
"posts": {
"title": "Proof-of-Spacetime(s)",
"value": "Filecoin is a protocol token whose blockchain runs on a novel proof, called Proof-of-Spacetime, where blocks are created by miners that are storing data."
},
"filecoin-testnet": {
"title": "Filecoin Testnet",
"value": "Until we launch, we are making lots of changes to Lotus. The Testnet is expected to bring a few significant fixes/improvements. During Testnet, you can retrieve test filecoin from our network faucet to use as collateral to start mining. Test filecoin do not have any value – the official filecoin tokens will not be released until Mainnet launch."
},
"filecoin-decentralized-storage-market": {
"title": "Filecoin Decentralized Storage Market",
"value": "Storage Market subsystem is the data entry point into the network. Miners only earn power from data stored in a storage deal and all deals live on the Filecoin network."
},
"filecoin-proof-parameters": {
"title": "Filecoin Proof Parameters",
"value": "The proving algorithms rely on a large binary parameter file."
},
"lotus-devnet": {
"title": "DevNet",
"value": "On the DevNets, you can store data as a storage client and also try how Filecoin mining works. The devnets are an important development tool for those who anticipate building applications on top of the Filecoin protocol or storing data on the decentralized storage market. "
},
"filecoin-distributed-storage-network": {
"title": "Filecoin Distributed Storage Network",
"value": "Filecoin is a distributed storage network based on a blockchain mechanism. Filecoin miners can elect to provide storage capacity for the network, and thereby earn units of the Filecoin cryptocurrency (FIL) by periodically producing cryptographic proofs that certify that they are providing the capacity specified."
},
"lotus-node": {
"title": "Lotus Node",
"value": "The Lotus Node is full of capabilities. It runs the Blockchain system, makes retrieval deals, does data transfer, supports block producer logic, and syncs and validates the chain."
},
"block-rewards": {
"title": "Block Reward",
"value": "Over the entire lifetime of the protocol, 1,400,000,000 FIL (TotalIssuance) will be given out to miners. The rate at which the funds are given out is set to halve every six years, smoothly (not a fixed jump like in Bitcoin)."
},
"block-producer-miner": {
"title": "Miner (Block Producer)",
"value": "The Block Producer Miner's logic. It currently shares an interface and process with the Lotus Node. A Block Producer chooses which messages to include in a block and is rewarded according to each message’s gas price and consumption, forming a market."
},
"lotus-miner": {
"title": "Miner (lotus-miner)",
"value": "The Miner's logic. It has its own dedicated process. Contributes to the network through Sector commitments and Proofs of Spacetime to prove that it is storing the sectors it has commited to."
},
"swarm-port": {
"title": "Swarm Port (Libp2p)",
"value": "The LibP2P Swarm manages groups of connections to peers, handles incoming and outgoing streams, and is part of the miners implementation. The port value is part of the Host interface."
},
"daemon": {
"title": "Lotus Daemon",
"value": "A Daemon is a program that runs as a background process. A Daemon in the context of the Filecoin Distributed Storage Network may enable applications to communicate with peers, handle protocols, participate in pubsub, and interact with a distributed hash table (DHT)."
},
"storage-deal": {
"title": "Storage deal",
"value": "One of the two types of deals in Filecoin markets. Storage deals are recorded on the blockchain and enforced by the protocol."
},
"retrieval-deal": {
"title": "Retrieval deal",
"value": "One of the two types of deals in Filecoin markets. Retrieval deals are off chain and enabled by micropayment channel by transacting parties."
},
"deal-cid": {
"title": "Deal CID",
"value": "CID is a format for referencing content in distributed information systems, it is a way to store information so it can be retrieved based on its content, not its location. DealCID specifically is used in storage deals."
},
"data-cid": {
"title": "Data CID",
"value": "CID is a format for referencing content in distributed information systems, it is a way to store information so it can be retrieved based on its content, not its location. DataCID specifically is used to represent the file that is stored in the Filecoin Distributed Storage Network."
},
"cid": {
"title": "CID",
"value": "A CID is a self-describing content-addressed identifier. It uses cryptographic hashes to achieve content addressing. It uses several multiformats to achieve flexible self-description, namely multihash for hashes, multicodec for data content types, and multibase to encode the CID itself into strings."
},
"total-network-power": {
"title": "Total Network Power",
"value": "A reference to all the Power Tables for every subchain, accounting for each Lotus Miner on chain."
},
"chain-block-height": {
"title": "Chain Block Height",
"value": "Chain block height is defined as the number of blocks in the chain between any given block and the very first block in the blockchain."
},
"block-height": {
"title": "Block Height",
"value": "Height of the Merkle Tree of a sector. A sector is a contiguous array of bytes that a miner puts together, seals, and performs Proofs of Spacetime on."
},
"blocktime": {
"title": "Blocktime",
"value": "The time it takes for a Block to propagate to the whole network."
}
}
{}

231
documentation/en/.library.json
View File

@ -1,232 +1,25 @@
{
"posts": [
{
"title": "Hardware Requirements",
"slug": "en+hardware",
"github": "en/hardware.md",
"title": "About Lotus",
"slug": "",
"github": "en/about.md",
"value": null,
"posts": [
{
"title": "Testing Configuration",
"slug": "en+hardware-mining",
"github": "en/hardware-mining.md",
"value": null
}
]
"posts": []
},
{
"title": "Setup",
"slug": "en+getting-started",
"github": "en/getting-started.md",
"value": null,
"posts": [
{
"title": "Arch Linux Installation",
"slug": "en+install-lotus-arch",
"github": "en/install-lotus-arch.md",
"value": null
},
{
"title": "Ubuntu Installation",
"slug": "en+install-lotus-ubuntu",
"github": "en/install-lotus-ubuntu.md",
"value": null
},
{
"title": "Fedora Installation",
"slug": "en+install-lotus-fedora",
"github": "en/install-lotus-fedora.md",
"value": null
},
{
"title": "MacOS Installation",
"slug": "en+install-lotus-macos",
"github": "en/install-lotus-macos.md",
"value": null
},
{
"title": "Updating Lotus",
"slug": "en+updating-lotus",
"github": "en/updating-lotus.md",
"value": null
},
{
"title": "Join Testnet",
"slug": "en+join-testnet",
"github": "en/join-testnet.md",
"value": null
},
{
"title": "Use Lotus with systemd",
"slug": "en+install-systemd-services",
"github": "en/install-systemd-services.md",
"value": null
},
{
"title": "Setup Troubleshooting",
"slug": "en+setup-troubleshooting",
"github": "en/setup-troubleshooting.md",
"value": null
},
{
"title": "Environment Variables",
"slug": "en+env-vars",
"github": "en/environment-vars.md",
"value": null
}
]
},
{
"title": "Architecture",
"title": "Lotus Architecture (WIP)",
"slug": "en+arch",
"github": "en/architecture.md",
"github": "en/architecture/architecture.md",
"value": null,
"posts": [
{
"title": "The Message Pool",
"slug": "en+mpool",
"github": "en/mpool.md",
"value": null
}
{
"title": "The Message Pool",
"slug": "en+mpool",
"github": "en/architecture/mpool.md",
"value": null
}
]
},
{
"title": "Storage Mining",
"slug": "en+mining",
"github": "en/mining.md",
"value": null,
"posts": [
{
"title": "Lotus Worker",
"slug": "en+lotus-worker",
"github": "en/mining-lotus-worker.md",
"value": null
},
{
"title": "Static Ports",
"slug": "en+setting-a-static-port",
"github": "en/setting-a-static-port.md",
"value": null
},
{
"title": "Mining Troubleshooting",
"slug": "en+mining-troubleshooting",
"github": "en/mining-troubleshooting.md",
"value": null
}
]
},
{
"title": "Storing Data",
"slug": "en+storing-data",
"github": "en/storing-data.md",
"value": null,
"posts": [
{
"title": "Storage Troubleshooting",
"slug": "en+storing-data-troubleshooting",
"github": "en/storing-data-troubleshooting.md",
"value": null
},
{
"title": "Information for Miners",
"slug": "en+info-for-miners",
"github": "en/miner-deals.md",
"value": null
},
{
"title": "IPFS Integration",
"slug": "en+ipfs-client-integration",
"github": "en/storing-ipfs-integration.md",
"value": null
}
]
},
{
"title": "Retrieving Data",
"slug": "en+retrieving-data",
"github": "en/retrieving-data.md",
"value": null,
"posts": []
},
{
"title": "Payment Channels",
"slug": "en+payment-channels",
"github": "en/payment-channels.md",
"value": null,
"posts": []
},
{
"title": "Command Line Interface",
"slug": "en+cli",
"github": "en/cli.md",
"value": null,
"posts": []
},
{
"title": "API",
"slug": "en+api",
"github": "en/api.md",
"value": null,
"posts": [
{
"title": "Remote API Support",
"slug": "en+api-scripting-support",
"github": "en/api-scripting-support.md",
"value": null
},
{
"title": "API Methods",
"slug": "en+api-methods",
"github": "en/api-methods.md",
"value": null
},
{
"title": "API Troubleshooting",
"slug": "en+api-troubleshooting",
"github": "en/api-troubleshooting.md",
"value": null
}
]
},
{
"title": "Developer Tools",
"slug": "en+dev-tools",
"github": "en/dev-tools.md",
"value": null,
"posts": [
{
"title": "Setup Local Devnet",
"slug": "en+setup-local-dev-net",
"github": "en/local-dev-net.md",
"value": null,
"posts": []
},
{
"title": "Jaeger Tracing",
"slug": "en+dev-tools-jaeger-tracing",
"github": "en/dev-tools-jaeger-tracing.md",
"value": null,
"posts": []
}
]
},
{
"title": "FAQs",
"slug": "en+faqs",
"github": "en/faqs.md",
"value": null,
"posts": []
},
{
"title": "Glossary",
"slug": "en+glossary",
"github": "en/.glossary.json",
"value": null,
"custom": {
"glossary": true
},
"posts": []
}
]
}

16
documentation/en/README.md Normal file
View File

@ -0,0 +1,16 @@
# Lotus documentation
This folder contains some Lotus documentation mostly intended for Lotus developers.
User documentation (including documentation for miners) has been moved to specific Lotus sections in https://docs.filecoin.io:
- https://docs.filecoin.io/get-started/lotus
- https://docs.filecoin.io/store/lotus
- https://docs.filecoin.io/mine/lotus
- https://docs.filecoin.io/build/lotus
## The Lotu.sh site
The https://lotu.sh and https://docs.lotu.sh sites are generated from this folder based on the index provided by [.library.json](.library.json). This is done at the [lotus-docs repository](https://github.com/filecoin-project/lotus-docs), which contains Lotus as a git submodule.
To update the site, the lotus-docs repository should be updated with the desired version for the lotus git submodule. Once pushed to master, it will be auto-deployed.

0
documentation/en/dev/WIP-arch-complementary-notes.md → documentation/en/WIP-arch-complementary-notes.md
View File

19
documentation/en/about.md Normal file
View File

@ -0,0 +1,19 @@
# Lotus
Lotus is an implementation of the **Filecoin Distributed Storage Network**.
It is written in Go and provides a suite of command-line applications:
- Lotus Node (`lotus`): a Filecoin Node: validates network transactions, manages a FIL wallet, can perform storage and retrieval deals.
- Lotus Miner (`lotus-miner`): a Filecoin miner. See the the respective Lotus Miner section in the Mine documentation.
- Lotus Worker (`lotus-worker`): a worker that assists miners to perform mining-related tasks. See its respective guide for more information.
The [Lotus user documentation](https://docs.filecoin.io/get-started/lotus) is part of the [Filecoin documentation site](https://docs.filecoin.io):
* To install and get started with Lotus, visit the [Get Started section](https://docs.filecoin.io/get-started/lotus).
* Information about how to perform deals on the Filecoin network using Lotus can be found in the [Store section](https://docs.filecoin.io/store/lotus).
* Miners looking to provide storage to the Network can find the latest guides in the [Mine section](https://docs.filecoin.io/mine/lotus).
* Developers and integrators that wish to use the Lotus APIs can start in the [Build section](https://docs.filecoin.io/mine/lotus).
For more details about Filecoin, check out the [Filecoin Docs](https://docs.filecoin.io) and [Filecoin Spec](https://spec.filecoin.io/).

39
documentation/en/api-methods.md
View File

@ -169,6 +169,7 @@
* [SyncState](#SyncState)
* [SyncSubmitBlock](#SyncSubmitBlock)
* [SyncUnmarkBad](#SyncUnmarkBad)
* [SyncValidateTipset](#SyncValidateTipset)
* [Wallet](#Wallet)
* [WalletBalance](#WalletBalance)
* [WalletDefaultAddress](#WalletDefaultAddress)
@ -181,6 +182,7 @@
* [WalletSetDefault](#WalletSetDefault)
* [WalletSign](#WalletSign)
* [WalletSignMessage](#WalletSignMessage)
* [WalletValidateAddress](#WalletValidateAddress)
* [WalletVerify](#WalletVerify)
##
@ -4379,6 +4381,28 @@ Inputs:
Response: `{}`
### SyncValidateTipset
SyncValidateTipset indicates whether the provided tipset is valid or not
Perms: read
Inputs:
```json
[
[
{
"/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
},
{
"/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve"
}
]
]
```
Response: `true`
## Wallet
@ -4585,6 +4609,21 @@ Response:
}
```
### WalletValidateAddress
WalletValidateAddress validates whether a given string can be decoded as a well-formed address
Perms: read
Inputs:
```json
[
"string value"
]
```
Response: `"t01234"`
### WalletVerify
WalletVerify takes an address, a signature, and some bytes, and indicates whether the signature is valid.
The address does not have to be in the wallet.

25
documentation/en/api-scripting-support.md
View File

@ -1,25 +0,0 @@
# Remote API Support
You may want to delegate the work **Lotus Miner** or **Lotus Node** performs to other machines.
Here is how to setup the necessary authorization and environment variables.
## Environment variables
Environmental variables are variables that are defined for the current shell and are inherited by any child shells or processes. Environmental variables are used to pass information into processes that are spawned from the shell.
Using the [JWT you generated](https://lotu.sh/en+api#how-do-i-generate-a-token-18865), you can assign it and the **multiaddr** to the appropriate environment variable.
```sh
# Lotus Node
FULLNODE_API_INFO="JWT_TOKEN:/ip4/127.0.0.1/tcp/1234/http"
# Lotus Miner
MINER_API_INFO="JWT_TOKEN:/ip4/127.0.0.1/tcp/2345/http"
```
You can also use `lotus auth api-info --perm admin` to quickly create _API_INFO env vars
- The **Lotus Node**'s `mutliaddr` is in `~/.lotus/api`.
- The default token is in `~/.lotus/token`.
- The **Lotus Miner**'s `multiaddr` is in `~/.lotusminer/config`.
- The default token is in `~/.lotusminer/token`.

36
documentation/en/api-troubleshooting.md
View File

@ -1,36 +0,0 @@
# API Troubleshooting
## Types: params
`params` must be an array. If there are no `params` you should still pass an empty array.
## Types: TipSet
For methods such as `Filecoin.StateMinerPower`, where the method accepts the argument of the type `TipSet`, you can pass `null` to use the current chain head.
```sh
curl -X POST \
-H "Content-Type: application/json" \
--data '{ "jsonrpc": "2.0", "method": "Filecoin.StateMinerPower", "params": ["t0101", null], "id": 3 }' \
'http://127.0.0.1:1234/rpc/v0'
```
## Types: Sending a CID
If you do not serialize the CID as a [JSON IPLD link](https://did-ipid.github.io/ipid-did-method/#txref), you will receive an error. Here is an example of a broken CURL request:
```sh
curl -X POST \
-H "Content-Type: application/json" \
--data '{ "jsonrpc": "2.0", "method":"Filecoin.ClientGetDealInfo", "params": ["bafyreiaxl446wlnu6t6dpq4ivrjf4gda4gvsoi4rr6mpxau7z25xvk5pl4"], "id": 0 }' \
'http://127.0.0.1:1234/rpc/v0'
```
To fix it, change the `params` property to:
```sh
curl -X POST \
-H "Content-Type: application/json" \
--data '{ "jsonrpc": "2.0", "method":"Filecoin.ClientGetDealInfo", "params": [{"/": "bafyreiaxl446wlnu6t6dpq4ivrjf4gda4gvsoi4rr6mpxau7z25xvk5pl4"}], "id": 0 }' \
'http://127.0.0.1:1234/rpc/v0'
```

85
documentation/en/api.md
View File

@ -1,85 +0,0 @@
# API
Here is an early overview of how to make API calls.
Implementation details for the **JSON-RPC** package are [here](https://github.com/filecoin-project/go-jsonrpc).
## Overview: How do you modify the config.toml to change the API endpoint?
API requests are made against `127.0.0.1:1234` unless you modify `.lotus/config.toml`.
Options:
- `http://[api:port]/rpc/v0` - HTTP endpoint
- `ws://[api:port]/rpc/v0` - Websocket endpoint
- `PUT http://[api:port]/rest/v0/import` - File import, it requires write permissions.
## What methods can I use?
For now, you can look into different files to find methods available to you based on your needs:
- [Both Lotus node + miner APIs](https://github.com/filecoin-project/lotus/blob/master/api/api_common.go)
- [Lotus node API](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go)
- [Lotus miner API](https://github.com/filecoin-project/lotus/blob/master/api/api_storage.go)
The necessary permissions for each are in [api/struct.go](https://github.com/filecoin-project/lotus/blob/master/api/struct.go).
## How do I make an API request?
To demonstrate making an API request, we will take the method `ChainHead` from [api/api_full.go](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go).
```go
ChainHead(context.Context) (*types.TipSet, error)
```
And create a CURL command. In this command, `ChainHead` is included as `{ "method": "Filecoin.ChainHead" }`:
```sh
curl -X POST \
-H "Content-Type: application/json" \
--data '{ "jsonrpc": "2.0", "method": "Filecoin.ChainHead", "params": [], "id": 3 }' \
'http://127.0.0.1:1234/rpc/v0'
```
If the request requires authorization, add an authorization header:
```sh
curl -X POST \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $(cat ~/.lotusminer/token)" \
--data '{ "jsonrpc": "2.0", "method": "Filecoin.ChainHead", "params": [], "id": 3 }' \
'http://127.0.0.1:1234/rpc/v0'
```
> In the future we will add a playground to make it easier to build and experiment with API requests.
## CURL authorization
To authorize your request, you will need to include the **JWT** in a HTTP header, for example:
```sh
-H "Authorization: Bearer $(cat ~/.lotusminer/token)"
```
Admin token is stored in `~/.lotus/token` for the **Lotus Node** or `~/.lotusminer/token` for the **Lotus Miner**.
## How do I generate a token?
To generate a JWT with custom permissions, use this command:
```sh
# Lotus Node
lotus auth create-token --perm admin
# Lotus Miner
lotus-miner auth create-token --perm admin
```
## What authorization level should I use?
When viewing [api/apistruct/struct.go](https://github.com/filecoin-project/lotus/blob/master/api/apistruct/struct.go), you will encounter these types:
- `read` - Read node state, no private data.
- `write` - Write to local store / chain, and `read` permissions.
- `sign` - Use private keys stored in wallet for signing, `read` and `write` permissions.
- `admin` - Manage permissions, `read`, `write`, and `sign` permissions.

2
documentation/en/architecture.md → documentation/en/architecture/architecture.md
View File

@ -6,7 +6,7 @@ Filecoin protocol, validating the blocks and state transitions.
The specification for the Filecoin protocol can be found [here](https://filecoin-project.github.io/specs/).
For information on how to setup and operate a Lotus node,
please follow the instructions [here](https://lotu.sh/en+getting-started).
please follow the instructions [here](en+getting-started).
# Components

0
documentation/en/mpool.md → documentation/en/architecture/mpool.md
View File

108
documentation/en/cli.md
View File

@ -1,108 +0,0 @@
# Lotus Command Line Interface
The Command Line Interface (CLI) is a convenient way to interact with
a Lotus node. You can use the CLI to operate your node,
get information about the blockchain,
manage your accounts and transfer funds,
create storage deals, and much more!
The CLI is intended to be self-documenting, so when in doubt, simply add `--help`
to whatever command you're trying to run! This will also display all of the
input parameters that can be provided to a command.
We highlight some of the commonly
used features of the CLI below.
All CLI commands should be run from the home directory of the Lotus project.
## Operating a Lotus node
### Starting up a node
```sh
lotus daemon
```
This command will start up your Lotus node, with its API port open at 1234.
You can pass `--api=<number>` to use a different port.
### Checking your sync progress
```sh
lotus sync status
```
This command will print your current tipset height under `Height`, and the target tipset height
under `Taregt`.
You can also run `lotus sync wait` to get constant updates on your sync progress.
### Getting the head tipset
```sh
lotus chain head
```
### Control the logging level
```sh
lotus log set-level
```
This command can be used to toggle the logging levels of the different
systems of a Lotus node. In decreasing order
of logging detail, the levels are `debug`, `info`, `warn`, and `error`.
As an example,
to set the `chain` and `blocksync` to log at the `debug` level, run
`lotus log set-level --system chain --system blocksync debug`.
To see the various logging system, run `lotus log list`.
### Find out what version of Lotus you're running
```sh
lotus version
```
## Managing your accounts
### Listing accounts in your wallet
```sh
lotus wallet list
```
### Creating a new account
```sh
lotus wallet new bls
```
This command will create a new BLS account in your wallet; these
addresses start with the prefix `t3`. Running `lotus wallet new secp256k1`
(or just `lotus wallet new`) will create
a new Secp256k1 account, which begins with the prefix `t1`.
### Getting an account's balance
```sh
lotus wallet balance <address>
```
### Transferring funds
```sh
lotus send --source=<source address> <destination address> <amount>
```
This command will transfer `amount` (in attoFIL) from `source address` to `destination address`.
### Importing an account into your wallet
```sh
lotus wallet import <path to private key>
```
This command will import an account whose private key is saved at the specified file.
### Exporting an account from your wallet
```sh
lotus wallet export <address>
```
This command will print out the private key of the specified address
if it is in your wallet. Always be careful with your private key!

0
documentation/en/dev/create-miner.md → documentation/en/create-miner.md
View File

3
documentation/en/dev-tools.md
View File

@ -1,3 +0,0 @@
# Developer Tools
> Running a local network can be a great way to understand how Lotus works and test your setup.

65
documentation/en/environment-vars.md
View File

@ -1,65 +0,0 @@
# Lotus Environment Variables
## Building
## Common
The environment variables are common across most lotus binaries.
### `LOTUS_FD_MAX`
Sets the file descriptor limit for the process. This should be set high (8192
or higher) if you ever notice 'too many open file descriptor' errors.
### `LOTUS_JAEGER`
This can be set to enable jaeger trace reporting. The value should be the url
of the jaeger trace collector, the default for most jaeger setups should be
`localhost:6831`.
### `LOTUS_DEV`
If set to a non-empty value, certain parts of the application will print more
verbose information to aid in development of the software. Not recommended for
end users.
## Lotus Daemon
### `LOTUS_PATH`
Sets the location for the lotus daemon on-disk repo. If left empty, this defaults to `~/.lotus`.
### `LOTUS_SKIP_GENESIS_CHECK`
Can be set to `_yes_` if you wish to run a lotus network with a different
genesis than the default one built into your lotus binary.
### `LOTUS_CHAIN_TIPSET_CACHE`
Sets the cache size for the chainstore tipset cache. The default value is 8192,
but if your usage of the lotus API involves frequent arbitrary tipset lookups,
you may want to increase this.
### `LOTUS_CHAIN_INDEX_CACHE`
Sets the cache size for the chainstore epoch index cache. The default value is 32768,
but if your usage of the lotus API involves frequent deep chain lookups for
block heights that are very far from the current chain height, you may want to
increase this.
### `LOTUS_BSYNC_MSG_WINDOW`
Set the initial maximum window size for message fetching blocksync requests. If
you have a slower internet connection and are having trouble syncing, you might
try lowering this down to 10-20 for a 'poor' internet connection.
## Lotus Miner
A number of environment variables are respected for configuring the behavior of the filecoin proving subsystem. For more details on those [see here](https://github.com/filecoin-project/rust-fil-proofs/#settings).
### `LOTUS_MINER_PATH`
Sets the location for the lotus miners on-disk repo. If left empty, this defaults to `~/.lotusminer`.

138
documentation/en/faqs.md
View File

@ -1,138 +0,0 @@
# Frequently Asked Questions
Here are some FAQs concerning the Lotus implementation and participation in
Testnet.
For questions concerning the broader Filecoin project, please
go [here](https://filecoin.io/faqs/).
## Introduction to Lotus
### What is Lotus?
Lotus is an implementation of the **Filecoin Distributed Storage Network**, written in Go.
It is designed to be modular and interoperable with any other implementation of the Filecoin Protocol.
More information about Lotus can be found [here](https://lotu.sh/).
### What are the components of Lotus?
Lotus is composed of two separate pieces that can talk to each other:
The Lotus Node can sync the blockchain, validating all blocks, transfers, and deals
along the way. It can also facilitate the creation of new storage deals. If you are not
interested in providing your own storage to the network, and do not want to produce blocks
yourself, then the Lotus Node is all you need!
The Lotus Miner does everything you need for the registration of storage, and the
production of new blocks. The Lotus Miner communicates with the network by talking
to a Lotus Node over the JSON-RPC API.
## Setting up a Lotus Node
### How do I set up a Lotus Node?
Follow the instructions found [here](https://lotu.sh/en+getting-started).
### Where can I get the latest version of Lotus?
Download the binary tagged as the `Latest Release` from the
[Lotus Github repo](https://github.com/filecoin-project/lotus/releases).
### What operating systems can Lotus run on?
Lotus can build and run on most Linux and MacOS systems with at least
8GB of RAM. Windows is not yet supported.
### How can I update to the latest version of Lotus?
To update Lotus, follow the instructions [here](https://lotu.sh/en+updating-lotus).
### How do I prepare a fresh installation of Lotus?
Stop the Lotus daemon, and delete all related files, including sealed and chain data by
running `rm ~/.lotus ~/.lotusminer`.
Then, install Lotus afresh by following the instructions
found [here](https://lotu.sh/en+getting-started).
### Can I configure where the node's config and data goes?
Yes! The `LOTUS_PATH` variable sets the path for where the Lotus node's data is written.
The `LOTUS_MINER_PATH` variable does the same for miner-specific information.
## Interacting with a Lotus Node
### How can I communicate with a Lotus Node?
Lotus Nodes have a command-line interface, as well as a JSON-RPC API.
### What are the commands I can send using the command-line interface?
The command-line interface is self-documenting, try running `lotus --help` from the `lotus` home
directory for more.
### How can I send a request over the JSON-RPC API?
Information on how to send a `cURL` request to the JSON-RPC API can be found
[here](https://lotu.sh/en+api). A JavaScript client is under development.
### What are the requests I can send over the JSON-RPC API?
Please have a look at the
[source code](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go)
for a list of methods supported by the JSON-RPC API.
## The Test Network
### What is Testnet?
Testnet is a live network of Lotus Nodes run by the
community for testing purposes.
It has 2 PiB of storage (and growing!) dedicated to it.
### Is FIL on the Testnet worth anything?
Nothing at all! Real-world incentives may be provided in a future phase of Testnet, but this is
yet to be confirmed.
### How can I see the status of Testnet?
The [dashboard](https://stats.testnet.filecoin.io/) displays the status of the network as
well as a ton
of other metrics you might find interesting.
## Mining with a Lotus Node on Testnet
### How do I get started mining with Lotus?
Follow the instructions found [here](https://lotu.sh/en+mining).
### What are the minimum hardware requirements?
An example test configuration, and minimum hardware requirements can be found
[here](https://lotu.sh/en+hardware-mining).
Note that these might NOT be the minimum requirements for mining on Mainnet.
### What are some GPUs that have been tested?
A list of benchmarked GPUs can be found [here](https://lotu.sh/en+hardware-mining#benchmarked-gpus-7393).
### Why is my GPU not being used when sealing a sector?
Sealing a sector does not involve constant GPU operations. It's possible
that your GPU simply isn't necessary at the moment you checked.
## Advanced questions
### Is there a Docker image for lotus?
Community-contributed Docker and Docker Compose examples are available
[here](https://github.com/filecoin-project/lotus/tree/master/tools/dockers/docker-examples).
### How can I run two miners on the same machine?
You can do so by changing the storage path variable for the second miner, e.g.,
`LOTUS_MINER_PATH=~/.lotusminer2`. You will also need to make sure that no ports collide.
### How do I setup my own local devnet?
Follow the instructions found [here](https://lotu.sh/en+setup-local-dev-net).

23
documentation/en/getting-started.md
View File

@ -1,23 +0,0 @@
# Lotus
Lotus is an implementation of the **Filecoin Distributed Storage Network**. You can run the Lotus software client to join the **Filecoin Testnet**.
For more details about Filecoin, check out the [Filecoin Docs](https://docs.filecoin.io) and [Filecoin Spec](https://filecoin-project.github.io/specs/).
## What can I learn here?
- How to install Lotus on [Arch Linux](https://lotu.sh/en+install-lotus-arch), [Ubuntu](https://lotu.sh/en+install-lotus-ubuntu), or [MacOS](https://lotu.sh/en+install-lotus-macos).
- Joining the [Lotus Testnet](https://lotu.sh/en+join-testnet).
- [Storing](https://lotu.sh/en+storing-data) or [retrieving](https://lotu.sh/en+retrieving-data) data.
- Mining Filecoin using the **Lotus Miner** in your [CLI](https://lotu.sh/en+mining).
## How is Lotus designed?
Lotus is architected modularly to keep clean API boundaries while using the same process. Installing Lotus will include two separate programs:
- The **Lotus Node**
- The **Lotus Miner**
The **Lotus Miner** is intended to be run on the machine that manages a single miner instance, and is meant to communicate with the **Lotus Node** via the websocket **JSON-RPC** API for all of the chain interaction needs.
This way, a mining operation may easily run a **Lotus Miner** or many of them, connected to one or many **Lotus Node** instances.

54
documentation/en/hardware-mining.md
View File

@ -1,54 +0,0 @@
# Protocol Labs Standard Testing Configuration
> This documentation page describes the standard testing configuration the Protocol Labs team has used to test **Lotus Miner**s on Lotus. There is no guarantee this testing configuration will be suitable for Filecoin storage mining at MainNet launch. If you need to buy new hardware to join the Filecoin Testnet, we recommend to buy no more hardware than you require for testing. To learn more please read this [Protocol Labs Standard Testing Configuration post](https://filecoin.io/blog/filecoin-testnet-mining/).
**Sector sizes** and **minimum pledged storage** required to mine blocks are two very important Filecoin Testnet parameters that impact hardware decisions. We will continue to refine all parameters during Testnet.
BECAUSE OF THIS, OUR STANDARD TESTING CONFIGURATION FOR FILECOIN MAINNET CAN AND WILL CHANGE. YOU HAVE BEEN WARNED.
## Example configuration
The setup below is a minimal example for sealing 32 GiB sectors on Lotus:
- 2 TB of hard drive space.
- 8 core CPU
- 128 GiB of RAM
Note that 1GB sectors don't require as high of specs, but are likely to be removed as we improve the performance of 32GB sector sealing.
For the first part of the sealing process, AMD CPU's are __highly recommended__, because of the `Intel SHA Extensions` instruction set that is available there ever since the `Zen` microarchitecture. Hence, AMD CPU's seem to perform much better on the testnet than other CPU's. Contrary to what the name implies, this extended instruction set is not available on recent Intel desktop/server chips.
## Testnet discoveries
- If you only have 128GiB of ram, enabling 256GB of **NVMe** swap on an SSD will help you avoid out-of-memory issues while mining.
## Benchmarked GPUs
GPUs are a must for getting **block rewards**. Here are a few that have been confirmed to generate **SNARKs** quickly enough to successfully mine blocks on the Lotus Testnet.
- GeForce RTX 2080 Ti
- GeForce RTX 2080 SUPER
- GeForce RTX 2080
- GeForce GTX 1080 Ti
- GeForce GTX 1080
- GeForce GTX 1060
## Testing other GPUs
If you want to test a GPU that is not explicitly supported, use the following global **environment variable**:
```sh
BELLMAN_CUSTOM_GPU="<NAME>:<NUMBER_OF_CORES>"
```
Here is an example of trying a GeForce GTX 1660 Ti with 1536 cores.
```sh
BELLMAN_CUSTOM_GPU="GeForce GTX 1660 Ti:1536"
```
To get the number of cores for your GPU, you will need to check your card’s specifications.
## Benchmarking
Here is a [benchmarking tool](https://github.com/filecoin-project/lotus/tree/master/cmd/lotus-bench) and a [GitHub issue thread](https://github.com/filecoin-project/lotus/issues/694) for those who wish to experiment with and contribute hardware setups for the **Filecoin Testnet**.

7
documentation/en/hardware.md
View File

@ -1,7 +0,0 @@
# Hardware
> This page is a work in progress. Exact mining requirements are still in the works.
Lotus can build and run on most [Linux](https://ubuntu.com/) and [MacOS](https://www.apple.com/macos) systems with at least 8GiB of RAM.
Windows is not yet supported.

51
documentation/en/install-lotus-arch.md
View File

@ -1,51 +0,0 @@
# Arch Linux Instructions
These steps will install the following dependencies:
- go (1.14 or higher)
- gcc (7.4.0 or higher)
- git (version 2 or higher)
- bzr (some go dependency needs this)
- jq
- pkg-config
- opencl-icd-loader
- opencl driver (like nvidia-opencl on arch) (for GPU acceleration)
- opencl-headers (build)
- rustup (proofs build)
- llvm (proofs build)
- clang (proofs build)
### Install dependencies
```sh
sudo pacman -Syu opencl-icd-loader gcc git bzr jq pkg-config opencl-icd-loader opencl-headers
```
### Install Go 1.14
Install the latest version of Go by following [the docs on their website](https://golang.org/doc/install).
### Clone the Lotus repository
```sh
git clone https://github.com/filecoin-project/lotus.git
cd lotus/
```
### Build the Lotus binaries from source and install
! **If you are running an AMD platform or if your CPU supports SHA extensions you will want to build the Filecoin proofs natively**
```sh
make clean && make all
sudo make install
```
#### Native Filecoin FFI building
```sh
env env RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make clean deps all
sudo make install
```
After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus Testnet](https://lotu.sh/en+join-testnet).

54
documentation/en/install-lotus-fedora.md
View File

@ -1,54 +0,0 @@
# Fedora Instructions
> tested on 30
**NOTE:** If you have an AMD GPU the opencl instructions may be incorrect...
These steps will install the following dependencies:
- go (1.14 or higher)
- gcc (7.4.0 or higher)
- git (version 2 or higher)
- bzr (some go dependency needs this)
- jq
- pkg-config
- rustup (proofs build)
- llvm (proofs build)
- clang (proofs build)
### Install dependencies
```sh
$ sudo dnf -y update
$ sudo dnf -y install gcc git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm
$ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
```
### Install Go 1.14
Install the latest version of Go by following [the docs on their website](https://golang.org/doc/install).
### Clone the Lotus repository
```sh
git clone https://github.com/filecoin-project/lotus.git
cd lotus/
```
### Build the Lotus binaries from source and install
! **If you are running an AMD platform or if your CPU supports SHA extensions you will want to build the Filecoin proofs natively**
```sh
$ make clean && make all
$ sudo make install
```
#### Native Filecoin FFI building
```sh
env env RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make clean deps all
sudo make install
```
After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus TestNet](https://lotu.sh/en+join-testnet).

62
documentation/en/install-lotus-macos.md
View File

@ -1,62 +0,0 @@
# MacOS Instructions
## Get XCode Command Line Tools
To check if you already have the XCode Command Line Tools installed via the CLI, run:
```sh
xcode-select -p
```
If this command returns a path, you can move on to the next step. Otherwise, to install via the CLI, run:
```sh
xcode-select --install
```
To update, run:
```sh
sudo rm -rf /Library/Developer/CommandLineTools
xcode-select --install
```
## Get HomeBrew
We recommend that MacOS users use [HomeBrew](https://brew.sh) to install each the necessary packages.
Check if you have HomeBrew:
```sh
brew -v
```
This command returns a version number if you have HomeBrew installed and nothing otherwise.
In your terminal, enter this command to install Homebrew:
```sh
/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
```
Use the command `brew install` to install the following packages:
```sh
brew install go bzr jq pkg-config rustup
```
Clone
```sh
git clone https://github.com/filecoin-project/lotus.git
cd lotus/
```
Build
```sh
make clean && make all
sudo make install
```
After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus Testnet](https://lotu.sh/en+join-testnet).

54
documentation/en/install-lotus-ubuntu.md
View File

@ -1,54 +0,0 @@
# Ubuntu Instructions
These steps will install the following dependencies:
- go (1.14 or higher)
- gcc (7.4.0 or higher)
- git (version 2 or higher)
- bzr (some go dependency needs this)
- jq
- pkg-config
- opencl-icd-loader
- opencl driver (like nvidia-opencl on arch) (for GPU acceleration)
- opencl-headers (build)
- rustup (proofs build)
- llvm (proofs build)
- clang (proofs build)
### Install dependencies
```sh
sudo apt update
sudo apt install mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl
sudo apt upgrade
```
### Install Go 1.14
Install the latest version of Go by following [the docs on their website](https://golang.org/doc/install).
### Clone the Lotus repository
```sh
git clone https://github.com/filecoin-project/lotus.git
cd lotus/
```
### Build the Lotus binaries from source and install
! **If you are running an AMD platform or if your CPU supports SHA extensions you will want to build the Filecoin proofs natively**
```sh
make clean && make all
sudo make install
```
#### Native Filecoin FFI building
```sh
env env RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make clean deps all
sudo make install
```
After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus Testnet](https://lotu.sh/en+join-testnet).

145
documentation/en/install-systemd-services.md
View File

@ -1,145 +0,0 @@
# Use Lotus with systemd
Lotus is capable of running as a systemd service daemon. You can find installable service files for systemd in the [lotus repo scripts directory](https://github.com/filecoin-project/lotus/tree/master/scripts) as files with `.service` extension. In order to install these service files, you can copy these `.service` files to the default systemd unit load path.
The services expect their binaries to be present in `/usr/local/bin/`. You can use `make` to install them by running:
```sh
$ sudo make install
```
for `lotus` and `lotus-miner` and
```sh
$ sudo make install-chainwatch
```
for the `chainwatch` tool.
## Installing services via `make`
If your host uses the default systemd unit load path, the `lotus-daemon` and `lotus-miner` services can be installed by running:
```sh
$ sudo make install-services
```
To install the the `lotus-chainwatch` service run:
```sh
$ sudo make install-chainwatch-service
```
You can install all services together by running:
```sh
$ sudo make install-all-services
```
The `lotus-daemon` and the `lotus-miner` services can be installed individually too by running:
```sh
$ sudo make install-daemon-service
```
and
```sh
$ sudo make install-miner-service
```
### Notes
When installing the `lotus-miner` and/or `lotus-chainwatch` service the `lotus-daemon` service gets automatically installed since the other two services depend on it being installed to run.
All `install-*-service*` commands will install the latest binaries in the lotus build folders to `/usr/local/bin/`. If you do not want to use the latest build binaries please copy the `*.service` files by hand.
## Removing via `make`
All services can beremoved via `make`. To remove all services together run:
```sh
$ sudo make clean-all-services
```
Individual services can be removed by running:
```sh
$ sudo make clean-chainwatch-services
$ sudo make clean-miner-services
$ sudo make clean-daemon-services
```
### Notes
The services will be stoppend and disabled when removed.
Removing the `lotus-daemon` service will automatically remove the depending services `lotus-miner` and `lotus-chainwatch`.
## Controlling services
All service can be controlled with the `systemctl`. A few basic control commands are listed below. To get detailed infos about the capabilities of the `systemctl` command please consult your distributions man pages by running:
```sh
$ man systemctl
```
### Start/Stop services
You can start the services by running:
```sh
$ sudo systemctl start lotus-daemon
$ sudo systemctl start lotus-miner
$ sudo systemctl start lotus-chainwatch
```
and can be stopped by running:
```sh
$ sudo systemctl stop lotus-daemon
$ sudo systemctl stop lotus-miner
$ sudo systemctl stop lotus-chainwatch
```
### Enabling services on startup
To enable the services to run automatically on startup execute:
```sh
$ sudo systemctl enable lotus-daemon
$ sudo systemctl enable lotus-miner
$ sudo systemctl enable lotus-chainwatch
```
To disable the services on startup run:
```sh
$ sudo systemctl disable lotus-daemon
$ sudo systemctl disable lotus-miner
$ sudo systemctl disable lotus-chainwatch
```
### Notes
Systemd will not let services be enabled or started without their requirements. Starting the `lotus-chainwatch` and/or `lotus-miner` service with automatically start the `lotus-daemon` service (if installed!). Stopping the `lotus-daemon` service will stop the other two services. The same pattern is executed for enabling and disabling the services.
## Interacting with service logs
Logs from the services can be reviewed using `journalctl`.
### Follow logs from a specific service unit
```sh
$ sudo journalctl -u lotus-daemon -f
```
### View logs in reverse order
```sh
$ sudo journalctl -u lotus-miner -r
```
### Log files
Besides the systemd service logs all services save their own log files in `/var/log/lotus/`.

0
documentation/en/dev-tools-jaeger-tracing.md → documentation/en/jaeger-tracing.md
View File

93
documentation/en/join-testnet.md
View File

@ -1,93 +0,0 @@
# Join Testnet
## Introduction
Anyone can set up a **Lotus Node** and connect to the **Lotus Testnet**. This is the best way to explore the current CLI and the **Filecoin Decentralized Storage Market**.
## Note: Using the Lotus Node from China
If you are trying to use `lotus` from China. You should set this **environment variable** on your machine:
```sh
export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/"
```
## Get started
Start the **daemon** using the default configuration in `./build`:
```sh
lotus daemon
```
In another terminal window, check your connection with peers:
```sh
lotus net peers | wc -l
```
In order to connect to the network, you need to be connected to at least 1 peer. If you’re seeing 0 peers, read our [troubleshooting notes](https://lotu.sh/en+setup-troubleshooting).
Make sure that you have a reasonable "open files limit" set on your machine, such as 10000. If you're seeing a lower value, such as 256 (default on macOS), read our [troubleshooting notes](https://lotu.sh/en+setup-troubleshooting) on how to update it prior to starting the Lotus daemon.
## Chain sync
While the daemon is running, the next requirement is to sync the chain. Run the command below to view the chain sync progress. To see current chain height, visit the [network stats page](https://stats.testnet.filecoin.io/).
```sh
lotus sync wait
```
- This step will take anywhere between a few hours to a couple of days.
- You will be able to perform **Lotus Testnet** operations after it is finished.
## Create your first address
Initialize a new wallet:
```sh
lotus wallet new
```
Sometimes your operating system may limit file name length to under 150 characters. You need to use a file system that supports long filenames.
Here is an example of the response:
```sh
t1aswwvjsae63tcrniz6x5ykvsuotlgkvlulnqpsi
```
- Visit the [faucet](http://spacerace.faucet.glif.io/) to add funds.
- Paste the address you created under REQUEST.
- Press the Request button.
## Check wallet address balance
Wallet balances in the Lotus Testnet are in **FIL**, the smallest denomination of FIL is an **attoFil**, where 1 attoFil = 10^-18 FIL.
```sh
lotus wallet balance <YOUR_NEW_ADDRESS>
```
You will not see any attoFIL in your wallet if your **chain** is not fully synced.
## Send FIL to another wallet
To send FIL to another wallet from your default account, use this command:
```
lotus send <target> <amount>
```
## Configure your node's connectivity
To effectively accept incoming storage & retrieval deals, your Lotus node needs to be accessible to other nodes on the network. To improve your connectivity, be sure to:
- [Set the multiaddresses for you miner to listen on](https://docs.filecoin.io/mine/connectivity/#setting-multiaddresses)
- [Maintain a healthy peer count](https://docs.filecoin.io/mine/connectivity/#checking-peer-count)
- [Enable port forwarding](https://docs.filecoin.io/mine/connectivity/#port-forwarding)
- [Configure your public IP address and port](https://docs.filecoin.io/mine/connectivity/#setting-a-public-ip-address)
## Monitor the dashboard
To see the latest network activity, including **chain block height**, **block height**, **blocktime**, **total network power**, largest **block producer miner**, check out the [monitoring dashboard](https://stats.testnet.filecoin.io).

54
documentation/en/local-dev-net.md
View File

@ -1,54 +0,0 @@
# Setup Local Devnet
Build the Lotus Binaries in debug mode, This enables the use of 2048 byte sectors.
```sh
make 2k
```
Set the `LOTUS_SKIP_GENESIS_CHECK` environment variable to `_yes_`. This tells your
Lotus node that it's okay if the genesis being used doesn't match any baked-in
genesis.
```sh
export LOTUS_SKIP_GENESIS_CHECK=_yes_
```
Download the 2048 byte parameters:
```sh
./lotus fetch-params 2048
```
Pre-seal some sectors:
```sh
./lotus-seed pre-seal --sector-size 2KiB --num-sectors 2
```
Create the genesis block and start up the first node:
```sh
./lotus-seed genesis new localnet.json
./lotus-seed genesis add-miner localnet.json ~/.genesis-sectors/pre-seal-t01000.json
./lotus daemon --lotus-make-genesis=devgen.car --genesis-template=localnet.json --bootstrap=false
```
Then, in another console, import the genesis miner key:
```sh
./lotus wallet import --as-default ~/.genesis-sectors/pre-seal-t01000.key
```
Set up the genesis miner:
```sh
./lotus-miner init --genesis-miner --actor=t01000 --sector-size=2KiB --pre-sealed-sectors=~/.genesis-sectors --pre-sealed-metadata=~/.genesis-sectors/pre-seal-t01000.json --nosync
```
Now, finally, start up the miner:
```sh
./lotus-miner run --nosync
```
If all went well, you will have your own local Lotus Devnet running.

39
documentation/en/miner-deals.md
View File

@ -1,39 +0,0 @@
# Information for Miners
Here is how a miner can get set up to accept storage deals. The first step is
to install a Lotus node and sync to the top of the chain.
## Set up an ask
```
lotus-miner set-price <price>
```
This command will set up your miner to accept deal proposals that meet the input price.
The price is inputted in FIL per GiB per epoch, and the default is 0.0000000005.
## Ensure you can be discovered
Clients need to be able to find you in order to make storage deals with you.
While there isn't necessarily anything you need to do to become discoverable, here are some things you can
try to check that people can connect to you.
To start off, make sure you are connected to at least some peers, and your port is
open and working.
### Connect to your own node
If you are in contact with someone else running Lotus, you can ask them to try connecting
to your node. To do so, provide them your peer ID, which you can get by running `lotus net id` on
your node.
They can then try running `lotus net findpeer <peerID>` to get your address(es), and can then
run `lotus net connect <address>` to connect to you. If successful, your node will now
appear on their peers list (run `lotus net peers` to check).
You can also check this by running a second instance of Lotus yourself.
### Query your own ask
A client should be able to find your ask by running `lotus client query-ask <minerID>`. If
someone is not able to retrieve your ask by doing so, then there is an issue with your node.

81
documentation/en/mining-lotus-worker.md
View File

@ -1,81 +0,0 @@
# Lotus Worker
The **Lotus Worker** is an extra process that can offload heavy processing tasks from your **Lotus Miner**. The sealing process automatically runs in the **Lotus Miner** process, but you can use the Worker on another machine communicating over a fast network to free up resources on the machine running the mining process.
## Note: Using the Lotus Worker from China
If you are trying to use `lotus-worker` from China. You should set this **environment variable** on your machine:
```sh
export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/"
```
## Get Started
Make sure that the `lotus-worker` is compiled and installed by running:
```sh
make lotus-worker
```
## Setting up the Miner
First, you will need to ensure your `lotus-miner`'s API is accessible over the network.
To do this, open up `~/.lotusminer/config.toml` (Or if you manually set `LOTUS_MINER_PATH`, look under that directory) and look for the API field.
Default config:
```toml
[API]
ListenAddress = "/ip4/127.0.0.1/tcp/2345/http"
RemoteListenAddress = "127.0.0.1:2345"
```
To make your node accessible over the local area network, you will need to determine your machines IP on the LAN, and change the `127.0.0.1` in the file to that address.
A more permissive and less secure option is to change it to `0.0.0.0`. This will allow anyone who can connect to your computer on that port to access the [API](https://lotu.sh/en+api). They will still need an auth token.
`RemoteListenAddress` must be set to an address which other nodes on your network will be able to reach.
Next, you will need to [create an authentication token](https://lotu.sh/en+api-scripting-support#generate-a-jwt-46). All Lotus APIs require authentication tokens to ensure your processes are as secure against attackers attempting to make unauthenticated requests to them.
### Connect the Lotus Worker
On the machine that will run `lotus-worker`, set the `MINER_API_INFO` environment variable to `TOKEN:MINER_NODE_MULTIADDR`. Where `TOKEN` is the token we created above, and `NIMER_NODE_MULTIADDR` is the `multiaddr` of the **Lotus Miner** API that was set in `config.toml`.
Once this is set, run:
```sh
lotus-worker run
```
If you are running multiple workers on the same host, you will need to specify the `--listen` flag and ensure each worker is on a different port.
To check that the **Lotus Worker** is connected to your **Lotus Miner**, run `lotus-miner sealing workers` and check that the remote worker count has increased.
```sh
why@computer ~/lotus> lotus-miner sealing workers
Worker 0, host computer
CPU: [ ] 0 core(s) in use
RAM: [|||||||||||||||||| ] 28% 18.1 GiB/62.7 GiB
VMEM: [|||||||||||||||||| ] 28% 18.1 GiB/62.7 GiB
GPU: GeForce RTX 2080, not used
Worker 1, host othercomputer
CPU: [ ] 0 core(s) in use
RAM: [|||||||||||||| ] 23% 14 GiB/62.7 GiB
VMEM: [|||||||||||||| ] 23% 14 GiB/62.7 GiB
GPU: GeForce RTX 2080, not used
```
### Running locally for manually managing process priority
You can also run the **Lotus Worker** on the same machine as your **Lotus Miner**, so you can manually manage the process priority.
To do so you have to first __disable all seal task types__ in the miner config. This is important to prevent conflicts between the two processes.
You can then run the miner on your local-loopback interface;
```sh
lotus-worker run
```

66
documentation/en/mining-troubleshooting.md
View File

@ -1,66 +0,0 @@
# Mining Troubleshooting
## Config: Filecoin Proof Parameters directory
If you want to put the **Filecoin Proof Parameters** in a different directory, use the following environment variable:
```sh
FIL_PROOFS_PARAMETER_CACHE
```
## Error: Can't acquire bellman.lock
The **Bellman** lockfile is created to lock a GPU for a process. This bug can occur when this file isn't properly cleaned up:
```sh
mining block failed: computing election proof: github.com/filecoin-project/lotus/miner.(*Miner).mineOne
```
This bug occurs when the miner can't acquire the `bellman.lock`. To fix it you need to stop the `lotus-miner` and remove `/tmp/bellman.lock`.
## Error: Failed to get api endpoint
```sh
lotus-miner info
# WARN main lotus-miner/main.go:73 failed to get api endpoint: (/Users/myrmidon/.lotusminer) %!w(*errors.errorString=&{API not running (no endpoint)}):
```
If you see this, that means your **Lotus Miner** isn't ready yet. You need to finish [syncing the chain](https://lotu.sh/en+join-testnet).
## Error: Your computer may not be fast enough
```sh
CAUTION: block production took longer than the block delay. Your computer may not be fast enough to keep up
```
If you see this, that means your computer is too slow and your blocks are not included in the chain, and you will not receive any rewards.
## Error: No space left on device
```sh
lotus-miner sectors pledge
# No space left on device (os error 28)
```
If you see this, that means `pledge-sector` wrote too much data to `$TMPDIR` which by default is the root partition (This is common for Linux setups). Usually your root partition does not get the largest partition of storage so you will need to change the environment variable to something else.
## Error: GPU unused
If you suspect that your GPU is not being used, first make sure it is properly configured as described in the [testing configuration page](hardware-mining.md). Once you've done that (and set the `BELLMAN_CUSTOM_GPU` as appropriate if necessary) you can verify your GPU is being used by running a quick lotus-bench benchmark.
First, to watch GPU utilization run `nvtop` in one terminal, then in a separate terminal, run:
```sh
make bench
./bench sealing --sector-size=2KiB
```
This process uses a fair amount of GPU, and generally takes ~4 minutes to complete. If you do not see any activity in nvtop from lotus during the entire process, it is likely something is misconfigured with your GPU.
## Checking Sync Progress
You can use this command to check how far behind you are on syncing:
```sh
date -d @$(./lotus chain getblock $(./lotus chain head) | jq .Timestamp)
```

149
documentation/en/mining.md
View File

@ -1,149 +0,0 @@
# Storage Mining
Here are instructions to learn how to perform storage mining. For hardware specifications please read [this](https://lotu.sh/en+hardware-mining).
It is useful to [join the Testnet](https://lotu.sh/en+join-testnet) prior to attempting storage mining for the first time.
## Note: Using the Lotus Miner from China
If you are trying to use `lotus-miner` from China. You should set this **environment variable** on your machine.
```sh
export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/"
```
## Get started
Please ensure that at least one **BLS address** (starts with `t3`) in your wallet exists with the following command:
```sh
lotus wallet list
```
If you do not have a bls address, create a new bls wallet:
```sh
lotus wallet new bls
```
With your wallet address:
- Visit the [faucet](http://spacerace.faucet.glif.io/)
- Paste the address you created under REQUEST.
- Press the Request button.
- Run `/lotus-miner init --owner=<blsAddress> --worker=<blsAddress>`
You will have to wait some time for this operation to complete.
## Mining
To mine:
```sh
lotus-miner run
```
If you are downloading **Filecoin Proof Parameters**, the download can take some time.
Get information about your miner:
```sh
lotus-miner info
# example: miner id `t0111`
```
**Seal** random data to start producing **PoSts**:
```sh
lotus-miner sectors pledge
```
- Warning: On Linux configurations, this command will write data to `$TMPDIR` which is not usually the largest partition. You should point the value to a larger partition if possible.
Get **miner power** and **sector usage**:
```sh
lotus state power
# returns total power
lotus state power <miner>
lotus state sectors <miner>
```
## Performance tuning
### `FIL_PROOFS_MAXIMIZE_CACHING=1` Environment variable
This env var can be used with `lotus-miner`, `lotus-worker`, and `lotus-bench` to make the precommit1 step faster at the cost of some memory use (1x sector size)
### `FIL_PROOFS_USE_GPU_COLUMN_BUILDER=1` Environment variable
This env var can be used with `lotus-miner`, `lotus-worker`, and `lotus-bench` to enable experimental precommit2 GPU acceleration
### Setting multiaddresses
Set multiaddresses for the miner to listen on in a miner's `config.toml` file
(by default, it is located at `~/.lotusminer/config.toml`). The `ListenAddresses` in this file should be interface listen addresses (usually `/ip4/0.0.0.0/tcp/PORT`), and the `AnnounceAddresses` should match the addresses being passed to `set-addrs`.
The addresses passed to `set-addrs` parameter in the commands below should be currently active and dialable; confirm they are before entering them.
Once the config file has been updated, set the on-chain record of the miner's listen addresses:
```
lotus-miner actor set-addrs <multiaddr_1> <multiaddr_2> ... <multiaddr_n>
```
This updates the `MinerInfo` object in the miner's actor, which will be looked up
when a client attempts to make a deal. Any number of addresses can be provided.
Example:
```
lotus-miner actor set-addrs /ip4/123.123.73.123/tcp/12345 /ip4/223.223.83.223/tcp/23456
```
# Separate address for windowPoSt messages
WindowPoSt is the mechanism through which storage is verified in Filecoin. It requires miners to submit proofs for all sectors every 24h, which require sending messages to the chain.
Because many other mining related actions require sending messages to the chain, and not all of those are "high value", it may be desirable to use a separate account to send PoSt messages from. This allows for setting lower GasFeeCaps on the lower value messages without creating head-of-line blocking problems for the PoSt messages in congested chain conditions
To set this up, first create a new account, and send it some funds for gas fees:
```sh
lotus wallet new bls
t3defg...
lotus send t3defg... 100
```
Next add the control address
```sh
lotus-miner actor control set t3defg...
Add t3defg...
Pass --really-do-it to actually execute this action
```
Now actually set the addresses
```sh
lotus-miner actor control set --really-do-it t3defg...
Add t3defg...
Message CID: bafy2..
```
Wait for the message to land on chain
```sh
lotus state wait-msg bafy2..
...
Exit Code: 0
...
```
Check miner control address list to make sure the address was correctly setup
```sh
lotus-miner actor control list
name ID key use balance
owner t01111 t3abcd... other 300 FIL
worker t01111 t3abcd... other 300 FIL
control-0 t02222 t3defg... post 100 FIL
```

111
documentation/en/payment-channels.md
View File

@ -1,111 +0,0 @@
# Payment Channels
Payment channels are used to transfer funds between two actors.
For example in lotus a payment channel is created when a client wants to fetch data from a provider.
The client sends vouchers for the payment channel, and the provider sends data in response.
The payment channel is created on-chain with an initial amount.
Vouchers allow the client and the provider to exchange funds incrementally off-chain.
The provider can submit vouchers to chain at any stage.
Either party to the payment channel can settle the payment channel on chain.
After a settlement period (currently 12 hours) either party to the payment channel can call collect on chain.
Collect sends the value of submitted vouchers to the channel recipient (the provider), and refunds the remaining channel balance to the channel creator (the client).
Vouchers have a lane, a nonce and a value, where vouchers with a higher nonce supersede vouchers with a lower nonce in the same lane.
Each deal is created on a different lane.
Note that payment channels and vouchers can be used for any situation in which two parties need to incrementally transfer value between each other off-chain.
## Using the CLI
For example a client creates a payment channel to a provider with value 10 FIL.
```sh
$ lotus paych add-funds <client addr> <provider addr> 10
<channel addr>
```
The client creates a voucher in lane 0 (implied) with nonce 1 (implied) and value 2.
```sh
$ lotus paych voucher create <channel addr> 2
<voucher>
```
The client sends the voucher to the provider and the provider adds the voucher to their local store.
```sh
$ lotus paych voucher add <channel addr> <voucher>
```
The provider sends some data to the client.
The client creates a voucher in lane 0 (implied) with nonce 2 (implied) and value 4.
```sh
$ lotus paych voucher create <channel addr> 4
<voucher>
```
The client sends the voucher to the provider and the provider adds the voucher and sends back more data.
etc.
The client can add value to the channel after it has been created by calling `paych add-funds` with the same client and provider addresses.
```sh
$ lotus paych add-funds <client addr> <provider addr> 5
<channel addr> # Same address as above. Channel now has 15
```
Once the client has received all their data, they may settle the channel.
Note that settlement doesn't have to be done immediately.
For example the client may keep the channel open as long as it wants to continue making deals with the provider.
```sh
$ lotus paych settle <channel addr>
```
The provider can submit vouchers to chain (note that lotus does this automatically when it sees a settle message appear on chain).
The provider may have received many vouchers with incrementally higher values.
The provider should submit the best vouchers. Note that there will be one best voucher for each lane.
```sh
$ lotus paych voucher best-spendable <channel addr>
<voucher>
<voucher>
<voucher>
$ lotus paych voucher submit <channel addr> <voucher>
```
Once the settlement period is over, either the client or provider can call collect to disburse funds.
```sh
$ lotus paych collect <channel addr>
```
Check the status of a channel that is still being created using `lotus paych status-by-from-to`.
```sh
$ lotus paych status-by-from-to <from addr> <to addr>
Creating channel
From: t3sb6xzvs6rhlziatagevxpp3dwapdolurtkpn4kyh3kgoo4tn5o7lutjqlsnvpceztlhxu3lzzfe34rvpsjgq
To: t1zip4sblhyrn4oxygzsm6nafbsynp2avmk3xafea
Pending Amt: 10000
Wait Sentinel: bafy2bzacedk2jidsyxcynusted35t5ipkhu2kpiodtwyjr3pimrhke6f5pqbm
```
Check the status of a channel that has been created using `lotus paych status`.
```sh
$ lotus paych status <channel addr>
Channel exists
Channel: t2nydpzhmeqkmid5smtqnowlr2mr5az6rexpmyv6i
From: t3sb6xzvs6rhlziatagevxpp3dwapdolurtkpn4kyh3kgoo4tn5o7lutjqlsnvpceztlhxu3lzzfe34rvpsjgq
To: t1zip4sblhyrn4oxygzsm6nafbsynp2avmk3xafea
Confirmed Amt: 10000
Pending Amt: 6000
Queued Amt: 3000
Voucher Redeemed Amt: 2000
```

27
documentation/en/retrieving-data.md
View File

@ -1,27 +0,0 @@
# Retrieving Data
> There are recent bug reports with these instructions. If you happen to encounter any problems, please create a [GitHub issue](https://github.com/filecoin-project/lotus/issues/new) and a maintainer will address the problem as soon as they can.
Here are the operations you can perform after you have stored and sealed a **Data CID** with the **Lotus Miner** in the network.
If you would like to learn how to store a **Data CID** on a miner, read the instructions [here](https://lotu.sh/en+storing-data).
## Find by Data CID
```sh
lotus client find <Data CID>
# LOCAL
# RETRIEVAL <miner>@<miner peerId>-<deal funds>-<size>
```
## Retrieve by Data CID
All fields are required.
```sh
lotus client retrieve <Data CID> <outfile>
```
If the outfile does not exist it will be created in the Lotus repository directory.
This command will initiate a **retrieval deal** and write the data to your computer. This process may take 2 to 10 minutes.

54
documentation/en/setting-a-static-port.md
View File

@ -1,54 +0,0 @@
# Static Ports
Depending on how your network is set up, you may need to set a static port to successfully connect to peers to perform storage deals with your **Lotus Miner**.
## Setup
To change the random **swarm port**, you may edit the `config.toml` file located under `$LOTUS_MINER_PATH`. The default location of this file is `$HOME/.lotusminer`.
To change the port to `1347`:
```sh
[Libp2p]
ListenAddresses = ["/ip4/0.0.0.0/tcp/1347", "/ip6/::/tcp/1347"]
```
After changing the port value, restart your **daemon**.
## Announce Addresses
If the **swarm port** is port-forwarded from another address, it is possible to control what addresses
are announced to the network.
```sh
[Libp2p]
AnnounceAddresses = ["/ip4/<public-ip>/tcp/1347"]
```
If non-empty, this array specifies the swarm addresses to announce to the network. If empty, the daemon will announce inferred swarm addresses.
Similarly, it is possible to set `NoAnnounceAddresses` with an array of addresses to not announce to the network.
## Ubuntu's Uncomplicated Firewall
Open firewall manually:
```sh
ufw allow 1347/tcp
```
Or open and modify the profile located at `/etc/ufw/applications.d/lotus-daemon`:
```sh
[Lotus Daemon]
title=Lotus Daemon
description=Lotus Daemon firewall rules
ports=1347/tcp
```
Then run these commands:
```sh
ufw update lotus-daemon
ufw allow lotus-daemon
```

46
documentation/en/setup-troubleshooting.md
View File

@ -1,46 +0,0 @@
# Setup Troubleshooting
## Config: Clearing data
Here is a command that will delete your chain data, stored wallets, stored data and any miners you have set up:
```sh
rm -rf ~/.lotus ~/.lotusminer
```
This command usually resolves any issues with running `lotus` but it is not always required for updates. We will share information about when resetting your chain data and miners is required for an update in the future.
## Error: Failed to connect bootstrap peer
```sh
WARN peermgr peermgr/peermgr.go:131 failed to connect to bootstrap peer: failed to dial : all dials failed
* [/ip4/147.75.80.17/tcp/1347] failed to negotiate security protocol: connected to wrong peer
```
- Try running the build steps again and make sure that you have the latest code from GitHub.
```sh
ERROR hello hello/hello.go:81 other peer has different genesis!
```
- Try deleting your file system's `~/.lotus` directory. Check that it exists with `ls ~/.lotus`.
```sh
- repo is already locked
```
- You already have another lotus daemon running.
## Config: Open files limit
On most systems you can check the open files limit with:
```sh
ulimit -n
```
You can also modify this number by using the `ulimit` command. It gives you the ability to control the resources available for the shell or process started by it. If the number is below 10000, you can change it with the following command prior to starting the Lotus daemon:
```sh
ulimit -n 10000
```

27
documentation/en/storing-data-troubleshooting.md
View File

@ -1,27 +0,0 @@
# Storage Troubleshooting
## Error: Routing: not found
```sh
WARN main lotus/main.go:72 routing: not found
```
- This miner is offline.
## Error: Failed to start deal
```sh
WARN main lotus/main.go:72 failed to start deal: computing commP failed: generating CommP: Piece must be at least 127 bytes
```
- There is a minimum file size of 127 bytes.
## Error: 0kb file response during retrieval
In order to retrieve a file, it must be sealed. Miners can check sealing progress with this command:
```sh
lotus-miner sectors list
```
When sealing is complete, `pSet: NO` will become `pSet: YES`. From now on the **Data CID** is [retrievable](https://lotu.sh/en+retrieving-data) from the **Lotus Miner**.

62
documentation/en/storing-data.md
View File

@ -1,62 +0,0 @@
# Storing Data
> There are recent bug reports with these instructions. If you happen to encounter any problems, please create a [GitHub issue](https://github.com/filecoin-project/lotus/issues/new) and a maintainer will address the problem as soon as they can.
Here are instructions for how to store data on the **Lotus Testnet**.
## Adding a file locally
Adding a file locally allows you to make miner deals on the **Lotus Testnet**.
```sh
lotus client import ./your-example-file.txt
```
Upon success, this command will return a **Data CID**.
## List your local files
The command to see a list of files by `CID`, `name`, `size` in bytes, and `status`:
```sh
lotus client local
```
An example of the output:
```sh
bafkreierupr5ioxn4obwly4i2a5cd2rwxqi6kwmcyyylifxjsmos7hrgpe Development/sample-1.txt 2332 ok
bafkreieuk7h4zs5alzpdyhlph4lxkefowvwdho3a3pml6j7dam5mipzaii Development/sample-2.txt 30618 ok
```
## Make a Miner Deal on Lotus Testnet
Get a list of all miners that can store data:
```sh
lotus state list-miners
```
Get the requirements of a miner you wish to store data with:
```sh
lotus client query-ask <miner>
```
Store a **Data CID** with a miner:
```sh
lotus client deal <Data CID> <miner> <price> <duration>
```
Check the status of a deal:
```sh
lotus client list-deals
```
- The `duration`, which represents how long the miner will keep your file hosted, is represented in blocks. Each block represents 25 seconds.
Upon success, this command will return a **Deal CID**.
The miner will need to **seal** the file before it can be retrieved. If the **Lotus Miner** is not running on a machine designed for sealing, the process will take a very long time.

20
documentation/en/storing-ipfs-integration.md
View File

@ -1,20 +0,0 @@
# IPFS Integration
Lotus supports making deals with data stored in IPFS, without having to re-import it into lotus.
To enable this integration, you need to have an IPFS daemon running in the background.
Then, open up `~/.lotus/config.toml` (or if you manually set `LOTUS_PATH`, look under that directory)
and look for the Client field, and set `UseIpfs` to `true`.
```toml
[Client]
UseIpfs = true
```
After restarting the lotus daemon, you should be able to make deals with data in your IPFS node:
```sh
$ ipfs add -r SomeData
QmSomeData
$ ./lotus client deal QmSomeData t01000 0.0000000001 80000
```

14
documentation/en/updating-lotus.md
View File

@ -1,14 +0,0 @@
# Updating Lotus
If you installed Lotus on your machine, you can upgrade to the latest version by doing the following:
```sh
# get the latest
git pull origin master
# clean and remake the binaries
make clean && make build
# instal binaries in correct location
make install # or sudo make install if necessary
```

47
extern/sector-storage/ffiwrapper/sealer_cgo.go vendored
View File

@ -545,34 +545,37 @@ func (sb *Sealer) FinalizeSector(ctx context.Context, sector abi.SectorID, keepU
defer done()
pf, err := openPartialFile(maxPieceSize, paths.Unsealed)
if xerrors.Is(err, os.ErrNotExist) {
return xerrors.Errorf("opening partial file: %w", err)
}
if err == nil {
var at uint64
for sr.HasNext() {
r, err := sr.NextRun()
if err != nil {
_ = pf.Close()
return err
}
var at uint64
for sr.HasNext() {
r, err := sr.NextRun()
if err != nil {
_ = pf.Close()
offset := at
at += r.Len
if !r.Val {
continue
}
err = pf.Free(storiface.PaddedByteIndex(abi.UnpaddedPieceSize(offset).Padded()), abi.UnpaddedPieceSize(r.Len).Padded())
if err != nil {
_ = pf.Close()
return xerrors.Errorf("free partial file range: %w", err)
}
}
if err := pf.Close(); err != nil {
return err
}
offset := at
at += r.Len
if !r.Val {
continue
}
err = pf.Free(storiface.PaddedByteIndex(abi.UnpaddedPieceSize(offset).Padded()), abi.UnpaddedPieceSize(r.Len).Padded())
if err != nil {
_ = pf.Close()
return xerrors.Errorf("free partial file range: %w", err)
} else {
if !xerrors.Is(err, os.ErrNotExist) {
return xerrors.Errorf("opening partial file: %w", err)
}
}
if err := pf.Close(); err != nil {
return err
}
}
paths, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTCache, 0, storiface.PathStorage)

10
extern/sector-storage/worker_local.go vendored
View File

@ -32,6 +32,7 @@ var pathTypes = []storiface.SectorFileType{storiface.FTUnsealed, storiface.FTSea
type WorkerConfig struct {
SealProof abi.RegisteredSealProof
TaskTypes []sealtasks.TaskType
NoSwap bool
}
type LocalWorker struct {
@ -41,6 +42,7 @@ type LocalWorker struct {
sindex stores.SectorIndex
ret storiface.WorkerReturn
executor func() (ffiwrapper.Storage, error)
noSwap bool
ct *workerCallTracker
acceptTasks map[sealtasks.TaskType]struct{}
@ -69,6 +71,7 @@ func newLocalWorker(executor func() (ffiwrapper.Storage, error), wcfg WorkerConf
},
acceptTasks: acceptTasks,
executor: executor,
noSwap: wcfg.NoSwap,
closing: make(chan struct{}),
}
@ -445,11 +448,16 @@ func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error) {
return storiface.WorkerInfo{}, xerrors.Errorf("getting memory info: %w", err)
}
memSwap := mem.VirtualTotal
if l.noSwap {
memSwap = 0
}
return storiface.WorkerInfo{
Hostname: hostname,
Resources: storiface.WorkerResources{
MemPhysical: mem.Total,
MemSwap: mem.VirtualTotal,
MemSwap: memSwap,
MemReserved: mem.VirtualUsed + mem.Total - mem.Available, // TODO: sub this process
CPUs: uint64(runtime.NumCPU()),
GPUs: gpus,

8
extern/storage-sealing/states_failed.go vendored
View File

@ -38,16 +38,16 @@ func (m *Sealing) checkPreCommitted(ctx statemachine.Context, sector SectorInfo)
tok, _, err := m.api.ChainHead(ctx.Context())
if err != nil {
log.Errorf("handleSealPrecommit1Failed(%d): temp error: %+v", sector.SectorNumber, err)
return nil, true
return nil, false
}
info, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, tok)
if err != nil {
log.Errorf("handleSealPrecommit1Failed(%d): temp error: %+v", sector.SectorNumber, err)
return nil, true
return nil, false
}
return info, false
return info, true
}
func (m *Sealing) handleSealPrecommit1Failed(ctx statemachine.Context, sector SectorInfo) error {
@ -108,7 +108,7 @@ func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorI
}
if pci, is := m.checkPreCommitted(ctx, sector); is && pci != nil {
if sector.PreCommitMessage != nil {
if sector.PreCommitMessage == nil {
log.Warn("sector %d is precommitted on chain, but we don't have precommit message", sector.SectorNumber)
return ctx.Send(SectorPreCommitLanded{TipSet: tok})
}

2
extern/test-vectors vendored

@ -1 +1 @@
Subproject commit 6bea015edddde116001a4251dce3c4a9966c25d9
Subproject commit 3a6e0b5e069b1452ce1a032aa315354d645f3ec4

11
go.mod
View File

@ -11,6 +11,7 @@ require (
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129
github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b
github.com/coreos/go-systemd/v22 v22.0.0
github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e
github.com/dgraph-io/badger/v2 v2.2007.2
@ -22,7 +23,7 @@ require (
github.com/fatih/color v1.8.0
github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200716204036-cddc56607e1d
github.com/filecoin-project/go-address v0.0.4
github.com/filecoin-project/go-bitfield v0.2.0
github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
github.com/filecoin-project/go-data-transfer v0.6.6
@ -38,7 +39,7 @@ require (
github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
github.com/filecoin-project/specs-actors v0.9.11
github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796
github.com/filecoin-project/test-vectors/schema v0.0.1
github.com/filecoin-project/test-vectors/schema v0.0.3
github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
github.com/go-kit/kit v0.10.0
github.com/go-ole/go-ole v1.2.4 // indirect
@ -54,10 +55,11 @@ require (
github.com/ipfs/go-blockservice v0.1.4-0.20200624145336-a978cec6e834
github.com/ipfs/go-cid v0.0.7
github.com/ipfs/go-cidutil v0.0.2
github.com/ipfs/go-datastore v0.4.4
github.com/ipfs/go-datastore v0.4.5
github.com/ipfs/go-ds-badger2 v0.1.1-0.20200708190120-187fc06f714e
github.com/ipfs/go-ds-leveldb v0.4.2
github.com/ipfs/go-ds-measure v0.1.0
github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459
github.com/ipfs/go-filestore v1.0.0
github.com/ipfs/go-fs-lock v0.0.6
github.com/ipfs/go-graphsync v0.2.1
@ -126,10 +128,13 @@ require (
golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980
golang.org/x/time v0.0.0-20191024005414-555d28b269f0
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
gopkg.in/cheggaaa/pb.v1 v1.0.28
gotest.tools v2.2.0+incompatible
launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect
)
replace github.com/filecoin-project/lotus => ./
replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v1.18.0
replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi

31
go.sum
View File

@ -112,6 +112,8 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA=
github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
@ -125,6 +127,14 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs=
github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY=
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b h1:OKALTB609+19AM7wsO0k8yMwAqjEIppcnYvyIhA+ZlQ=
github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b/go.mod h1:hU7vhtrqonEphNF+xt8/lHdaBprxmV1h8BOGrd9XwmQ=
github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3 h1:2+dpIJzYMSbLi0587YXpi8tOJT52qCOI/1I0UNThc/I=
github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@ -220,6 +230,8 @@ github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 h1:t6qDiuGYYngDqaLc2ZUvdtAg4UN
github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoCcSuHN1XcbN0c6KBh7yvP5fs=
github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7mkT0qCJxzGI2Q=
github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3 h1:HQa4+yCYsLq1TLM0kopeAhSCLbtZ541cWEi5N5rO+9g=
github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8=
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
@ -260,8 +272,8 @@ github.com/filecoin-project/specs-actors v0.9.11 h1:TnpG7HAeiUrfj0mJM7UaPW0P2137
github.com/filecoin-project/specs-actors v0.9.11/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys=
github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk=
github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g=
github.com/filecoin-project/test-vectors/schema v0.0.1 h1:5fNF76nl4qolEvcIsjc0kUADlTMVHO73tW4kXXPnsus=
github.com/filecoin-project/test-vectors/schema v0.0.1/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E=
github.com/filecoin-project/test-vectors/schema v0.0.3 h1:1zuBo25B3016inbygYLgYFdpJ2m1BDTbAOCgABRleiU=
github.com/filecoin-project/test-vectors/schema v0.0.3/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as=
github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ=
@ -274,12 +286,16 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF6M5KiVP0=
github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1/go.mod h1:0eHX/BVySxPc6SE2mZRoppGq7qcEagxdmQnA3dzork8=
github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs=
github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo=
@ -341,6 +357,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf h1:gFVkHXmVAhEbxZVDln5V9GKrLaluNoFHDbrZwAWZgws=
github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@ -481,6 +499,8 @@ github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13X
github.com/ipfs/go-datastore v0.4.2/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
github.com/ipfs/go-datastore v0.4.4 h1:rjvQ9+muFaJ+QZ7dN5B1MSDNQ0JVZKkkES/rMZmA8X8=
github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
github.com/ipfs/go-datastore v0.4.5 h1:cwOUcGMLdLPWgu3SlrCckCMznaGADbPqE0r8h768/Dg=
github.com/ipfs/go-datastore v0.4.5/go.mod h1:eXTcaaiN6uOlVCLS9GjJUJtlvJfM3xk23w3fyfrmmJs=
github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk=
github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps=
github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8=
@ -499,6 +519,8 @@ github.com/ipfs/go-ds-leveldb v0.4.2 h1:QmQoAJ9WkPMUfBLnu1sBVy0xWWlJPg0m4kRAiJL9
github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s=
github.com/ipfs/go-ds-measure v0.1.0 h1:vE4TyY4aeLeVgnnPBC5QzKIjKrqzha0NCujTfgvVbVQ=
github.com/ipfs/go-ds-measure v0.1.0/go.mod h1:1nDiFrhLlwArTME1Ees2XaBOl49OoCgd2A3f8EchMSY=
github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459 h1:W3YMLEvOXqdW+sYMiguhWP6txJwQvIQqhvpU8yAMGQs=
github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459/go.mod h1:oh4liWHulKcDKVhCska5NLelE3MatWl+1FwSz3tY91g=
github.com/ipfs/go-filestore v1.0.0 h1:QR7ekKH+q2AGiWDc7W2Q0qHuYSRZGUJqUn0GsegEPb0=
github.com/ipfs/go-filestore v1.0.0/go.mod h1:/XOCuNtIe2f1YPbiXdYvD0BKLA0JR1MgPiFOdcuu9SM=
github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0=
@ -506,6 +528,7 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28
github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE=
github.com/ipfs/go-graphsync v0.2.1 h1:MdehhqBSuTI2LARfKLkpYnt0mUrqHs/mtuDnESXHBfU=
github.com/ipfs/go-graphsync v0.2.1/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10=
github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk=
github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk=
github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08=
github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw=
@ -1495,6 +1518,8 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd h1:zkO/Lhoka23X63N9OSzpSeROEUQ5ODw47tM3YWjygbs=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc=
golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -1513,6 +1538,7 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCc
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@ -1614,6 +1640,7 @@ golang.org/x/sys v0.0.0-20190902133755-9109b7679e13/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

2
node/builder.go
View File

@ -35,6 +35,7 @@ import (
"github.com/filecoin-project/lotus/chain/gen/slashfilter"
"github.com/filecoin-project/lotus/chain/market"
"github.com/filecoin-project/lotus/chain/messagepool"
"github.com/filecoin-project/lotus/chain/messagesigner"
"github.com/filecoin-project/lotus/chain/metrics"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
@ -260,6 +261,7 @@ func Online() Option {
Override(new(*store.ChainStore), modules.ChainStore),
Override(new(*stmgr.StateManager), stmgr.NewStateManager),
Override(new(*wallet.Wallet), wallet.NewWallet),
Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner),
Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker),
Override(new(dtypes.ChainGCBlockstore), modules.ChainGCBlockstore),

65
node/impl/full/mpool.go
View File

@ -10,8 +10,7 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/messagepool"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/messagesigner"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
@ -22,9 +21,7 @@ type MpoolAPI struct {
WalletAPI
GasAPI
Chain *store.ChainStore
Mpool *messagepool.MessagePool
MessageSigner *messagesigner.MessageSigner
PushLocks *dtypes.MpoolLocker
}
@ -114,12 +111,14 @@ func (a *MpoolAPI) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (ci
}
func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) {
cp := *msg
msg = &cp
inMsg := *msg
fromA, err := a.Stmgr.ResolveToKeyAddress(ctx, msg.From, nil)
if err != nil {
return nil, xerrors.Errorf("getting key address: %w", err)
}
{
fromA, err := a.Stmgr.ResolveToKeyAddress(ctx, msg.From, nil)
if err != nil {
return nil, xerrors.Errorf("getting key address: %w", err)
}
done, err := a.PushLocks.TakeLock(ctx, fromA)
if err != nil {
return nil, xerrors.Errorf("taking lock: %w", err)
@ -131,7 +130,7 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe
return nil, xerrors.Errorf("MpoolPushMessage expects message nonce to be 0, was %d", msg.Nonce)
}
msg, err := a.GasAPI.GasEstimateMessageGas(ctx, msg, spec, types.EmptyTSK)
msg, err = a.GasAPI.GasEstimateMessageGas(ctx, msg, spec, types.EmptyTSK)
if err != nil {
return nil, xerrors.Errorf("GasEstimateMessageGas error: %w", err)
}
@ -143,33 +142,31 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe
inJson, outJson)
}
sign := func(from address.Address, nonce uint64) (*types.SignedMessage, error) {
msg.Nonce = nonce
if msg.From.Protocol() == address.ID {
log.Warnf("Push from ID address (%s), adjusting to %s", msg.From, from)
msg.From = from
}
b, err := a.WalletBalance(ctx, msg.From)
if err != nil {
return nil, xerrors.Errorf("mpool push: getting origin balance: %w", err)
}
if b.LessThan(msg.Value) {
return nil, xerrors.Errorf("mpool push: not enough funds: %s < %s", b, msg.Value)
}
return a.WalletSignMessage(ctx, from, msg)
if msg.From.Protocol() == address.ID {
log.Warnf("Push from ID address (%s), adjusting to %s", msg.From, fromA)
msg.From = fromA
}
var m *types.SignedMessage
again:
m, err = a.Mpool.PushWithNonce(ctx, msg.From, sign)
if err == messagepool.ErrTryAgain {
log.Debugf("temporary failure while pushing message: %s; retrying", err)
goto again
b, err := a.WalletBalance(ctx, msg.From)
if err != nil {
return nil, xerrors.Errorf("mpool push: getting origin balance: %w", err)
}
return m, err
if b.LessThan(msg.Value) {
return nil, xerrors.Errorf("mpool push: not enough funds: %s < %s", b, msg.Value)
}
smsg, err := a.MessageSigner.SignMessage(ctx, msg)
if err != nil {
return nil, xerrors.Errorf("mpool push: failed to sign message: %w", err)
}
_, err = a.Mpool.Push(smsg)
if err != nil {
return nil, xerrors.Errorf("mpool push: failed to push message: %w", err)
}
return smsg, err
}
func (a *MpoolAPI) MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) {

19
node/impl/full/sync.go
View File

@ -126,3 +126,22 @@ func (a *SyncAPI) SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error
return reason, nil
}
func (a *SyncAPI) SyncValidateTipset(ctx context.Context, tsk types.TipSetKey) (bool, error) {
ts, err := a.Syncer.ChainStore().LoadTipSet(tsk)
if err != nil {
return false, err
}
fts, err := a.Syncer.ChainStore().TryFillTipSet(ts)
if err != nil {
return false, err
}
err = a.Syncer.ValidateTipSet(ctx, fts, false)
if err != nil {
return false, err
}
return true, nil
}

4
node/impl/full/wallet.go
View File

@ -90,3 +90,7 @@ func (a *WalletAPI) WalletImport(ctx context.Context, ki *types.KeyInfo) (addres
func (a *WalletAPI) WalletDelete(ctx context.Context, addr address.Address) error {
return a.Wallet.DeleteKey(addr)
}
func (a *WalletAPI) WalletValidateAddress(ctx context.Context, str string) (address.Address, error) {
return address.NewFromString(str)
}

34
node/repo/fsrepo.go
View File

@ -65,7 +65,8 @@ var ErrRepoExists = xerrors.New("repo exists")
// FsRepo is struct for repo, use NewFS to create
type FsRepo struct {
path string
path string
configPath string
}
var _ Repo = &FsRepo{}
@ -78,10 +79,15 @@ func NewFS(path string) (*FsRepo, error) {
}
return &FsRepo{
path: path,
path: path,
configPath: filepath.Join(path, fsConfig),
}, nil
}
func (fsr *FsRepo) SetConfigPath(cfgPath string) {
fsr.configPath = cfgPath
}
func (fsr *FsRepo) Exists() (bool, error) {
_, err := os.Stat(filepath.Join(fsr.path, fsDatastore))
notexist := os.IsNotExist(err)
@ -115,9 +121,7 @@ func (fsr *FsRepo) Init(t RepoType) error {
}
func (fsr *FsRepo) initConfig(t RepoType) error {
cfgP := filepath.Join(fsr.path, fsConfig)
_, err := os.Stat(cfgP)
_, err := os.Stat(fsr.configPath)
if err == nil {
// exists
return nil
@ -125,7 +129,7 @@ func (fsr *FsRepo) initConfig(t RepoType) error {
return err
}
c, err := os.Create(cfgP)
c, err := os.Create(fsr.configPath)
if err != nil {
return err
}
@ -215,16 +219,18 @@ func (fsr *FsRepo) Lock(repoType RepoType) (LockedRepo, error) {
return nil, xerrors.Errorf("could not lock the repo: %w", err)
}
return &fsLockedRepo{
path: fsr.path,
repoType: repoType,
closer: closer,
path: fsr.path,
configPath: fsr.configPath,
repoType: repoType,
closer: closer,
}, nil
}
type fsLockedRepo struct {
path string
repoType RepoType
closer io.Closer
path string
configPath string
repoType RepoType
closer io.Closer
ds map[string]datastore.Batching
dsErr error
@ -277,7 +283,7 @@ func (fsr *fsLockedRepo) Config() (interface{}, error) {
}
func (fsr *fsLockedRepo) loadConfigFromDisk() (interface{}, error) {
return config.FromFile(fsr.join(fsConfig), defConfForType(fsr.repoType))
return config.FromFile(fsr.configPath, defConfForType(fsr.repoType))
}
func (fsr *fsLockedRepo) SetConfig(c func(interface{})) error {
@ -306,7 +312,7 @@ func (fsr *fsLockedRepo) SetConfig(c func(interface{})) error {
}
// write buffer of TOML bytes to config file
err = ioutil.WriteFile(fsr.join(fsConfig), buf.Bytes(), 0644)
err = ioutil.WriteFile(fsr.configPath, buf.Bytes(), 0644)
if err != nil {
return err
}

537
storage/wdpost_changehandler.go Normal file
View File

@ -0,0 +1,537 @@
package storage
import (
"context"
"sync"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/go-state-types/dline"
"github.com/filecoin-project/lotus/chain/types"
)
const SubmitConfidence = 4
type CompleteGeneratePoSTCb func(posts []miner.SubmitWindowedPoStParams, err error)
type CompleteSubmitPoSTCb func(err error)
type changeHandlerAPI interface {
StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error)
startGeneratePoST(ctx context.Context, ts *types.TipSet, deadline *dline.Info, onComplete CompleteGeneratePoSTCb) context.CancelFunc
startSubmitPoST(ctx context.Context, ts *types.TipSet, deadline *dline.Info, posts []miner.SubmitWindowedPoStParams, onComplete CompleteSubmitPoSTCb) context.CancelFunc
onAbort(ts *types.TipSet, deadline *dline.Info)
failPost(err error, ts *types.TipSet, deadline *dline.Info)
}
type changeHandler struct {
api changeHandlerAPI
actor address.Address
proveHdlr *proveHandler
submitHdlr *submitHandler
}
func newChangeHandler(api changeHandlerAPI, actor address.Address) *changeHandler {
posts := newPostsCache()
p := newProver(api, posts)
s := newSubmitter(api, posts)
return &changeHandler{api: api, actor: actor, proveHdlr: p, submitHdlr: s}
}
func (ch *changeHandler) start() {
go ch.proveHdlr.run()
go ch.submitHdlr.run()
}
func (ch *changeHandler) update(ctx context.Context, revert *types.TipSet, advance *types.TipSet) error {
// Get the current deadline period
di, err := ch.api.StateMinerProvingDeadline(ctx, ch.actor, advance.Key())
if err != nil {
return err
}
if !di.PeriodStarted() {
return nil // not proving anything yet
}
hc := &headChange{
ctx: ctx,
revert: revert,
advance: advance,
di: di,
}
select {
case ch.proveHdlr.hcs <- hc:
case <-ch.proveHdlr.shutdownCtx.Done():
case <-ctx.Done():
}
select {
case ch.submitHdlr.hcs <- hc:
case <-ch.submitHdlr.shutdownCtx.Done():
case <-ctx.Done():
}
return nil
}
func (ch *changeHandler) shutdown() {
ch.proveHdlr.shutdown()
ch.submitHdlr.shutdown()
}
func (ch *changeHandler) currentTSDI() (*types.TipSet, *dline.Info) {
return ch.submitHdlr.currentTSDI()
}
// postsCache keeps a cache of PoSTs for each proving window
type postsCache struct {
added chan *postInfo
lk sync.RWMutex
cache map[abi.ChainEpoch][]miner.SubmitWindowedPoStParams
}
func newPostsCache() *postsCache {
return &postsCache{
added: make(chan *postInfo, 16),
cache: make(map[abi.ChainEpoch][]miner.SubmitWindowedPoStParams),
}
}
func (c *postsCache) add(di *dline.Info, posts []miner.SubmitWindowedPoStParams) {
c.lk.Lock()
defer c.lk.Unlock()
// TODO: clear cache entries older than chain finality
c.cache[di.Open] = posts
c.added <- &postInfo{
di: di,
posts: posts,
}
}
func (c *postsCache) get(di *dline.Info) ([]miner.SubmitWindowedPoStParams, bool) {
c.lk.RLock()
defer c.lk.RUnlock()
posts, ok := c.cache[di.Open]
return posts, ok
}
type headChange struct {
ctx context.Context
revert *types.TipSet
advance *types.TipSet
di *dline.Info
}
type currentPost struct {
di *dline.Info
abort context.CancelFunc
}
type postResult struct {
ts *types.TipSet
currPost *currentPost
posts []miner.SubmitWindowedPoStParams
err error
}
// proveHandler generates proofs
type proveHandler struct {
api changeHandlerAPI
posts *postsCache
postResults chan *postResult
hcs chan *headChange
current *currentPost
shutdownCtx context.Context
shutdown context.CancelFunc
// Used for testing
processedHeadChanges chan *headChange
processedPostResults chan *postResult
}
func newProver(
api changeHandlerAPI,
posts *postsCache,
) *proveHandler {
ctx, cancel := context.WithCancel(context.Background())
return &proveHandler{
api: api,
posts: posts,
postResults: make(chan *postResult),
hcs: make(chan *headChange),
shutdownCtx: ctx,
shutdown: cancel,
}
}
func (p *proveHandler) run() {
// Abort proving on shutdown
defer func() {
if p.current != nil {
p.current.abort()
}
}()
for p.shutdownCtx.Err() == nil {
select {
case <-p.shutdownCtx.Done():
return
case hc := <-p.hcs:
// Head changed
p.processHeadChange(hc.ctx, hc.advance, hc.di)
if p.processedHeadChanges != nil {
p.processedHeadChanges <- hc
}
case res := <-p.postResults:
// Proof generation complete
p.processPostResult(res)
if p.processedPostResults != nil {
p.processedPostResults <- res
}
}
}
}
func (p *proveHandler) processHeadChange(ctx context.Context, newTS *types.TipSet, di *dline.Info) {
// If the post window has expired, abort the current proof
if p.current != nil && newTS.Height() >= p.current.di.Close {
// Cancel the context on the current proof
p.current.abort()
// Clear out the reference to the proof so that we can immediately
// start generating a new proof, without having to worry about state
// getting clobbered when the abort completes
p.current = nil
}
// Only generate one proof at a time
if p.current != nil {
return
}
// If the proof for the current post window has been generated, check the
// next post window
_, complete := p.posts.get(di)
for complete {
di = nextDeadline(di)
_, complete = p.posts.get(di)
}
// Check if the chain is above the Challenge height for the post window
if newTS.Height() < di.Challenge {
return
}
p.current = &currentPost{di: di}
curr := p.current
p.current.abort = p.api.startGeneratePoST(ctx, newTS, di, func(posts []miner.SubmitWindowedPoStParams, err error) {
p.postResults <- &postResult{ts: newTS, currPost: curr, posts: posts, err: err}
})
}
func (p *proveHandler) processPostResult(res *postResult) {
di := res.currPost.di
if res.err != nil {
// Proving failed so inform the API
p.api.failPost(res.err, res.ts, di)
log.Warnf("Aborted window post Proving (Deadline: %+v)", di)
p.api.onAbort(res.ts, di)
// Check if the current post has already been aborted
if p.current == res.currPost {
// If the current post was not already aborted, setting it to nil
// marks it as complete so that a new post can be started
p.current = nil
}
return
}
// Completed processing this proving window
p.current = nil
// Add the proofs to the cache
p.posts.add(di, res.posts)
}
type submitResult struct {
pw *postWindow
err error
}
type SubmitState string
const (
SubmitStateStart SubmitState = "SubmitStateStart"
SubmitStateSubmitting SubmitState = "SubmitStateSubmitting"
SubmitStateComplete SubmitState = "SubmitStateComplete"
)
type postWindow struct {
ts *types.TipSet
di *dline.Info
submitState SubmitState
abort context.CancelFunc
}
type postInfo struct {
di *dline.Info
posts []miner.SubmitWindowedPoStParams
}
// submitHandler submits proofs on-chain
type submitHandler struct {
api changeHandlerAPI
posts *postsCache
submitResults chan *submitResult
hcs chan *headChange
postWindows map[abi.ChainEpoch]*postWindow
getPostWindowReqs chan *getPWReq
shutdownCtx context.Context
shutdown context.CancelFunc
currentCtx context.Context
currentTS *types.TipSet
currentDI *dline.Info
getTSDIReq chan chan *tsdi
// Used for testing
processedHeadChanges chan *headChange
processedSubmitResults chan *submitResult
processedPostReady chan *postInfo
}
func newSubmitter(
api changeHandlerAPI,
posts *postsCache,
) *submitHandler {
ctx, cancel := context.WithCancel(context.Background())
return &submitHandler{
api: api,
posts: posts,
submitResults: make(chan *submitResult),
hcs: make(chan *headChange),
postWindows: make(map[abi.ChainEpoch]*postWindow),
getPostWindowReqs: make(chan *getPWReq),
getTSDIReq: make(chan chan *tsdi),
shutdownCtx: ctx,
shutdown: cancel,
}
}
func (s *submitHandler) run() {
// On shutdown, abort in-progress submits
defer func() {
for _, pw := range s.postWindows {
if pw.abort != nil {
pw.abort()
}
}
}()
for s.shutdownCtx.Err() == nil {
select {
case <-s.shutdownCtx.Done():
return
case hc := <-s.hcs:
// Head change
s.processHeadChange(hc.ctx, hc.revert, hc.advance, hc.di)
if s.processedHeadChanges != nil {
s.processedHeadChanges <- hc
}
case pi := <-s.posts.added:
// Proof generated
s.processPostReady(pi)
if s.processedPostReady != nil {
s.processedPostReady <- pi
}
case res := <-s.submitResults:
// Submit complete
s.processSubmitResult(res)
if s.processedSubmitResults != nil {
s.processedSubmitResults <- res
}
case pwreq := <-s.getPostWindowReqs:
// used by getPostWindow() to sync with run loop
pwreq.out <- s.postWindows[pwreq.di.Open]
case out := <-s.getTSDIReq:
// used by currentTSDI() to sync with run loop
out <- &tsdi{ts: s.currentTS, di: s.currentDI}
}
}
}
// processHeadChange is called when the chain head changes
func (s *submitHandler) processHeadChange(ctx context.Context, revert *types.TipSet, advance *types.TipSet, di *dline.Info) {
s.currentCtx = ctx
s.currentTS = advance
s.currentDI = di
// Start tracking the current post window if we're not already
// TODO: clear post windows older than chain finality
if _, ok := s.postWindows[di.Open]; !ok {
s.postWindows[di.Open] = &postWindow{
di: di,
ts: advance,
submitState: SubmitStateStart,
}
}
// Apply the change to all post windows
for _, pw := range s.postWindows {
s.processHeadChangeForPW(ctx, revert, advance, pw)
}
}
func (s *submitHandler) processHeadChangeForPW(ctx context.Context, revert *types.TipSet, advance *types.TipSet, pw *postWindow) {
revertedToPrevDL := revert != nil && revert.Height() < pw.di.Open
expired := advance.Height() >= pw.di.Close
// If the chain was reverted back to the previous deadline, or if the post
// window has expired, abort submit
if pw.submitState == SubmitStateSubmitting && (revertedToPrevDL || expired) {
// Replace the aborted postWindow with a new one so that we can
// submit again at any time without the state getting clobbered
// when the abort completes
abort := pw.abort
if abort != nil {
pw = &postWindow{
di: pw.di,
ts: advance,
submitState: SubmitStateStart,
}
s.postWindows[pw.di.Open] = pw
// Abort the current submit
abort()
}
} else if pw.submitState == SubmitStateComplete && revertedToPrevDL {
// If submit for this deadline has completed, but the chain was
// reverted back to the previous deadline, reset the submit state to the
// starting state, so that it can be resubmitted
pw.submitState = SubmitStateStart
}
// Submit the proof to chain if the proof has been generated and the chain
// height is above confidence
s.submitIfReady(ctx, advance, pw)
}
// processPostReady is called when a proof generation completes
func (s *submitHandler) processPostReady(pi *postInfo) {
pw, ok := s.postWindows[pi.di.Open]
if ok {
s.submitIfReady(s.currentCtx, s.currentTS, pw)
}
}
// submitIfReady submits a proof if the chain is high enough and the proof
// has been generated for this deadline
func (s *submitHandler) submitIfReady(ctx context.Context, advance *types.TipSet, pw *postWindow) {
// If the window has expired, there's nothing more to do.
if advance.Height() >= pw.di.Close {
return
}
// Check if we're already submitting, or already completed submit
if pw.submitState != SubmitStateStart {
return
}
// Check if we've reached the confidence height to submit
if advance.Height() < pw.di.Open+SubmitConfidence {
return
}
// Check if the proofs have been generated for this deadline
posts, ok := s.posts.get(pw.di)
if !ok {
return
}
// If there was nothing to prove, move straight to the complete state
if len(posts) == 0 {
pw.submitState = SubmitStateComplete
return
}
// Start submitting post
pw.submitState = SubmitStateSubmitting
pw.abort = s.api.startSubmitPoST(ctx, advance, pw.di, posts, func(err error) {
s.submitResults <- &submitResult{pw: pw, err: err}
})
}
// processSubmitResult is called with the response to a submit
func (s *submitHandler) processSubmitResult(res *submitResult) {
if res.err != nil {
// Submit failed so inform the API and go back to the start state
s.api.failPost(res.err, res.pw.ts, res.pw.di)
log.Warnf("Aborted window post Submitting (Deadline: %+v)", res.pw.di)
s.api.onAbort(res.pw.ts, res.pw.di)
res.pw.submitState = SubmitStateStart
return
}
// Submit succeeded so move to complete state
res.pw.submitState = SubmitStateComplete
}
type tsdi struct {
ts *types.TipSet
di *dline.Info
}
func (s *submitHandler) currentTSDI() (*types.TipSet, *dline.Info) {
out := make(chan *tsdi)
s.getTSDIReq <- out
res := <-out
return res.ts, res.di
}
type getPWReq struct {
di *dline.Info
out chan *postWindow
}
func (s *submitHandler) getPostWindow(di *dline.Info) *postWindow {
out := make(chan *postWindow)
s.getPostWindowReqs <- &getPWReq{di: di, out: out}
return <-out
}
// nextDeadline gets deadline info for the subsequent deadline
func nextDeadline(currentDeadline *dline.Info) *dline.Info {
periodStart := currentDeadline.PeriodStart
newDeadline := currentDeadline.Index + 1
if newDeadline == miner.WPoStPeriodDeadlines {
newDeadline = 0
periodStart = periodStart + miner.WPoStProvingPeriod
}
return NewDeadlineInfo(periodStart, newDeadline, currentDeadline.CurrentEpoch)
}
func NewDeadlineInfo(periodStart abi.ChainEpoch, deadlineIdx uint64, currEpoch abi.ChainEpoch) *dline.Info {
return dline.NewInfo(periodStart, deadlineIdx, currEpoch, miner.WPoStPeriodDeadlines, miner.WPoStProvingPeriod, miner.WPoStChallengeWindow, miner.WPoStChallengeLookback, miner.FaultDeclarationCutoff)
}

1173
storage/wdpost_changehandler_test.go Normal file
View File

File diff suppressed because it is too large Load Diff

38
storage/wdpost_nextdl_test.go Normal file
View File

@ -0,0 +1,38 @@
package storage
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
)
func TestNextDeadline(t *testing.T) {
periodStart := abi.ChainEpoch(0)
deadlineIdx := 0
currentEpoch := abi.ChainEpoch(10)
di := NewDeadlineInfo(periodStart, uint64(deadlineIdx), currentEpoch)
require.EqualValues(t, 0, di.Index)
require.EqualValues(t, 0, di.PeriodStart)
require.EqualValues(t, -20, di.Challenge)
require.EqualValues(t, 0, di.Open)
require.EqualValues(t, 60, di.Close)
for i := 1; i < 1+int(miner.WPoStPeriodDeadlines)*2; i++ {
di = nextDeadline(di)
deadlineIdx = i % int(miner.WPoStPeriodDeadlines)
expPeriodStart := int(miner.WPoStProvingPeriod) * (i / int(miner.WPoStPeriodDeadlines))
expOpen := expPeriodStart + deadlineIdx*int(miner.WPoStChallengeWindow)
expClose := expOpen + int(miner.WPoStChallengeWindow)
expChallenge := expOpen - int(miner.WPoStChallengeLookback)
//fmt.Printf("%d: %d@%d %d-%d (%d)\n", i, expPeriodStart, deadlineIdx, expOpen, expClose, expChallenge)
require.EqualValues(t, deadlineIdx, di.Index)
require.EqualValues(t, expPeriodStart, di.PeriodStart)
require.EqualValues(t, expOpen, di.Open)
require.EqualValues(t, expClose, di.Close)
require.EqualValues(t, expChallenge, di.Challenge)
}
}

193
storage/wdpost_run.go
View File

@ -29,15 +29,21 @@ import (
"github.com/filecoin-project/lotus/journal"
)
func (s *WindowPoStScheduler) failPost(err error, deadline *dline.Info) {
func (s *WindowPoStScheduler) failPost(err error, ts *types.TipSet, deadline *dline.Info) {
journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} {
c := evtCommon{Error: err}
if ts != nil {
c.Deadline = deadline
c.Height = ts.Height()
c.TipSet = ts.Cids()
}
return WdPoStSchedulerEvt{
evtCommon: s.getEvtCommon(err),
evtCommon: c,
State: SchedulerStateFaulted,
}
})
log.Errorf("TODO")
log.Errorf("Got err %w - TODO handle errors", err)
/*s.failLk.Lock()
if eps > s.failed {
s.failed = eps
@ -45,67 +51,134 @@ func (s *WindowPoStScheduler) failPost(err error, deadline *dline.Info) {
s.failLk.Unlock()*/
}
func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *dline.Info, ts *types.TipSet) {
ctx, abort := context.WithCancel(ctx)
s.abort = abort
s.activeDeadline = deadline
journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} {
return WdPoStSchedulerEvt{
evtCommon: s.getEvtCommon(nil),
State: SchedulerStateStarted,
// recordProofsEvent records a successful proofs_processed event in the
// journal, even if it was a noop (no partitions).
func (s *WindowPoStScheduler) recordProofsEvent(partitions []miner.PoStPartition, mcid cid.Cid) {
journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStProofs], func() interface{} {
return &WdPoStProofsProcessedEvt{
evtCommon: s.getEvtCommon(nil),
Partitions: partitions,
MessageCID: mcid,
}
})
}
// startGeneratePoST kicks off the process of generating a PoST
func (s *WindowPoStScheduler) startGeneratePoST(
ctx context.Context,
ts *types.TipSet,
deadline *dline.Info,
completeGeneratePoST CompleteGeneratePoSTCb,
) context.CancelFunc {
ctx, abort := context.WithCancel(ctx)
go func() {
defer abort()
ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.doPost")
defer span.End()
// recordProofsEvent records a successful proofs_processed event in the
// journal, even if it was a noop (no partitions).
recordProofsEvent := func(partitions []miner.PoStPartition, mcid cid.Cid) {
journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStProofs], func() interface{} {
return &WdPoStProofsProcessedEvt{
evtCommon: s.getEvtCommon(nil),
Partitions: partitions,
MessageCID: mcid,
}
})
}
posts, err := s.runPost(ctx, *deadline, ts)
if err != nil {
log.Errorf("run window post failed: %+v", err)
s.failPost(err, deadline)
return
}
if len(posts) == 0 {
recordProofsEvent(nil, cid.Undef)
return
}
for i := range posts {
post := &posts[i]
sm, err := s.submitPost(ctx, post)
if err != nil {
log.Errorf("submit window post failed: %+v", err)
s.failPost(err, deadline)
} else {
recordProofsEvent(post.Partitions, sm.Cid())
}
}
journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} {
return WdPoStSchedulerEvt{
evtCommon: s.getEvtCommon(nil),
State: SchedulerStateSucceeded,
State: SchedulerStateStarted,
}
})
posts, err := s.runGeneratePoST(ctx, ts, deadline)
completeGeneratePoST(posts, err)
}()
return abort
}
// runGeneratePoST generates the PoST
func (s *WindowPoStScheduler) runGeneratePoST(
ctx context.Context,
ts *types.TipSet,
deadline *dline.Info,
) ([]miner.SubmitWindowedPoStParams, error) {
ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.generatePoST")
defer span.End()
posts, err := s.runPost(ctx, *deadline, ts)
if err != nil {
log.Errorf("runPost failed: %+v", err)
return nil, err
}
if len(posts) == 0 {
s.recordProofsEvent(nil, cid.Undef)
}
return posts, nil
}
// startSubmitPoST kicks of the process of submitting PoST
func (s *WindowPoStScheduler) startSubmitPoST(
ctx context.Context,
ts *types.TipSet,
deadline *dline.Info,
posts []miner.SubmitWindowedPoStParams,
completeSubmitPoST CompleteSubmitPoSTCb,
) context.CancelFunc {
ctx, abort := context.WithCancel(ctx)
go func() {
defer abort()
err := s.runSubmitPoST(ctx, ts, deadline, posts)
if err == nil {
journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} {
return WdPoStSchedulerEvt{
evtCommon: s.getEvtCommon(nil),
State: SchedulerStateSucceeded,
}
})
}
completeSubmitPoST(err)
}()
return abort
}
// runSubmitPoST submits PoST
func (s *WindowPoStScheduler) runSubmitPoST(
ctx context.Context,
ts *types.TipSet,
deadline *dline.Info,
posts []miner.SubmitWindowedPoStParams,
) error {
if len(posts) == 0 {
return nil
}
ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.submitPoST")
defer span.End()
// Get randomness from tickets
commEpoch := deadline.Open
commRand, err := s.api.ChainGetRandomnessFromTickets(ctx, ts.Key(), crypto.DomainSeparationTag_PoStChainCommit, commEpoch, nil)
if err != nil {
err = xerrors.Errorf("failed to get chain randomness from tickets for windowPost (ts=%d; deadline=%d): %w", ts.Height(), commEpoch, err)
log.Errorf("submitPost failed: %+v", err)
return err
}
var submitErr error
for i := range posts {
// Add randomness to PoST
post := &posts[i]
post.ChainCommitEpoch = commEpoch
post.ChainCommitRand = commRand
// Submit PoST
sm, submitErr := s.submitPost(ctx, post)
if submitErr != nil {
log.Errorf("submit window post failed: %+v", submitErr)
} else {
s.recordProofsEvent(post.Partitions, sm.Cid())
}
}
return submitErr
}
func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check bitfield.BitField) (bitfield.BitField, error) {
@ -396,7 +469,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty
rand, err := s.api.ChainGetRandomnessFromBeacon(ctx, ts.Key(), crypto.DomainSeparationTag_WindowedPoStChallengeSeed, di.Challenge, buf.Bytes())
if err != nil {
return nil, xerrors.Errorf("failed to get chain randomness for window post (ts=%d; deadline=%d): %w", ts.Height(), di, err)
return nil, xerrors.Errorf("failed to get chain randomness from beacon for window post (ts=%d; deadline=%d): %w", ts.Height(), di, err)
}
// Get the partitions for the given deadline
@ -540,19 +613,6 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty
posts = append(posts, params)
}
// Compute randomness after generating proofs so as to reduce the impact
// of chain reorgs (which change randomness)
commEpoch := di.Open
commRand, err := s.api.ChainGetRandomnessFromTickets(ctx, ts.Key(), crypto.DomainSeparationTag_PoStChainCommit, commEpoch, nil)
if err != nil {
return nil, xerrors.Errorf("failed to get chain randomness for window post (ts=%d; deadline=%d): %w", ts.Height(), commEpoch, err)
}
for i := range posts {
posts[i].ChainCommitEpoch = commEpoch
posts[i].ChainCommitRand = commRand
}
return posts, nil
}
@ -593,6 +653,7 @@ func (s *WindowPoStScheduler) batchPartitions(partitions []api.Partition) ([][]a
}
batches = append(batches, partitions[i:end])
}
return batches, nil
}

6
storage/wdpost_run_test.go
View File

@ -11,6 +11,7 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/crypto"
@ -177,7 +178,10 @@ func TestWDPostDoPost(t *testing.T) {
FaultDeclarationCutoff: miner0.FaultDeclarationCutoff,
}
ts := mockTipSet(t)
scheduler.doPost(ctx, di, ts)
scheduler.startGeneratePoST(ctx, ts, di, func(posts []miner.SubmitWindowedPoStParams, err error) {
scheduler.startSubmitPoST(ctx, ts, di, posts, func(err error) {})
})
// Read the window PoST messages
for i := 0; i < expectedMsgCount; i++ {

145
storage/wdpost_sched.go
View File

@ -22,8 +22,6 @@ import (
"go.opencensus.io/trace"
)
const StartConfidence = 4 // TODO: config
type WindowPoStScheduler struct {
api storageMinerApi
feeCfg config.MinerFeeConfig
@ -31,16 +29,11 @@ type WindowPoStScheduler struct {
faultTracker sectorstorage.FaultTracker
proofType abi.RegisteredPoStProof
partitionSectors uint64
ch *changeHandler
actor address.Address
worker address.Address
cur *types.TipSet
// if a post is in progress, this indicates for which ElectionPeriodStart
activeDeadline *dline.Info
abort context.CancelFunc
evtTypes [4]journal.EventType
// failed abi.ChainEpoch // eps
@ -77,16 +70,17 @@ func NewWindowedPoStScheduler(api storageMinerApi, fc config.MinerFeeConfig, sb
}, nil
}
func deadlineEquals(a, b *dline.Info) bool {
if a == nil || b == nil {
return b == a
}
return a.PeriodStart == b.PeriodStart && a.Index == b.Index && a.Challenge == b.Challenge
type changeHandlerAPIImpl struct {
storageMinerApi
*WindowPoStScheduler
}
func (s *WindowPoStScheduler) Run(ctx context.Context) {
defer s.abortActivePoSt()
// Initialize change handler
chImpl := &changeHandlerAPIImpl{storageMinerApi: s.api, WindowPoStScheduler: s}
s.ch = newChangeHandler(chImpl, s.actor)
defer s.ch.shutdown()
s.ch.start()
var notifs <-chan []*api.HeadChange
var err error
@ -125,17 +119,18 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) {
continue
}
if err := s.update(ctx, chg.Val); err != nil {
log.Errorf("%+v", err)
}
ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.headChange")
s.update(ctx, nil, chg.Val)
span.End()
gotCur = true
continue
}
ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.headChange")
var lowest, highest *types.TipSet = s.cur, nil
var lowest, highest *types.TipSet = nil, nil
for _, change := range changes {
if change.Val == nil {
@ -149,12 +144,7 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) {
}
}
if err := s.revert(ctx, lowest); err != nil {
log.Error("handling head reverts in window post sched: %+v", err)
}
if err := s.update(ctx, highest); err != nil {
log.Error("handling head updates in window post sched: %+v", err)
}
s.update(ctx, lowest, highest)
span.End()
case <-ctx.Done():
@ -163,95 +153,40 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) {
}
}
func (s *WindowPoStScheduler) revert(ctx context.Context, newLowest *types.TipSet) error {
if s.cur == newLowest {
return nil
func (s *WindowPoStScheduler) update(ctx context.Context, revert, apply *types.TipSet) {
if apply == nil {
log.Error("no new tipset in window post WindowPoStScheduler.update")
return
}
s.cur = newLowest
newDeadline, err := s.api.StateMinerProvingDeadline(ctx, s.actor, newLowest.Key())
err := s.ch.update(ctx, revert, apply)
if err != nil {
return err
log.Errorf("handling head updates in window post sched: %+v", err)
}
if !deadlineEquals(s.activeDeadline, newDeadline) {
s.abortActivePoSt()
}
return nil
}
func (s *WindowPoStScheduler) update(ctx context.Context, new *types.TipSet) error {
if new == nil {
return xerrors.Errorf("no new tipset in window post sched update")
}
di, err := s.api.StateMinerProvingDeadline(ctx, s.actor, new.Key())
if err != nil {
return err
}
if deadlineEquals(s.activeDeadline, di) {
return nil // already working on this deadline
}
if !di.PeriodStarted() {
return nil // not proving anything yet
}
s.abortActivePoSt()
// TODO: wait for di.Challenge here, will give us ~10min more to compute windowpost
// (Need to get correct deadline above, which is tricky)
if di.Open+StartConfidence >= new.Height() {
log.Info("not starting window post yet, waiting for startconfidence", di.Open, di.Open+StartConfidence, new.Height())
return nil
}
/*s.failLk.Lock()
if s.failed > 0 {
s.failed = 0
s.activeEPS = 0
}
s.failLk.Unlock()*/
log.Infof("at %d, do window post for P %d, dd %d", new.Height(), di.PeriodStart, di.Index)
s.doPost(ctx, di, new)
return nil
// onAbort is called when generating proofs or submitting proofs is aborted
func (s *WindowPoStScheduler) onAbort(ts *types.TipSet, deadline *dline.Info) {
journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} {
c := evtCommon{}
if ts != nil {
c.Deadline = deadline
c.Height = ts.Height()
c.TipSet = ts.Cids()
}
return WdPoStSchedulerEvt{
evtCommon: c,
State: SchedulerStateAborted,
}
})
}
func (s *WindowPoStScheduler) abortActivePoSt() {
if s.activeDeadline == nil {
return // noop
}
if s.abort != nil {
s.abort()
journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} {
return WdPoStSchedulerEvt{
evtCommon: s.getEvtCommon(nil),
State: SchedulerStateAborted,
}
})
log.Warnf("Aborting window post (Deadline: %+v)", s.activeDeadline)
}
s.activeDeadline = nil
s.abort = nil
}
// getEvtCommon populates and returns common attributes from state, for a
// WdPoSt journal event.
func (s *WindowPoStScheduler) getEvtCommon(err error) evtCommon {
c := evtCommon{Error: err}
if s.cur != nil {
c.Deadline = s.activeDeadline
c.Height = s.cur.Height()
c.TipSet = s.cur.Cids()
currentTS, currentDeadline := s.ch.currentTSDI()
if currentTS != nil {
c.Deadline = currentDeadline
c.Height = currentTS.Height()
c.TipSet = currentTS.Cids()
}
return c
}

2
tools/dockers/docker-examples/README.md
View File

@ -11,7 +11,7 @@ In this `docker-examples/` directory are community-contributed Docker and Docker
- local node for a developer (`api-local-`)
- hosted endpoint for apps / multiple developers (`api-hosted-`)
- **For a local devnet or shared devnet**
- basic local devnet (also see [lotus docs on setting up a local devnet](https://lotu.sh/en+setup-local-dev-net))
- basic local devnet (also see [lotus docs on setting up a local devnet](https://docs.filecoin.io/build/local-devnet/))
- shared devnet