cerc-io/lotus
15
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #5679 from filecoin-project/next

Browse Source 
1.5.1 staging branch
This commit is contained in:
Łukasz Magiera 2021-03-10 17:21:56 +01:00 committed by GitHub
commit 1673362795
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
204 changed files with 10777 additions and 2602 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
.circleci/config.yml
View File

@ -2,6 +2,7 @@ version: 2.1
orbs:
go: gotest/tools@0.0.13
aws-cli: circleci/aws-cli@1.3.2
packer: salaxander/packer@0.0.3
executors:
golang:
@ -277,6 +278,11 @@ jobs:
- install-deps
- prepare
- run: make calibnet
- run: mkdir linux-calibnet && mv lotus lotus-miner lotus-worker linux-calibnet
- persist_to_workspace:
root: "."
paths:
- linux-calibnet
build-lotus-soup:
description: |
Compile `lotus-soup` Testground test plan
@ -289,7 +295,7 @@ jobs:
- run: cd extern/filecoin-ffi && make
- run:
name: "go get lotus@master"
command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master
command: cd testplans/lotus-soup && go mod edit -replace=github.com/filecoin-project/lotus=../..
- run:
name: "build lotus-soup testplan"
command: pushd testplans/lotus-soup && go build -tags=testground .
@ -583,6 +589,22 @@ jobs:
docker push $<<parameters.account-url>>/<<parameters.repo>>:${tag}
done
publish-packer:
description: build and push AWS IAM and DigitalOcean droplet.
executor:
name: packer/default
packer-version: 1.6.6
steps:
- checkout
- attach_workspace:
at: "."
- packer/build:
template: tools/packer/lotus.pkr.hcl
args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG"
- packer/build:
template: tools/packer/lotus.pkr.hcl
args: "-var ci_workspace_bins=./linux-calibnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG"
workflows:
version: 2.1
ci:
@ -683,3 +705,15 @@ workflows:
path: .
repo: lotus-dev
tag: '${CIRCLE_SHA1:0:8}'
- publish-packer:
requires:
- build-all
- build-ntwk-calibration
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+$/

13
api/api_common.go
View File

@ -11,8 +11,6 @@ import (
"github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
protocol "github.com/libp2p/go-libp2p-core/protocol"
"github.com/filecoin-project/lotus/build"
)
type Common interface {
@ -33,6 +31,7 @@ type Common interface {
NetPubsubScores(context.Context) ([]PubsubScore, error)
NetAutoNatStatus(context.Context) (NatInfo, error)
NetAgentVersion(ctx context.Context, p peer.ID) (string, error)
NetPeerInfo(context.Context, peer.ID) (*ExtendedPeerInfo, error)
// NetBandwidthStats returns statistics about the nodes total bandwidth
// usage and current rate across all peers and protocols.
@ -57,7 +56,7 @@ type Common interface {
ID(context.Context) (peer.ID, error)
// Version provides information about API provider
Version(context.Context) (Version, error)
Version(context.Context) (APIVersion, error)
LogList(context.Context) ([]string, error)
LogSetLevel(context.Context, string, string) error
@ -71,15 +70,15 @@ type Common interface {
Closing(context.Context) (<-chan struct{}, error)
}
// Version provides various build-time information
type Version struct {
// APIVersion provides various build-time information
type APIVersion struct {
Version string
// APIVersion is a binary encoded semver version of the remote implementing
// this api
//
// See APIVersion in build/version.go
APIVersion build.Version
APIVersion Version
// TODO: git commit / os / genesis cid?
@ -87,7 +86,7 @@ type Version struct {
BlockDelay uint64
}
func (v Version) String() string {
func (v APIVersion) String() string {
return fmt.Sprintf("%s+api%s", v.Version, v.APIVersion.String())
}

10
api/api_full.go
View File

@ -32,6 +32,14 @@ import (
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
//go:generate go run github.com/golang/mock/mockgen -destination=mocks/mock_full.go -package=mocks . FullNode
// ChainIO abstracts operations for accessing raw IPLD objects.
type ChainIO interface {
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
ChainHasObj(context.Context, cid.Cid) (bool, error)
}
// FullNode API is a low-level interface to the Filecoin network full node
type FullNode interface {
Common
@ -862,6 +870,8 @@ const (
func (v SyncStateStage) String() string {
switch v {
case StageIdle:
return "idle"
case StageHeaders:
return "header sync"
case StagePersistHeaders:

5
api/api_storage.go
View File

@ -36,7 +36,7 @@ type StorageMiner interface {
MiningBase(context.Context) (*types.TipSet, error)
// Temp api for testing
PledgeSector(context.Context) error
PledgeSector(context.Context) (abi.SectorID, error)
// Get the status of a given sector by ID
SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (SectorInfo, error)
@ -238,6 +238,9 @@ type AddressConfig struct {
PreCommitControl []address.Address
CommitControl []address.Address
TerminateControl []address.Address
DisableOwnerFallback bool
DisableWorkerFallback bool
}
// PendingDealInfo has info about pending deals and when they are due to be

4
api/api_worker.go
View File

@ -9,12 +9,10 @@ import (
"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
"github.com/filecoin-project/lotus/build"
)
type WorkerAPI interface {
Version(context.Context) (build.Version, error)
Version(context.Context) (Version, error)
// TODO: Info() (name, ...) ?
TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) // TaskType -> Weight

68
api/apibstore/apibstore.go
View File

@ -1,68 +0,0 @@
package apibstore
import (
"context"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/lib/blockstore"
)
type ChainIO interface {
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
ChainHasObj(context.Context, cid.Cid) (bool, error)
}
type apiBStore struct {
api ChainIO
}
func NewAPIBlockstore(cio ChainIO) blockstore.Blockstore {
return &apiBStore{
api: cio,
}
}
func (a *apiBStore) DeleteBlock(cid.Cid) error {
return xerrors.New("not supported")
}
func (a *apiBStore) Has(c cid.Cid) (bool, error) {
return a.api.ChainHasObj(context.TODO(), c)
}
func (a *apiBStore) Get(c cid.Cid) (blocks.Block, error) {
bb, err := a.api.ChainReadObj(context.TODO(), c)
if err != nil {
return nil, err
}
return blocks.NewBlockWithCid(bb, c)
}
func (a *apiBStore) GetSize(c cid.Cid) (int, error) {
bb, err := a.api.ChainReadObj(context.TODO(), c)
if err != nil {
return 0, err
}
return len(bb), nil
}
func (a *apiBStore) Put(blocks.Block) error {
return xerrors.New("not supported")
}
func (a *apiBStore) PutMany([]blocks.Block) error {
return xerrors.New("not supported")
}
func (a *apiBStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
return nil, xerrors.New("not supported")
}
func (a *apiBStore) HashOnRead(enabled bool) {
return
}
var _ blockstore.Blockstore = &apiBStore{}

18
api/apistruct/struct.go
View File

@ -33,7 +33,6 @@ import (
"github.com/filecoin-project/specs-storage/storage"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
"github.com/filecoin-project/lotus/chain/types"
@ -60,12 +59,13 @@ type CommonStruct struct {
NetBandwidthStatsByPeer func(ctx context.Context) (map[string]metrics.Stats, error) `perm:"read"`
NetBandwidthStatsByProtocol func(ctx context.Context) (map[protocol.ID]metrics.Stats, error) `perm:"read"`
NetAgentVersion func(ctx context.Context, p peer.ID) (string, error) `perm:"read"`
NetPeerInfo func(context.Context, peer.ID) (*api.ExtendedPeerInfo, error) `perm:"read"`
NetBlockAdd func(ctx context.Context, acl api.NetBlockList) error `perm:"admin"`
NetBlockRemove func(ctx context.Context, acl api.NetBlockList) error `perm:"admin"`
NetBlockList func(ctx context.Context) (api.NetBlockList, error) `perm:"read"`
ID func(context.Context) (peer.ID, error) `perm:"read"`
Version func(context.Context) (api.Version, error) `perm:"read"`
Version func(context.Context) (api.APIVersion, error) `perm:"read"`
LogList func(context.Context) ([]string, error) `perm:"write"`
LogSetLevel func(context.Context, string, string) error `perm:"write"`
@ -304,7 +304,7 @@ type StorageMinerStruct struct {
MarketPendingDeals func(ctx context.Context) (api.PendingDealInfo, error) `perm:"write"`
MarketPublishPendingDeals func(ctx context.Context) error `perm:"admin"`
PledgeSector func(context.Context) error `perm:"write"`
PledgeSector func(context.Context) (abi.SectorID, error) `perm:"write"`
SectorsStatus func(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (api.SectorInfo, error) `perm:"read"`
SectorsList func(context.Context) ([]abi.SectorNumber, error) `perm:"read"`
@ -389,7 +389,7 @@ type WorkerStruct struct {
Internal struct {
// TODO: lower perms
Version func(context.Context) (build.Version, error) `perm:"admin"`
Version func(context.Context) (api.Version, error) `perm:"admin"`
TaskTypes func(context.Context) (map[sealtasks.TaskType]struct{}, error) `perm:"admin"`
Paths func(context.Context) ([]stores.StoragePath, error) `perm:"admin"`
@ -540,13 +540,17 @@ func (c *CommonStruct) NetAgentVersion(ctx context.Context, p peer.ID) (string,
return c.Internal.NetAgentVersion(ctx, p)
}
func (c *CommonStruct) NetPeerInfo(ctx context.Context, p peer.ID) (*api.ExtendedPeerInfo, error) {
return c.Internal.NetPeerInfo(ctx, p)
}
// ID implements API.ID
func (c *CommonStruct) ID(ctx context.Context) (peer.ID, error) {
return c.Internal.ID(ctx)
}
// Version implements API.Version
func (c *CommonStruct) Version(ctx context.Context) (api.Version, error) {
func (c *CommonStruct) Version(ctx context.Context) (api.APIVersion, error) {
return c.Internal.Version(ctx)
}
@ -1274,7 +1278,7 @@ func (c *StorageMinerStruct) ActorAddressConfig(ctx context.Context) (api.Addres
return c.Internal.ActorAddressConfig(ctx)
}
func (c *StorageMinerStruct) PledgeSector(ctx context.Context) error {
func (c *StorageMinerStruct) PledgeSector(ctx context.Context) (abi.SectorID, error) {
return c.Internal.PledgeSector(ctx)
}
@ -1610,7 +1614,7 @@ func (c *StorageMinerStruct) CheckProvable(ctx context.Context, pp abi.Registere
// WorkerStruct
func (w *WorkerStruct) Version(ctx context.Context) (build.Version, error) {
func (w *WorkerStruct) Version(ctx context.Context) (api.Version, error) {
return w.Internal.Version(ctx)
}

4
api/docgen/docgen.go
View File

@ -113,7 +113,7 @@ func init() {
addExample(network.Connected)
addExample(dtypes.NetworkName("lotus"))
addExample(api.SyncStateStage(1))
addExample(build.FullAPIVersion)
addExample(api.FullAPIVersion)
addExample(api.PCHInbound)
addExample(time.Minute)
addExample(datatransfer.TransferID(3))
@ -123,6 +123,8 @@ func init() {
addExample(retrievalmarket.DealStatusNew)
addExample(network.ReachabilityPublic)
addExample(build.NewestNetworkVersion)
addExample(map[string]int{"name": 42})
addExample(map[string]time.Time{"name": time.Unix(1615243938, 0).UTC()})
addExample(&types.ExecutionTrace{
Msg: exampleValue("init", reflect.TypeOf(&types.Message{}), nil).(*types.Message),
MsgRct: exampleValue("init", reflect.TypeOf(&types.MessageReceipt{}), nil).(*types.MessageReceipt),

2987
api/mocks/mock_full.go Normal file
View File

File diff suppressed because it is too large Load Diff

4
api/test/paych.go
View File

@ -15,7 +15,7 @@ import (
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/apibstore"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/adt"
"github.com/filecoin-project/lotus/chain/actors/builtin"
@ -132,7 +132,7 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) {
t.Fatal("Unable to settle payment channel")
}
creatorStore := adt.WrapStore(ctx, cbor.NewCborStore(apibstore.NewAPIBlockstore(paymentCreator)))
creatorStore := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewAPIBlockstore(paymentCreator)))
// wait for the receiver to submit their vouchers
ev := events.NewEvents(ctx, paymentCreator)

19
api/test/tape.go
View File

@ -6,7 +6,6 @@ import (
"testing"
"time"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/network"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
@ -75,23 +74,9 @@ func testTapeFix(t *testing.T, b APIBuilder, blocktime time.Duration, after bool
<-done
}()
err = miner.PledgeSector(ctx)
sid, err := miner.PledgeSector(ctx)
require.NoError(t, err)
// Wait till done.
var sectorNo abi.SectorNumber
for {
s, err := miner.SectorsList(ctx) // Note - the test builder doesn't import genesis sectors into FSM
require.NoError(t, err)
fmt.Printf("Sectors: %d\n", len(s))
if len(s) == 1 {
sectorNo = s[0]
break
}
build.Clock.Sleep(100 * time.Millisecond)
}
fmt.Printf("All sectors is fsm\n")
// If before, we expect the precommit to fail
@ -103,7 +88,7 @@ func testTapeFix(t *testing.T, b APIBuilder, blocktime time.Duration, after bool
}
for {
st, err := miner.SectorsStatus(ctx, sectorNo, false)
st, err := miner.SectorsStatus(ctx, sid.Number, false)
require.NoError(t, err)
if st.State == successState {
break

6
api/test/test.go
View File

@ -155,13 +155,13 @@ var MineNext = miner.MineReq{
}
func (ts *testSuite) testVersion(t *testing.T) {
build.RunningNodeType = build.NodeFull
api.RunningNodeType = api.NodeFull
ctx := context.Background()
apis, _ := ts.makeNodes(t, OneFull, OneMiner)
api := apis[0]
napi := apis[0]
v, err := api.Version(ctx)
v, err := napi.Version(ctx)
if err != nil {
t.Fatal(err)
}

2
api/test/window_post.go
View File

@ -162,7 +162,7 @@ func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n,
log.Errorf("WAIT")
}
log.Errorf("PLEDGING %d", i)
err := miner.PledgeSector(ctx)
_, err := miner.PledgeSector(ctx)
require.NoError(t, err)
}

16
api/types.go
View File

@ -3,6 +3,7 @@ package api
import (
"encoding/json"
"fmt"
"time"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-state-types/abi"
@ -99,3 +100,18 @@ type NetBlockList struct {
IPAddrs []string
IPSubnets []string
}
type ExtendedPeerInfo struct {
ID peer.ID
Agent string
Addrs []string
Protocols []string
ConnMgrMeta *ConnMgrInfo
}
type ConnMgrInfo struct {
FirstSeen time.Time
Value int
Tags map[string]int
Conns map[string]time.Time
}

71
api/version.go Normal file
View File

@ -0,0 +1,71 @@
package api
import (
"fmt"
xerrors "golang.org/x/xerrors"
)
type Version uint32
func newVer(major, minor, patch uint8) Version {
return Version(uint32(major)<<16 | uint32(minor)<<8 | uint32(patch))
}
// Ints returns (major, minor, patch) versions
func (ve Version) Ints() (uint32, uint32, uint32) {
v := uint32(ve)
return (v & majorOnlyMask) >> 16, (v & minorOnlyMask) >> 8, v & patchOnlyMask
}
func (ve Version) String() string {
vmj, vmi, vp := ve.Ints()
return fmt.Sprintf("%d.%d.%d", vmj, vmi, vp)
}
func (ve Version) EqMajorMinor(v2 Version) bool {
return ve&minorMask == v2&minorMask
}
type NodeType int
const (
NodeUnknown NodeType = iota
NodeFull
NodeMiner
NodeWorker
)
var RunningNodeType NodeType
func VersionForType(nodeType NodeType) (Version, error) {
switch nodeType {
case NodeFull:
return FullAPIVersion, nil
case NodeMiner:
return MinerAPIVersion, nil
case NodeWorker:
return WorkerAPIVersion, nil
default:
return Version(0), xerrors.Errorf("unknown node type %d", nodeType)
}
}
// semver versions of the rpc api exposed
var (
FullAPIVersion = newVer(1, 1, 0)
MinerAPIVersion = newVer(1, 0, 1)
WorkerAPIVersion = newVer(1, 0, 0)
)
//nolint:varcheck,deadcode
const (
majorMask = 0xff0000
minorMask = 0xffff00
patchMask = 0xffffff
majorOnlyMask = 0xff0000
minorOnlyMask = 0x00ff00
patchOnlyMask = 0x0000ff
)

66
blockstore/api.go Normal file
View File

@ -0,0 +1,66 @@
package blockstore
import (
"context"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
)
type ChainIO interface {
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
ChainHasObj(context.Context, cid.Cid) (bool, error)
}
type apiBlockstore struct {
api ChainIO
}
// This blockstore is adapted in the constructor.
var _ BasicBlockstore = (*apiBlockstore)(nil)
func NewAPIBlockstore(cio ChainIO) Blockstore {
bs := &apiBlockstore{api: cio}
return Adapt(bs) // return an adapted blockstore.
}
func (a *apiBlockstore) DeleteBlock(cid.Cid) error {
return xerrors.New("not supported")
}
func (a *apiBlockstore) Has(c cid.Cid) (bool, error) {
return a.api.ChainHasObj(context.TODO(), c)
}
func (a *apiBlockstore) Get(c cid.Cid) (blocks.Block, error) {
bb, err := a.api.ChainReadObj(context.TODO(), c)
if err != nil {
return nil, err
}
return blocks.NewBlockWithCid(bb, c)
}
func (a *apiBlockstore) GetSize(c cid.Cid) (int, error) {
bb, err := a.api.ChainReadObj(context.TODO(), c)
if err != nil {
return 0, err
}
return len(bb), nil
}
func (a *apiBlockstore) Put(blocks.Block) error {
return xerrors.New("not supported")
}
func (a *apiBlockstore) PutMany([]blocks.Block) error {
return xerrors.New("not supported")
}
func (a *apiBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
return nil, xerrors.New("not supported")
}
func (a *apiBlockstore) HashOnRead(enabled bool) {
return
}

64
lib/blockstore/badger/blockstore.go → blockstore/badger/blockstore.go
View File

@ -16,7 +16,7 @@ import (
logger "github.com/ipfs/go-log/v2"
pool "github.com/libp2p/go-buffer-pool"
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/lotus/blockstore"
)
var (
@ -110,10 +110,7 @@ func Open(opts Options) (*Blockstore, error) {
return nil, fmt.Errorf("failed to open badger blockstore: %w", err)
}
bs := &Blockstore{
DB: db,
}
bs := &Blockstore{DB: db}
if p := opts.Prefix; p != "" {
bs.prefixing = true
bs.prefix = []byte(p)
@ -134,6 +131,25 @@ func (b *Blockstore) Close() error {
return b.DB.Close()
}
// CollectGarbage runs garbage collection on the value log
func (b *Blockstore) CollectGarbage() error {
if atomic.LoadInt64(&b.state) != stateOpen {
return ErrBlockstoreClosed
}
var err error
for err == nil {
err = b.DB.RunValueLogGC(0.125)
}
if err == badger.ErrNoRewrite {
// not really an error in this case
return nil
}
return err
}
// View implements blockstore.Viewer, which leverages zero-copy read-only
// access to values.
func (b *Blockstore) View(cid cid.Cid, fn func([]byte) error) error {
@ -321,6 +337,44 @@ func (b *Blockstore) DeleteBlock(cid cid.Cid) error {
})
}
func (b *Blockstore) DeleteMany(cids []cid.Cid) error {
if atomic.LoadInt64(&b.state) != stateOpen {
return ErrBlockstoreClosed
}
batch := b.DB.NewWriteBatch()
defer batch.Cancel()
// toReturn tracks the byte slices to return to the pool, if we're using key
// prefixing. we can't return each slice to the pool after each Set, because
// badger holds on to the slice.
var toReturn [][]byte
if b.prefixing {
toReturn = make([][]byte, 0, len(cids))
defer func() {
for _, b := range toReturn {
KeyPool.Put(b)
}
}()
}
for _, cid := range cids {
k, pooled := b.PooledStorageKey(cid)
if pooled {
toReturn = append(toReturn, k)
}
if err := batch.Delete(k); err != nil {
return err
}
}
err := batch.Flush()
if err != nil {
err = fmt.Errorf("failed to delete blocks from badger blockstore: %w", err)
}
return err
}
// AllKeysChan implements Blockstore.AllKeysChan.
func (b *Blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
if atomic.LoadInt64(&b.state) != stateOpen {

11
lib/blockstore/badger/blockstore_test.go → blockstore/badger/blockstore_test.go
View File

@ -6,8 +6,9 @@ import (
"testing"
blocks "github.com/ipfs/go-block-format"
blockstore "github.com/ipfs/go-ipfs-blockstore"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/lotus/blockstore"
)
func TestBadgerBlockstore(t *testing.T) {
@ -60,8 +61,8 @@ func TestStorageKey(t *testing.T) {
require.Equal(t, k3, k2)
}
func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (bs blockstore.Blockstore, path string) {
return func(tb testing.TB) (bs blockstore.Blockstore, path string) {
func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) {
return func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) {
tb.Helper()
path, err := ioutil.TempDir("", "")
@ -82,8 +83,8 @@ func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (
}
}
func openBlockstore(optsSupplier func(path string) Options) func(tb testing.TB, path string) (bs blockstore.Blockstore, err error) {
return func(tb testing.TB, path string) (bs blockstore.Blockstore, err error) {
func openBlockstore(optsSupplier func(path string) Options) func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) {
return func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) {
tb.Helper()
return Open(optsSupplier(path))
}

9
lib/blockstore/badger/blockstore_test_suite.go → blockstore/badger/blockstore_test_suite.go
View File

@ -8,18 +8,19 @@ import (
"strings"
"testing"
"github.com/filecoin-project/lotus/lib/blockstore"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
u "github.com/ipfs/go-ipfs-util"
"github.com/filecoin-project/lotus/blockstore"
"github.com/stretchr/testify/require"
)
// TODO: move this to go-ipfs-blockstore.
type Suite struct {
NewBlockstore func(tb testing.TB) (bs blockstore.Blockstore, path string)
OpenBlockstore func(tb testing.TB, path string) (bs blockstore.Blockstore, err error)
NewBlockstore func(tb testing.TB) (bs blockstore.BasicBlockstore, path string)
OpenBlockstore func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error)
}
func (s *Suite) RunTests(t *testing.T, prefix string) {
@ -290,7 +291,7 @@ func (s *Suite) TestDelete(t *testing.T) {
}
func insertBlocks(t *testing.T, bs blockstore.Blockstore, count int) []cid.Cid {
func insertBlocks(t *testing.T, bs blockstore.BasicBlockstore, count int) []cid.Cid {
keys := make([]cid.Cid, count)
for i := 0; i < count; i++ {
block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))

95
blockstore/blockstore.go Normal file
View File

@ -0,0 +1,95 @@
package blockstore
import (
cid "github.com/ipfs/go-cid"
ds "github.com/ipfs/go-datastore"
logging "github.com/ipfs/go-log/v2"
blockstore "github.com/ipfs/go-ipfs-blockstore"
)
var log = logging.Logger("blockstore")
var ErrNotFound = blockstore.ErrNotFound
// Blockstore is the blockstore interface used by Lotus. It is the union
// of the basic go-ipfs blockstore, with other capabilities required by Lotus,
// e.g. View or Sync.
type Blockstore interface {
blockstore.Blockstore
blockstore.Viewer
BatchDeleter
}
// BasicBlockstore is an alias to the original IPFS Blockstore.
type BasicBlockstore = blockstore.Blockstore
type Viewer = blockstore.Viewer
type BatchDeleter interface {
DeleteMany(cids []cid.Cid) error
}
// WrapIDStore wraps the underlying blockstore in an "identity" blockstore.
// The ID store filters out all puts for blocks with CIDs using the "identity"
// hash function. It also extracts inlined blocks from CIDs using the identity
// hash function and returns them on get/has, ignoring the contents of the
// blockstore.
func WrapIDStore(bstore blockstore.Blockstore) Blockstore {
if is, ok := bstore.(*idstore); ok {
// already wrapped
return is
}
if bs, ok := bstore.(Blockstore); ok {
// we need to wrap our own because we don't want to neuter the DeleteMany method
// the underlying blockstore has implemented an (efficient) DeleteMany
return NewIDStore(bs)
}
// The underlying blockstore does not implement DeleteMany, so we need to shim it.
// This is less efficient as it'll iterate and perform single deletes.
return NewIDStore(Adapt(bstore))
}
// FromDatastore creates a new blockstore backed by the given datastore.
func FromDatastore(dstore ds.Batching) Blockstore {
return WrapIDStore(blockstore.NewBlockstore(dstore))
}
type adaptedBlockstore struct {
blockstore.Blockstore
}
var _ Blockstore = (*adaptedBlockstore)(nil)
func (a *adaptedBlockstore) View(cid cid.Cid, callback func([]byte) error) error {
blk, err := a.Get(cid)
if err != nil {
return err
}
return callback(blk.RawData())
}
func (a *adaptedBlockstore) DeleteMany(cids []cid.Cid) error {
for _, cid := range cids {
err := a.DeleteBlock(cid)
if err != nil {
return err
}
}
return nil
}
// Adapt adapts a standard blockstore to a Lotus blockstore by
// enriching it with the extra methods that Lotus requires (e.g. View, Sync).
//
// View proxies over to Get and calls the callback with the value supplied by Get.
// Sync noops.
func Adapt(bs blockstore.Blockstore) Blockstore {
if ret, ok := bs.(Blockstore); ok {
return ret
}
return &adaptedBlockstore{bs}
}

174
blockstore/buffered.go Normal file
View File

@ -0,0 +1,174 @@
package blockstore
import (
"context"
"os"
block "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
)
// buflog is a logger for the buffered blockstore. It is subscoped from the
// blockstore logger.
var buflog = log.Named("buf")
type BufferedBlockstore struct {
read Blockstore
write Blockstore
}
func NewBuffered(base Blockstore) *BufferedBlockstore {
var buf Blockstore
if os.Getenv("LOTUS_DISABLE_VM_BUF") == "iknowitsabadidea" {
buflog.Warn("VM BLOCKSTORE BUFFERING IS DISABLED")
buf = base
} else {
buf = NewMemory()
}
bs := &BufferedBlockstore{
read: base,
write: buf,
}
return bs
}
func NewTieredBstore(r Blockstore, w Blockstore) *BufferedBlockstore {
return &BufferedBlockstore{
read: r,
write: w,
}
}
var (
_ Blockstore = (*BufferedBlockstore)(nil)
_ Viewer = (*BufferedBlockstore)(nil)
)
func (bs *BufferedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
a, err := bs.read.AllKeysChan(ctx)
if err != nil {
return nil, err
}
b, err := bs.write.AllKeysChan(ctx)
if err != nil {
return nil, err
}
out := make(chan cid.Cid)
go func() {
defer close(out)
for a != nil || b != nil {
select {
case val, ok := <-a:
if !ok {
a = nil
} else {
select {
case out <- val:
case <-ctx.Done():
return
}
}
case val, ok := <-b:
if !ok {
b = nil
} else {
select {
case out <- val:
case <-ctx.Done():
return
}
}
}
}
}()
return out, nil
}
func (bs *BufferedBlockstore) DeleteBlock(c cid.Cid) error {
if err := bs.read.DeleteBlock(c); err != nil {
return err
}
return bs.write.DeleteBlock(c)
}
func (bs *BufferedBlockstore) DeleteMany(cids []cid.Cid) error {
if err := bs.read.DeleteMany(cids); err != nil {
return err
}
return bs.write.DeleteMany(cids)
}
func (bs *BufferedBlockstore) View(c cid.Cid, callback func([]byte) error) error {
// both stores are viewable.
if err := bs.write.View(c, callback); err == ErrNotFound {
// not found in write blockstore; fall through.
} else {
return err // propagate errors, or nil, i.e. found.
}
return bs.read.View(c, callback)
}
func (bs *BufferedBlockstore) Get(c cid.Cid) (block.Block, error) {
if out, err := bs.write.Get(c); err != nil {
if err != ErrNotFound {
return nil, err
}
} else {
return out, nil
}
return bs.read.Get(c)
}
func (bs *BufferedBlockstore) GetSize(c cid.Cid) (int, error) {
s, err := bs.read.GetSize(c)
if err == ErrNotFound || s == 0 {
return bs.write.GetSize(c)
}
return s, err
}
func (bs *BufferedBlockstore) Put(blk block.Block) error {
has, err := bs.read.Has(blk.Cid()) // TODO: consider dropping this check
if err != nil {
return err
}
if has {
return nil
}
return bs.write.Put(blk)
}
func (bs *BufferedBlockstore) Has(c cid.Cid) (bool, error) {
has, err := bs.write.Has(c)
if err != nil {
return false, err
}
if has {
return true, nil
}
return bs.read.Has(c)
}
func (bs *BufferedBlockstore) HashOnRead(hor bool) {
bs.read.HashOnRead(hor)
bs.write.HashOnRead(hor)
}
func (bs *BufferedBlockstore) PutMany(blks []block.Block) error {
return bs.write.PutMany(blks)
}
func (bs *BufferedBlockstore) Read() Blockstore {
return bs.read
}

25
blockstore/cached.go Normal file
View File

@ -0,0 +1,25 @@
package blockstore
import (
"context"
blockstore "github.com/ipfs/go-ipfs-blockstore"
)
type CacheOpts = blockstore.CacheOpts
func DefaultCacheOpts() CacheOpts {
return CacheOpts{
HasBloomFilterSize: 0,
HasBloomFilterHashes: 0,
HasARCCacheSize: 512 << 10,
}
}
func CachedBlockstore(ctx context.Context, bs Blockstore, opts CacheOpts) (Blockstore, error) {
cached, err := blockstore.CachedBlockstore(ctx, bs, opts)
if err != nil {
return nil, err
}
return WrapIDStore(cached), nil
}

9
blockstore/doc.go Normal file
View File

@ -0,0 +1,9 @@
// Package blockstore and subpackages contain most of the blockstore
// implementations used by Lotus.
//
// Blockstores not ultimately constructed out of the building blocks in this
// package may not work properly.
//
// This package re-exports parts of the go-ipfs-blockstore package such that
// no other package needs to import it directly, for ergonomics and traceability.
package blockstore

53
lib/blockstore/fallbackstore.go → blockstore/fallback.go
View File

@ -9,48 +9,61 @@ import (
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
blockstore "github.com/ipfs/go-ipfs-blockstore"
logging "github.com/ipfs/go-log"
)
var log = logging.Logger("blockstore")
type FallbackStore struct {
blockstore.Blockstore
fallbackGetBlock func(context.Context, cid.Cid) (blocks.Block, error)
lk sync.RWMutex
// UnwrapFallbackStore takes a blockstore, and returns the underlying blockstore
// if it was a FallbackStore. Otherwise, it just returns the supplied store
// unmodified.
func UnwrapFallbackStore(bs Blockstore) (Blockstore, bool) {
if fbs, ok := bs.(*FallbackStore); ok {
return fbs.Blockstore, true
}
return bs, false
}
func (fbs *FallbackStore) SetFallback(fg func(context.Context, cid.Cid) (blocks.Block, error)) {
// FallbackStore is a read-through store that queries another (potentially
// remote) source if the block is not found locally. If the block is found
// during the fallback, it stores it in the local store.
type FallbackStore struct {
Blockstore
lk sync.RWMutex
// missFn is the function that will be invoked on a local miss to pull the
// block from elsewhere.
missFn func(context.Context, cid.Cid) (blocks.Block, error)
}
var _ Blockstore = (*FallbackStore)(nil)
func (fbs *FallbackStore) SetFallback(missFn func(context.Context, cid.Cid) (blocks.Block, error)) {
fbs.lk.Lock()
defer fbs.lk.Unlock()
fbs.fallbackGetBlock = fg
fbs.missFn = missFn
}
func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
log.Errorw("fallbackstore: Block not found locally, fetching from the network", "cid", c)
log.Warnf("fallbackstore: block not found locally, fetching from the network; cid: %s", c)
fbs.lk.RLock()
defer fbs.lk.RUnlock()
if fbs.fallbackGetBlock == nil {
if fbs.missFn == nil {
// FallbackStore wasn't configured yet (chainstore/bitswap aren't up yet)
// Wait for a bit and retry
fbs.lk.RUnlock()
time.Sleep(5 * time.Second)
fbs.lk.RLock()
if fbs.fallbackGetBlock == nil {
log.Errorw("fallbackstore: fallbackGetBlock not configured yet")
return nil, blockstore.ErrNotFound
if fbs.missFn == nil {
log.Errorw("fallbackstore: missFn not configured yet")
return nil, ErrNotFound
}
}
ctx, cancel := context.WithTimeout(context.TODO(), 120*time.Second)
defer cancel()
b, err := fbs.fallbackGetBlock(ctx, c)
b, err := fbs.missFn(ctx, c)
if err != nil {
return nil, err
}
@ -69,7 +82,7 @@ func (fbs *FallbackStore) Get(c cid.Cid) (blocks.Block, error) {
switch err {
case nil:
return b, nil
case blockstore.ErrNotFound:
case ErrNotFound:
return fbs.getFallback(c)
default:
return b, err
@ -81,7 +94,7 @@ func (fbs *FallbackStore) GetSize(c cid.Cid) (int, error) {
switch err {
case nil:
return sz, nil
case blockstore.ErrNotFound:
case ErrNotFound:
b, err := fbs.getFallback(c)
if err != nil {
return 0, err
@ -91,5 +104,3 @@ func (fbs *FallbackStore) GetSize(c cid.Cid) (int, error) {
return sz, err
}
}
var _ blockstore.Blockstore = &FallbackStore{}

174
blockstore/idstore.go Normal file
View File

@ -0,0 +1,174 @@
package blockstore
import (
"context"
"io"
"golang.org/x/xerrors"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
mh "github.com/multiformats/go-multihash"
)
var _ Blockstore = (*idstore)(nil)
type idstore struct {
bs Blockstore
}
func NewIDStore(bs Blockstore) Blockstore {
return &idstore{bs: bs}
}
func decodeCid(cid cid.Cid) (inline bool, data []byte, err error) {
if cid.Prefix().MhType != mh.IDENTITY {
return false, nil, nil
}
dmh, err := mh.Decode(cid.Hash())
if err != nil {
return false, nil, err
}
if dmh.Code == mh.IDENTITY {
return true, dmh.Digest, nil
}
return false, nil, err
}
func (b *idstore) Has(cid cid.Cid) (bool, error) {
inline, _, err := decodeCid(cid)
if err != nil {
return false, xerrors.Errorf("error decoding Cid: %w", err)
}
if inline {
return true, nil
}
return b.bs.Has(cid)
}
func (b *idstore) Get(cid cid.Cid) (blocks.Block, error) {
inline, data, err := decodeCid(cid)
if err != nil {
return nil, xerrors.Errorf("error decoding Cid: %w", err)
}
if inline {
return blocks.NewBlockWithCid(data, cid)
}
return b.bs.Get(cid)
}
func (b *idstore) GetSize(cid cid.Cid) (int, error) {
inline, data, err := decodeCid(cid)
if err != nil {
return 0, xerrors.Errorf("error decoding Cid: %w", err)
}
if inline {
return len(data), err
}
return b.bs.GetSize(cid)
}
func (b *idstore) View(cid cid.Cid, cb func([]byte) error) error {
inline, data, err := decodeCid(cid)
if err != nil {
return xerrors.Errorf("error decoding Cid: %w", err)
}
if inline {
return cb(data)
}
return b.bs.View(cid, cb)
}
func (b *idstore) Put(blk blocks.Block) error {
inline, _, err := decodeCid(blk.Cid())
if err != nil {
return xerrors.Errorf("error decoding Cid: %w", err)
}
if inline {
return nil
}
return b.bs.Put(blk)
}
func (b *idstore) PutMany(blks []blocks.Block) error {
toPut := make([]blocks.Block, 0, len(blks))
for _, blk := range blks {
inline, _, err := decodeCid(blk.Cid())
if err != nil {
return xerrors.Errorf("error decoding Cid: %w", err)
}
if inline {
continue
}
toPut = append(toPut, blk)
}
if len(toPut) > 0 {
return b.bs.PutMany(toPut)
}
return nil
}
func (b *idstore) DeleteBlock(cid cid.Cid) error {
inline, _, err := decodeCid(cid)
if err != nil {
return xerrors.Errorf("error decoding Cid: %w", err)
}
if inline {
return nil
}
return b.bs.DeleteBlock(cid)
}
func (b *idstore) DeleteMany(cids []cid.Cid) error {
toDelete := make([]cid.Cid, 0, len(cids))
for _, cid := range cids {
inline, _, err := decodeCid(cid)
if err != nil {
return xerrors.Errorf("error decoding Cid: %w", err)
}
if inline {
continue
}
toDelete = append(toDelete, cid)
}
if len(toDelete) > 0 {
return b.bs.DeleteMany(toDelete)
}
return nil
}
func (b *idstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
return b.bs.AllKeysChan(ctx)
}
func (b *idstore) HashOnRead(enabled bool) {
b.bs.HashOnRead(enabled)
}
func (b *idstore) Close() error {
if c, ok := b.bs.(io.Closer); ok {
return c.Close()
}
return nil
}

44
lib/ipfsbstore/ipfsbstore.go → blockstore/ipfs.go
View File

@ -1,4 +1,4 @@
package ipfsbstore
package blockstore
import (
"bytes"
@ -16,16 +16,16 @@ import (
iface "github.com/ipfs/interface-go-ipfs-core"
"github.com/ipfs/interface-go-ipfs-core/options"
"github.com/ipfs/interface-go-ipfs-core/path"
"github.com/filecoin-project/lotus/lib/blockstore"
)
type IpfsBstore struct {
type IPFSBlockstore struct {
ctx context.Context
api, offlineAPI iface.CoreAPI
}
func NewIpfsBstore(ctx context.Context, onlineMode bool) (*IpfsBstore, error) {
var _ BasicBlockstore = (*IPFSBlockstore)(nil)
func NewLocalIPFSBlockstore(ctx context.Context, onlineMode bool) (Blockstore, error) {
localApi, err := httpapi.NewLocalApi()
if err != nil {
return nil, xerrors.Errorf("getting local ipfs api: %w", err)
@ -34,6 +34,7 @@ func NewIpfsBstore(ctx context.Context, onlineMode bool) (*IpfsBstore, error) {
if err != nil {
return nil, xerrors.Errorf("setting offline mode: %s", err)
}
offlineAPI := api
if onlineMode {
offlineAPI, err = localApi.WithOptions(options.Api.Offline(true))
@ -42,14 +43,16 @@ func NewIpfsBstore(ctx context.Context, onlineMode bool) (*IpfsBstore, error) {
}
}
return &IpfsBstore{
bs := &IPFSBlockstore{
ctx: ctx,
api: api,
offlineAPI: offlineAPI,
}, nil
}
return Adapt(bs), nil
}
func NewRemoteIpfsBstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineMode bool) (*IpfsBstore, error) {
func NewRemoteIPFSBlockstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineMode bool) (Blockstore, error) {
httpApi, err := httpapi.NewApi(maddr)
if err != nil {
return nil, xerrors.Errorf("setting remote ipfs api: %w", err)
@ -58,6 +61,7 @@ func NewRemoteIpfsBstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineM
if err != nil {
return nil, xerrors.Errorf("applying offline mode: %s", err)
}
offlineAPI := api
if onlineMode {
offlineAPI, err = httpApi.WithOptions(options.Api.Offline(true))
@ -66,18 +70,20 @@ func NewRemoteIpfsBstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineM
}
}
return &IpfsBstore{
bs := &IPFSBlockstore{
ctx: ctx,
api: api,
offlineAPI: offlineAPI,
}, nil
}
return Adapt(bs), nil
}
func (i *IpfsBstore) DeleteBlock(cid cid.Cid) error {
func (i *IPFSBlockstore) DeleteBlock(cid cid.Cid) error {
return xerrors.Errorf("not supported")
}
func (i *IpfsBstore) Has(cid cid.Cid) (bool, error) {
func (i *IPFSBlockstore) Has(cid cid.Cid) (bool, error) {
_, err := i.offlineAPI.Block().Stat(i.ctx, path.IpldPath(cid))
if err != nil {
// The underlying client is running in Offline mode.
@ -93,7 +99,7 @@ func (i *IpfsBstore) Has(cid cid.Cid) (bool, error) {
return true, nil
}
func (i *IpfsBstore) Get(cid cid.Cid) (blocks.Block, error) {
func (i *IPFSBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
rd, err := i.api.Block().Get(i.ctx, path.IpldPath(cid))
if err != nil {
return nil, xerrors.Errorf("getting ipfs block: %w", err)
@ -107,7 +113,7 @@ func (i *IpfsBstore) Get(cid cid.Cid) (blocks.Block, error) {
return blocks.NewBlockWithCid(data, cid)
}
func (i *IpfsBstore) GetSize(cid cid.Cid) (int, error) {
func (i *IPFSBlockstore) GetSize(cid cid.Cid) (int, error) {
st, err := i.api.Block().Stat(i.ctx, path.IpldPath(cid))
if err != nil {
return 0, xerrors.Errorf("getting ipfs block: %w", err)
@ -116,7 +122,7 @@ func (i *IpfsBstore) GetSize(cid cid.Cid) (int, error) {
return st.Size(), nil
}
func (i *IpfsBstore) Put(block blocks.Block) error {
func (i *IPFSBlockstore) Put(block blocks.Block) error {
mhd, err := multihash.Decode(block.Cid().Hash())
if err != nil {
return err
@ -128,7 +134,7 @@ func (i *IpfsBstore) Put(block blocks.Block) error {
return err
}
func (i *IpfsBstore) PutMany(blocks []blocks.Block) error {
func (i *IPFSBlockstore) PutMany(blocks []blocks.Block) error {
// TODO: could be done in parallel
for _, block := range blocks {
@ -140,12 +146,10 @@ func (i *IpfsBstore) PutMany(blocks []blocks.Block) error {
return nil
}
func (i *IpfsBstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
func (i *IPFSBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
return nil, xerrors.Errorf("not supported")
}
func (i *IpfsBstore) HashOnRead(enabled bool) {
func (i *IPFSBlockstore) HashOnRead(enabled bool) {
return // TODO: We could technically support this, but..
}
var _ blockstore.Blockstore = &IpfsBstore{}

34
lib/blockstore/memstore.go → blockstore/mem.go
View File

@ -7,20 +7,32 @@ import (
"github.com/ipfs/go-cid"
)
// MemStore is a terminal blockstore that keeps blocks in memory.
type MemStore map[cid.Cid]blocks.Block
// NewMemory returns a temporary memory-backed blockstore.
func NewMemory() MemBlockstore {
return make(MemBlockstore)
}
func (m MemStore) DeleteBlock(k cid.Cid) error {
// MemBlockstore is a terminal blockstore that keeps blocks in memory.
type MemBlockstore map[cid.Cid]blocks.Block
func (m MemBlockstore) DeleteBlock(k cid.Cid) error {
delete(m, k)
return nil
}
func (m MemStore) Has(k cid.Cid) (bool, error) {
func (m MemBlockstore) DeleteMany(ks []cid.Cid) error {
for _, k := range ks {
delete(m, k)
}
return nil
}
func (m MemBlockstore) Has(k cid.Cid) (bool, error) {
_, ok := m[k]
return ok, nil
}
func (m MemStore) View(k cid.Cid, callback func([]byte) error) error {
func (m MemBlockstore) View(k cid.Cid, callback func([]byte) error) error {
b, ok := m[k]
if !ok {
return ErrNotFound
@ -28,7 +40,7 @@ func (m MemStore) View(k cid.Cid, callback func([]byte) error) error {
return callback(b.RawData())
}
func (m MemStore) Get(k cid.Cid) (blocks.Block, error) {
func (m MemBlockstore) Get(k cid.Cid) (blocks.Block, error) {
b, ok := m[k]
if !ok {
return nil, ErrNotFound
@ -37,7 +49,7 @@ func (m MemStore) Get(k cid.Cid) (blocks.Block, error) {
}
// GetSize returns the CIDs mapped BlockSize
func (m MemStore) GetSize(k cid.Cid) (int, error) {
func (m MemBlockstore) GetSize(k cid.Cid) (int, error) {
b, ok := m[k]
if !ok {
return 0, ErrNotFound
@ -46,7 +58,7 @@ func (m MemStore) GetSize(k cid.Cid) (int, error) {
}
// Put puts a given block to the underlying datastore
func (m MemStore) Put(b blocks.Block) error {
func (m MemBlockstore) Put(b blocks.Block) error {
// Convert to a basic block for safety, but try to reuse the existing
// block if it's already a basic block.
k := b.Cid()
@ -64,7 +76,7 @@ func (m MemStore) Put(b blocks.Block) error {
// PutMany puts a slice of blocks at the same time using batching
// capabilities of the underlying datastore whenever possible.
func (m MemStore) PutMany(bs []blocks.Block) error {
func (m MemBlockstore) PutMany(bs []blocks.Block) error {
for _, b := range bs {
_ = m.Put(b) // can't fail
}
@ -74,7 +86,7 @@ func (m MemStore) PutMany(bs []blocks.Block) error {
// AllKeysChan returns a channel from which
// the CIDs in the Blockstore can be read. It should respect
// the given context, closing the channel if it becomes Done.
func (m MemStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
func (m MemBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
ch := make(chan cid.Cid, len(m))
for k := range m {
ch <- k
@ -85,6 +97,6 @@ func (m MemStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
// HashOnRead specifies if every read block should be
// rehashed to make sure it matches its CID.
func (m MemStore) HashOnRead(enabled bool) {
func (m MemBlockstore) HashOnRead(enabled bool) {
// no-op
}

154
blockstore/metrics.go Normal file
View File

@ -0,0 +1,154 @@
package blockstore
import (
"time"
"go.opencensus.io/stats"
"go.opencensus.io/stats/view"
"go.opencensus.io/tag"
)
//
// Currently unused, but kept in repo in case we introduce one of the candidate
// cache implementations (Freecache, Ristretto), both of which report these
// metrics.
//
// CacheMetricsEmitInterval is the interval at which metrics are emitted onto
// OpenCensus.
var CacheMetricsEmitInterval = 5 * time.Second
var (
CacheName, _ = tag.NewKey("cache_name")
)
// CacheMeasures groups all metrics emitted by the blockstore caches.
var CacheMeasures = struct {
HitRatio *stats.Float64Measure
Hits *stats.Int64Measure
Misses *stats.Int64Measure
Entries *stats.Int64Measure
QueriesServed *stats.Int64Measure
Adds *stats.Int64Measure
Updates *stats.Int64Measure
Evictions *stats.Int64Measure
CostAdded *stats.Int64Measure
CostEvicted *stats.Int64Measure
SetsDropped *stats.Int64Measure
SetsRejected *stats.Int64Measure
QueriesDropped *stats.Int64Measure
}{
HitRatio: stats.Float64("blockstore/cache/hit_ratio", "Hit ratio of blockstore cache", stats.UnitDimensionless),
Hits: stats.Int64("blockstore/cache/hits", "Total number of hits at blockstore cache", stats.UnitDimensionless),
Misses: stats.Int64("blockstore/cache/misses", "Total number of misses at blockstore cache", stats.UnitDimensionless),
Entries: stats.Int64("blockstore/cache/entry_count", "Total number of entries currently in the blockstore cache", stats.UnitDimensionless),
QueriesServed: stats.Int64("blockstore/cache/queries_served", "Total number of queries served by the blockstore cache", stats.UnitDimensionless),
Adds: stats.Int64("blockstore/cache/adds", "Total number of adds to blockstore cache", stats.UnitDimensionless),
Updates: stats.Int64("blockstore/cache/updates", "Total number of updates in blockstore cache", stats.UnitDimensionless),
Evictions: stats.Int64("blockstore/cache/evictions", "Total number of evictions from blockstore cache", stats.UnitDimensionless),
CostAdded: stats.Int64("blockstore/cache/cost_added", "Total cost (byte size) of entries added into blockstore cache", stats.UnitBytes),
CostEvicted: stats.Int64("blockstore/cache/cost_evicted", "Total cost (byte size) of entries evicted by blockstore cache", stats.UnitBytes),
SetsDropped: stats.Int64("blockstore/cache/sets_dropped", "Total number of sets dropped by blockstore cache", stats.UnitDimensionless),
SetsRejected: stats.Int64("blockstore/cache/sets_rejected", "Total number of sets rejected by blockstore cache", stats.UnitDimensionless),
QueriesDropped: stats.Int64("blockstore/cache/queries_dropped", "Total number of queries dropped by blockstore cache", stats.UnitDimensionless),
}
// CacheViews groups all cache-related default views.
var CacheViews = struct {
HitRatio *view.View
Hits *view.View
Misses *view.View
Entries *view.View
QueriesServed *view.View
Adds *view.View
Updates *view.View
Evictions *view.View
CostAdded *view.View
CostEvicted *view.View
SetsDropped *view.View
SetsRejected *view.View
QueriesDropped *view.View
}{
HitRatio: &view.View{
Measure: CacheMeasures.HitRatio,
Aggregation: view.LastValue(),
TagKeys: []tag.Key{CacheName},
},
Hits: &view.View{
Measure: CacheMeasures.Hits,
Aggregation: view.LastValue(),
TagKeys: []tag.Key{CacheName},
},
Misses: &view.View{
Measure: CacheMeasures.Misses,
Aggregation: view.LastValue(),
TagKeys: []tag.Key{CacheName},
},
Entries: &view.View{
Measure: CacheMeasures.Entries,
Aggregation: view.LastValue(),
TagKeys: []tag.Key{CacheName},
},
QueriesServed: &view.View{
Measure: CacheMeasures.QueriesServed,
Aggregation: view.LastValue(),
TagKeys: []tag.Key{CacheName},
},
Adds: &view.View{
Measure: CacheMeasures.Adds,
Aggregation: view.LastValue(),
TagKeys: []tag.Key{CacheName},
},
Updates: &view.View{
Measure: CacheMeasures.Updates,
Aggregation: view.LastValue(),
TagKeys: []tag.Key{CacheName},
},
Evictions: &view.View{
Measure: CacheMeasures.Evictions,
Aggregation: view.LastValue(),
TagKeys: []tag.Key{CacheName},
},
CostAdded: &view.View{
Measure: CacheMeasures.CostAdded,
Aggregation: view.LastValue(),
TagKeys: []tag.Key{CacheName},
},
CostEvicted: &view.View{
Measure: CacheMeasures.CostEvicted,
Aggregation: view.LastValue(),
TagKeys: []tag.Key{CacheName},
},
SetsDropped: &view.View{
Measure: CacheMeasures.SetsDropped,
Aggregation: view.LastValue(),
TagKeys: []tag.Key{CacheName},
},
SetsRejected: &view.View{
Measure: CacheMeasures.SetsRejected,
Aggregation: view.LastValue(),
TagKeys: []tag.Key{CacheName},
},
QueriesDropped: &view.View{
Measure: CacheMeasures.QueriesDropped,
Aggregation: view.LastValue(),
TagKeys: []tag.Key{CacheName},
},
}
// DefaultViews exports all default views for this package.
var DefaultViews = []*view.View{
CacheViews.HitRatio,
CacheViews.Hits,
CacheViews.Misses,
CacheViews.Entries,
CacheViews.QueriesServed,
CacheViews.Adds,
CacheViews.Updates,
CacheViews.Evictions,
CacheViews.CostAdded,
CacheViews.CostEvicted,
CacheViews.SetsDropped,
CacheViews.SetsRejected,
CacheViews.QueriesDropped,
}

38
blockstore/splitstore/markset.go Normal file
View File

@ -0,0 +1,38 @@
package splitstore
import (
"path/filepath"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
)
// MarkSet is a utility to keep track of seen CID, and later query for them.
//
// * If the expected dataset is large, it can be backed by a datastore (e.g. bbolt).
// * If a probabilistic result is acceptable, it can be backed by a bloom filter (default).
type MarkSet interface {
Mark(cid.Cid) error
Has(cid.Cid) (bool, error)
Close() error
}
// markBytes is deliberately a non-nil empty byte slice for serialization.
var markBytes = []byte{}
type MarkSetEnv interface {
Create(name string, sizeHint int64) (MarkSet, error)
Close() error
}
func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error) {
switch mtype {
case "", "bloom":
return NewBloomMarkSetEnv()
case "bolt":
return NewBoltMarkSetEnv(filepath.Join(path, "markset.bolt"))
default:
return nil, xerrors.Errorf("unknown mark set type %s", mtype)
}
}

77
blockstore/splitstore/markset_bloom.go Normal file
View File

@ -0,0 +1,77 @@
package splitstore
import (
"crypto/rand"
"crypto/sha256"
"golang.org/x/xerrors"
bbloom "github.com/ipfs/bbloom"
cid "github.com/ipfs/go-cid"
)
const (
BloomFilterMinSize = 10_000_000
BloomFilterProbability = 0.01
)
type BloomMarkSetEnv struct{}
var _ MarkSetEnv = (*BloomMarkSetEnv)(nil)
type BloomMarkSet struct {
salt []byte
bf *bbloom.Bloom
}
var _ MarkSet = (*BloomMarkSet)(nil)
func NewBloomMarkSetEnv() (*BloomMarkSetEnv, error) {
return &BloomMarkSetEnv{}, nil
}
func (e *BloomMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
size := int64(BloomFilterMinSize)
for size < sizeHint {
size += BloomFilterMinSize
}
salt := make([]byte, 4)
_, err := rand.Read(salt)
if err != nil {
return nil, xerrors.Errorf("error reading salt: %w", err)
}
bf, err := bbloom.New(float64(size), BloomFilterProbability)
if err != nil {
return nil, xerrors.Errorf("error creating bloom filter: %w", err)
}
return &BloomMarkSet{salt: salt, bf: bf}, nil
}
func (e *BloomMarkSetEnv) Close() error {
return nil
}
func (s *BloomMarkSet) saltedKey(cid cid.Cid) []byte {
hash := cid.Hash()
key := make([]byte, len(s.salt)+len(hash))
n := copy(key, s.salt)
copy(key[n:], hash)
rehash := sha256.Sum256(key)
return rehash[:]
}
func (s *BloomMarkSet) Mark(cid cid.Cid) error {
s.bf.Add(s.saltedKey(cid))
return nil
}
func (s *BloomMarkSet) Has(cid cid.Cid) (bool, error) {
return s.bf.Has(s.saltedKey(cid)), nil
}
func (s *BloomMarkSet) Close() error {
return nil
}

81
blockstore/splitstore/markset_bolt.go Normal file
View File

@ -0,0 +1,81 @@
package splitstore
import (
"time"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
bolt "go.etcd.io/bbolt"
)
type BoltMarkSetEnv struct {
db *bolt.DB
}
var _ MarkSetEnv = (*BoltMarkSetEnv)(nil)
type BoltMarkSet struct {
db *bolt.DB
bucketId []byte
}
var _ MarkSet = (*BoltMarkSet)(nil)
func NewBoltMarkSetEnv(path string) (*BoltMarkSetEnv, error) {
db, err := bolt.Open(path, 0644,
&bolt.Options{
Timeout: 1 * time.Second,
NoSync: true,
})
if err != nil {
return nil, err
}
return &BoltMarkSetEnv{db: db}, nil
}
func (e *BoltMarkSetEnv) Create(name string, hint int64) (MarkSet, error) {
bucketId := []byte(name)
err := e.db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists(bucketId)
if err != nil {
return xerrors.Errorf("error creating bolt db bucket %s: %w", name, err)
}
return nil
})
if err != nil {
return nil, err
}
return &BoltMarkSet{db: e.db, bucketId: bucketId}, nil
}
func (e *BoltMarkSetEnv) Close() error {
return e.db.Close()
}
func (s *BoltMarkSet) Mark(cid cid.Cid) error {
return s.db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket(s.bucketId)
return b.Put(cid.Hash(), markBytes)
})
}
func (s *BoltMarkSet) Has(cid cid.Cid) (result bool, err error) {
err = s.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(s.bucketId)
v := b.Get(cid.Hash())
result = v != nil
return nil
})
return result, err
}
func (s *BoltMarkSet) Close() error {
return s.db.Update(func(tx *bolt.Tx) error {
return tx.DeleteBucket(s.bucketId)
})
}

138
blockstore/splitstore/markset_test.go Normal file
View File

@ -0,0 +1,138 @@
package splitstore
import (
"io/ioutil"
"testing"
cid "github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
)
func TestBoltMarkSet(t *testing.T) {
testMarkSet(t, "bolt")
}
func TestBloomMarkSet(t *testing.T) {
testMarkSet(t, "bloom")
}
func testMarkSet(t *testing.T, lsType string) {
t.Helper()
path, err := ioutil.TempDir("", "sweep-test.*")
if err != nil {
t.Fatal(err)
}
env, err := OpenMarkSetEnv(path, lsType)
if err != nil {
t.Fatal(err)
}
defer env.Close() //nolint:errcheck
hotSet, err := env.Create("hot", 0)
if err != nil {
t.Fatal(err)
}
coldSet, err := env.Create("cold", 0)
if err != nil {
t.Fatal(err)
}
makeCid := func(key string) cid.Cid {
h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
if err != nil {
t.Fatal(err)
}
return cid.NewCidV1(cid.Raw, h)
}
mustHave := func(s MarkSet, cid cid.Cid) {
has, err := s.Has(cid)
if err != nil {
t.Fatal(err)
}
if !has {
t.Fatal("mark not found")
}
}
mustNotHave := func(s MarkSet, cid cid.Cid) {
has, err := s.Has(cid)
if err != nil {
t.Fatal(err)
}
if has {
t.Fatal("unexpected mark")
}
}
k1 := makeCid("a")
k2 := makeCid("b")
k3 := makeCid("c")
k4 := makeCid("d")
hotSet.Mark(k1) //nolint
hotSet.Mark(k2) //nolint
coldSet.Mark(k3) //nolint
mustHave(hotSet, k1)
mustHave(hotSet, k2)
mustNotHave(hotSet, k3)
mustNotHave(hotSet, k4)
mustNotHave(coldSet, k1)
mustNotHave(coldSet, k2)
mustHave(coldSet, k3)
mustNotHave(coldSet, k4)
// close them and reopen to redo the dance
err = hotSet.Close()
if err != nil {
t.Fatal(err)
}
err = coldSet.Close()
if err != nil {
t.Fatal(err)
}
hotSet, err = env.Create("hot", 0)
if err != nil {
t.Fatal(err)
}
coldSet, err = env.Create("cold", 0)
if err != nil {
t.Fatal(err)
}
hotSet.Mark(k3) //nolint
hotSet.Mark(k4) //nolint
coldSet.Mark(k1) //nolint
mustNotHave(hotSet, k1)
mustNotHave(hotSet, k2)
mustHave(hotSet, k3)
mustHave(hotSet, k4)
mustHave(coldSet, k1)
mustNotHave(coldSet, k2)
mustNotHave(coldSet, k3)
mustNotHave(coldSet, k4)
err = hotSet.Close()
if err != nil {
t.Fatal(err)
}
err = coldSet.Close()
if err != nil {
t.Fatal(err)
}
}

1069
blockstore/splitstore/splitstore.go Normal file
View File

File diff suppressed because it is too large Load Diff

255
blockstore/splitstore/splitstore_test.go Normal file
View File

@ -0,0 +1,255 @@
package splitstore
import (
"context"
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/types/mock"
cid "github.com/ipfs/go-cid"
datastore "github.com/ipfs/go-datastore"
dssync "github.com/ipfs/go-datastore/sync"
logging "github.com/ipfs/go-log/v2"
)
func init() {
CompactionThreshold = 5
CompactionCold = 1
CompactionBoundary = 2
logging.SetLogLevel("splitstore", "DEBUG")
}
func testSplitStore(t *testing.T, cfg *Config) {
chain := &mockChain{}
// genesis
genBlock := mock.MkBlock(nil, 0, 0)
genTs := mock.TipSet(genBlock)
chain.push(genTs)
// the myriads of stores
ds := dssync.MutexWrap(datastore.NewMapDatastore())
hot := blockstore.NewMemorySync()
cold := blockstore.NewMemorySync()
// put the genesis block to cold store
blk, err := genBlock.ToStorageBlock()
if err != nil {
t.Fatal(err)
}
err = cold.Put(blk)
if err != nil {
t.Fatal(err)
}
// open the splitstore
ss, err := Open("", ds, hot, cold, cfg)
if err != nil {
t.Fatal(err)
}
defer ss.Close() //nolint
err = ss.Start(chain)
if err != nil {
t.Fatal(err)
}
// make some tipsets, but not enough to cause compaction
mkBlock := func(curTs *types.TipSet, i int) *types.TipSet {
blk := mock.MkBlock(curTs, uint64(i), uint64(i))
sblk, err := blk.ToStorageBlock()
if err != nil {
t.Fatal(err)
}
err = ss.Put(sblk)
if err != nil {
t.Fatal(err)
}
ts := mock.TipSet(blk)
chain.push(ts)
return ts
}
mkGarbageBlock := func(curTs *types.TipSet, i int) {
blk := mock.MkBlock(curTs, uint64(i), uint64(i))
sblk, err := blk.ToStorageBlock()
if err != nil {
t.Fatal(err)
}
err = ss.Put(sblk)
if err != nil {
t.Fatal(err)
}
}
waitForCompaction := func() {
for atomic.LoadInt32(&ss.compacting) == 1 {
time.Sleep(100 * time.Millisecond)
}
}
curTs := genTs
for i := 1; i < 5; i++ {
curTs = mkBlock(curTs, i)
waitForCompaction()
}
mkGarbageBlock(genTs, 1)
// count objects in the cold and hot stores
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
countBlocks := func(bs blockstore.Blockstore) int {
count := 0
ch, err := bs.AllKeysChan(ctx)
if err != nil {
t.Fatal(err)
}
for range ch {
count++
}
return count
}
coldCnt := countBlocks(cold)
hotCnt := countBlocks(hot)
if coldCnt != 1 {
t.Errorf("expected %d blocks, but got %d", 1, coldCnt)
}
if hotCnt != 5 {
t.Errorf("expected %d blocks, but got %d", 5, hotCnt)
}
// trigger a compaction
for i := 5; i < 10; i++ {
curTs = mkBlock(curTs, i)
waitForCompaction()
}
coldCnt = countBlocks(cold)
hotCnt = countBlocks(hot)
if !cfg.EnableFullCompaction {
if coldCnt != 5 {
t.Errorf("expected %d cold blocks, but got %d", 5, coldCnt)
}
if hotCnt != 5 {
t.Errorf("expected %d hot blocks, but got %d", 5, hotCnt)
}
}
if cfg.EnableFullCompaction && !cfg.EnableGC {
if coldCnt != 3 {
t.Errorf("expected %d cold blocks, but got %d", 3, coldCnt)
}
if hotCnt != 7 {
t.Errorf("expected %d hot blocks, but got %d", 7, hotCnt)
}
}
if cfg.EnableFullCompaction && cfg.EnableGC {
if coldCnt != 2 {
t.Errorf("expected %d cold blocks, but got %d", 2, coldCnt)
}
if hotCnt != 7 {
t.Errorf("expected %d hot blocks, but got %d", 7, hotCnt)
}
}
}
func TestSplitStoreSimpleCompaction(t *testing.T) {
testSplitStore(t, &Config{TrackingStoreType: "mem"})
}
func TestSplitStoreFullCompactionWithoutGC(t *testing.T) {
testSplitStore(t, &Config{
TrackingStoreType: "mem",
EnableFullCompaction: true,
})
}
func TestSplitStoreFullCompactionWithGC(t *testing.T) {
testSplitStore(t, &Config{
TrackingStoreType: "mem",
EnableFullCompaction: true,
EnableGC: true,
})
}
type mockChain struct {
sync.Mutex
tipsets []*types.TipSet
listener func(revert []*types.TipSet, apply []*types.TipSet) error
}
func (c *mockChain) push(ts *types.TipSet) {
c.Lock()
c.tipsets = append(c.tipsets, ts)
c.Unlock()
if c.listener != nil {
err := c.listener(nil, []*types.TipSet{ts})
if err != nil {
log.Errorf("mockchain: error dispatching listener: %s", err)
}
}
}
func (c *mockChain) GetTipsetByHeight(_ context.Context, epoch abi.ChainEpoch, _ *types.TipSet, _ bool) (*types.TipSet, error) {
c.Lock()
defer c.Unlock()
iEpoch := int(epoch)
if iEpoch > len(c.tipsets) {
return nil, fmt.Errorf("bad epoch %d", epoch)
}
return c.tipsets[iEpoch-1], nil
}
func (c *mockChain) GetHeaviestTipSet() *types.TipSet {
c.Lock()
defer c.Unlock()
return c.tipsets[len(c.tipsets)-1]
}
func (c *mockChain) SubscribeHeadChanges(change func(revert []*types.TipSet, apply []*types.TipSet) error) {
c.listener = change
}
func (c *mockChain) WalkSnapshot(_ context.Context, ts *types.TipSet, epochs abi.ChainEpoch, _ bool, _ bool, f func(cid.Cid) error) error {
c.Lock()
defer c.Unlock()
start := int(ts.Height()) - 1
end := start - int(epochs)
if end < 0 {
end = -1
}
for i := start; i > end; i-- {
ts := c.tipsets[i]
for _, cid := range ts.Cids() {
err := f(cid)
if err != nil {
return err
}
}
}
return nil
}

109
blockstore/splitstore/tracking.go Normal file
View File

@ -0,0 +1,109 @@
package splitstore
import (
"path/filepath"
"sync"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-state-types/abi"
cid "github.com/ipfs/go-cid"
)
// TrackingStore is a persistent store that tracks blocks that are added
// to the hotstore, tracking the epoch at which they are written.
type TrackingStore interface {
Put(cid.Cid, abi.ChainEpoch) error
PutBatch([]cid.Cid, abi.ChainEpoch) error
Get(cid.Cid) (abi.ChainEpoch, error)
Delete(cid.Cid) error
DeleteBatch([]cid.Cid) error
ForEach(func(cid.Cid, abi.ChainEpoch) error) error
Sync() error
Close() error
}
// OpenTrackingStore opens a tracking store of the specified type in the
// specified path.
func OpenTrackingStore(path string, ttype string) (TrackingStore, error) {
switch ttype {
case "", "bolt":
return OpenBoltTrackingStore(filepath.Join(path, "tracker.bolt"))
case "mem":
return NewMemTrackingStore(), nil
default:
return nil, xerrors.Errorf("unknown tracking store type %s", ttype)
}
}
// NewMemTrackingStore creates an in-memory tracking store.
// This is only useful for test or situations where you don't want to open the
// real tracking store (eg concurrent read only access on a node's datastore)
func NewMemTrackingStore() *MemTrackingStore {
return &MemTrackingStore{tab: make(map[cid.Cid]abi.ChainEpoch)}
}
// MemTrackingStore is a simple in-memory tracking store
type MemTrackingStore struct {
sync.Mutex
tab map[cid.Cid]abi.ChainEpoch
}
var _ TrackingStore = (*MemTrackingStore)(nil)
func (s *MemTrackingStore) Put(cid cid.Cid, epoch abi.ChainEpoch) error {
s.Lock()
defer s.Unlock()
s.tab[cid] = epoch
return nil
}
func (s *MemTrackingStore) PutBatch(cids []cid.Cid, epoch abi.ChainEpoch) error {
s.Lock()
defer s.Unlock()
for _, cid := range cids {
s.tab[cid] = epoch
}
return nil
}
func (s *MemTrackingStore) Get(cid cid.Cid) (abi.ChainEpoch, error) {
s.Lock()
defer s.Unlock()
epoch, ok := s.tab[cid]
if ok {
return epoch, nil
}
return 0, xerrors.Errorf("missing tracking epoch for %s", cid)
}
func (s *MemTrackingStore) Delete(cid cid.Cid) error {
s.Lock()
defer s.Unlock()
delete(s.tab, cid)
return nil
}
func (s *MemTrackingStore) DeleteBatch(cids []cid.Cid) error {
s.Lock()
defer s.Unlock()
for _, cid := range cids {
delete(s.tab, cid)
}
return nil
}
func (s *MemTrackingStore) ForEach(f func(cid.Cid, abi.ChainEpoch) error) error {
s.Lock()
defer s.Unlock()
for cid, epoch := range s.tab {
err := f(cid, epoch)
if err != nil {
return err
}
}
return nil
}
func (s *MemTrackingStore) Sync() error { return nil }
func (s *MemTrackingStore) Close() error { return nil }

120
blockstore/splitstore/tracking_bolt.go Normal file
View File

@ -0,0 +1,120 @@
package splitstore
import (
"time"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
bolt "go.etcd.io/bbolt"
"github.com/filecoin-project/go-state-types/abi"
)
type BoltTrackingStore struct {
db *bolt.DB
bucketId []byte
}
var _ TrackingStore = (*BoltTrackingStore)(nil)
func OpenBoltTrackingStore(path string) (*BoltTrackingStore, error) {
opts := &bolt.Options{
Timeout: 1 * time.Second,
NoSync: true,
}
db, err := bolt.Open(path, 0644, opts)
if err != nil {
return nil, err
}
bucketId := []byte("tracker")
err = db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists(bucketId)
if err != nil {
return xerrors.Errorf("error creating bolt db bucket %s: %w", string(bucketId), err)
}
return nil
})
if err != nil {
_ = db.Close()
return nil, err
}
return &BoltTrackingStore{db: db, bucketId: bucketId}, nil
}
func (s *BoltTrackingStore) Put(cid cid.Cid, epoch abi.ChainEpoch) error {
val := epochToBytes(epoch)
return s.db.Batch(func(tx *bolt.Tx) error {
b := tx.Bucket(s.bucketId)
return b.Put(cid.Hash(), val)
})
}
func (s *BoltTrackingStore) PutBatch(cids []cid.Cid, epoch abi.ChainEpoch) error {
val := epochToBytes(epoch)
return s.db.Batch(func(tx *bolt.Tx) error {
b := tx.Bucket(s.bucketId)
for _, cid := range cids {
err := b.Put(cid.Hash(), val)
if err != nil {
return err
}
}
return nil
})
}
func (s *BoltTrackingStore) Get(cid cid.Cid) (epoch abi.ChainEpoch, err error) {
err = s.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(s.bucketId)
val := b.Get(cid.Hash())
if val == nil {
return xerrors.Errorf("missing tracking epoch for %s", cid)
}
epoch = bytesToEpoch(val)
return nil
})
return epoch, err
}
func (s *BoltTrackingStore) Delete(cid cid.Cid) error {
return s.db.Batch(func(tx *bolt.Tx) error {
b := tx.Bucket(s.bucketId)
return b.Delete(cid.Hash())
})
}
func (s *BoltTrackingStore) DeleteBatch(cids []cid.Cid) error {
return s.db.Batch(func(tx *bolt.Tx) error {
b := tx.Bucket(s.bucketId)
for _, cid := range cids {
err := b.Delete(cid.Hash())
if err != nil {
return xerrors.Errorf("error deleting %s", cid)
}
}
return nil
})
}
func (s *BoltTrackingStore) ForEach(f func(cid.Cid, abi.ChainEpoch) error) error {
return s.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(s.bucketId)
return b.ForEach(func(k, v []byte) error {
cid := cid.NewCidV1(cid.Raw, k)
epoch := bytesToEpoch(v)
return f(cid, epoch)
})
})
}
func (s *BoltTrackingStore) Sync() error {
return s.db.Sync()
}
func (s *BoltTrackingStore) Close() error {
return s.db.Close()
}

130
blockstore/splitstore/tracking_test.go Normal file
View File

@ -0,0 +1,130 @@
package splitstore
import (
"io/ioutil"
"testing"
cid "github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
"github.com/filecoin-project/go-state-types/abi"
)
func TestBoltTrackingStore(t *testing.T) {
testTrackingStore(t, "bolt")
}
func testTrackingStore(t *testing.T, tsType string) {
t.Helper()
makeCid := func(key string) cid.Cid {
h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
if err != nil {
t.Fatal(err)
}
return cid.NewCidV1(cid.Raw, h)
}
mustHave := func(s TrackingStore, cid cid.Cid, epoch abi.ChainEpoch) {
val, err := s.Get(cid)
if err != nil {
t.Fatal(err)
}
if val != epoch {
t.Fatal("epoch mismatch")
}
}
mustNotHave := func(s TrackingStore, cid cid.Cid) {
_, err := s.Get(cid)
if err == nil {
t.Fatal("expected error")
}
}
path, err := ioutil.TempDir("", "snoop-test.*")
if err != nil {
t.Fatal(err)
}
s, err := OpenTrackingStore(path, tsType)
if err != nil {
t.Fatal(err)
}
k1 := makeCid("a")
k2 := makeCid("b")
k3 := makeCid("c")
k4 := makeCid("d")
s.Put(k1, 1) //nolint
s.Put(k2, 2) //nolint
s.Put(k3, 3) //nolint
s.Put(k4, 4) //nolint
mustHave(s, k1, 1)
mustHave(s, k2, 2)
mustHave(s, k3, 3)
mustHave(s, k4, 4)
s.Delete(k1) // nolint
s.Delete(k2) // nolint
mustNotHave(s, k1)
mustNotHave(s, k2)
mustHave(s, k3, 3)
mustHave(s, k4, 4)
s.PutBatch([]cid.Cid{k1}, 1) //nolint
s.PutBatch([]cid.Cid{k2}, 2) //nolint
mustHave(s, k1, 1)
mustHave(s, k2, 2)
mustHave(s, k3, 3)
mustHave(s, k4, 4)
allKeys := map[string]struct{}{
k1.String(): {},
k2.String(): {},
k3.String(): {},
k4.String(): {},
}
err = s.ForEach(func(k cid.Cid, _ abi.ChainEpoch) error {
_, ok := allKeys[k.String()]
if !ok {
t.Fatal("unexpected key")
}
delete(allKeys, k.String())
return nil
})
if err != nil {
t.Fatal(err)
}
if len(allKeys) != 0 {
t.Fatal("not all keys were returned")
}
// no close and reopen and ensure the keys still exist
err = s.Close()
if err != nil {
t.Fatal(err)
}
s, err = OpenTrackingStore(path, tsType)
if err != nil {
t.Fatal(err)
}
mustHave(s, k1, 1)
mustHave(s, k2, 2)
mustHave(s, k3, 3)
mustHave(s, k4, 4)
s.Close() //nolint:errcheck
}

81
blockstore/sync.go Normal file
View File

@ -0,0 +1,81 @@
package blockstore
import (
"context"
"sync"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
)
// NewMemorySync returns a thread-safe in-memory blockstore.
func NewMemorySync() *SyncBlockstore {
return &SyncBlockstore{bs: make(MemBlockstore)}
}
// SyncBlockstore is a terminal blockstore that is a synchronized version
// of MemBlockstore.
type SyncBlockstore struct {
mu sync.RWMutex
bs MemBlockstore // specifically use a memStore to save indirection overhead.
}
func (m *SyncBlockstore) DeleteBlock(k cid.Cid) error {
m.mu.Lock()
defer m.mu.Unlock()
return m.bs.DeleteBlock(k)
}
func (m *SyncBlockstore) DeleteMany(ks []cid.Cid) error {
m.mu.Lock()
defer m.mu.Unlock()
return m.bs.DeleteMany(ks)
}
func (m *SyncBlockstore) Has(k cid.Cid) (bool, error) {
m.mu.RLock()
defer m.mu.RUnlock()
return m.bs.Has(k)
}
func (m *SyncBlockstore) View(k cid.Cid, callback func([]byte) error) error {
m.mu.RLock()
defer m.mu.RUnlock()
return m.bs.View(k, callback)
}
func (m *SyncBlockstore) Get(k cid.Cid) (blocks.Block, error) {
m.mu.RLock()
defer m.mu.RUnlock()
return m.bs.Get(k)
}
func (m *SyncBlockstore) GetSize(k cid.Cid) (int, error) {
m.mu.RLock()
defer m.mu.RUnlock()
return m.bs.GetSize(k)
}
func (m *SyncBlockstore) Put(b blocks.Block) error {
m.mu.Lock()
defer m.mu.Unlock()
return m.bs.Put(b)
}
func (m *SyncBlockstore) PutMany(bs []blocks.Block) error {
m.mu.Lock()
defer m.mu.Unlock()
return m.bs.PutMany(bs)
}
func (m *SyncBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
m.mu.RLock()
defer m.mu.RUnlock()
// this blockstore implementation doesn't do any async work.
return m.bs.AllKeysChan(ctx)
}
func (m *SyncBlockstore) HashOnRead(enabled bool) {
// noop
}

76
lib/timedbs/timedbs.go → blockstore/timed.go
View File

@ -1,4 +1,4 @@
package timedbs
package blockstore
import (
"context"
@ -10,37 +10,37 @@ import (
"github.com/ipfs/go-cid"
"github.com/raulk/clock"
"go.uber.org/multierr"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/lib/blockstore"
)
// TimedCacheBS is a blockstore that keeps blocks for at least the specified
// caching interval before discarding them. Garbage collection must be started
// and stopped by calling Start/Stop.
// TimedCacheBlockstore is a blockstore that keeps blocks for at least the
// specified caching interval before discarding them. Garbage collection must
// be started and stopped by calling Start/Stop.
//
// Under the covers, it's implemented with an active and an inactive blockstore
// that are rotated every cache time interval. This means all blocks will be
// stored at most 2x the cache interval.
type TimedCacheBS struct {
//
// Create a new instance by calling the NewTimedCacheBlockstore constructor.
type TimedCacheBlockstore struct {
mu sync.RWMutex
active, inactive blockstore.MemStore
active, inactive MemBlockstore
clock clock.Clock
interval time.Duration
closeCh chan struct{}
doneRotatingCh chan struct{}
}
func NewTimedCacheBS(cacheTime time.Duration) *TimedCacheBS {
return &TimedCacheBS{
active: blockstore.NewTemporary(),
inactive: blockstore.NewTemporary(),
interval: cacheTime,
clock: build.Clock,
func NewTimedCacheBlockstore(interval time.Duration) *TimedCacheBlockstore {
b := &TimedCacheBlockstore{
active: NewMemory(),
inactive: NewMemory(),
interval: interval,
clock: clock.New(),
}
return b
}
func (t *TimedCacheBS) Start(ctx context.Context) error {
func (t *TimedCacheBlockstore) Start(_ context.Context) error {
t.mu.Lock()
defer t.mu.Unlock()
if t.closeCh != nil {
@ -65,11 +65,11 @@ func (t *TimedCacheBS) Start(ctx context.Context) error {
return nil
}
func (t *TimedCacheBS) Stop(ctx context.Context) error {
func (t *TimedCacheBlockstore) Stop(_ context.Context) error {
t.mu.Lock()
defer t.mu.Unlock()
if t.closeCh == nil {
return fmt.Errorf("not started started")
return fmt.Errorf("not started")
}
select {
case <-t.closeCh:
@ -80,15 +80,15 @@ func (t *TimedCacheBS) Stop(ctx context.Context) error {
return nil
}
func (t *TimedCacheBS) rotate() {
newBs := blockstore.NewTemporary()
func (t *TimedCacheBlockstore) rotate() {
newBs := NewMemory()
t.mu.Lock()
t.inactive, t.active = t.active, newBs
t.mu.Unlock()
}
func (t *TimedCacheBS) Put(b blocks.Block) error {
func (t *TimedCacheBlockstore) Put(b blocks.Block) error {
// Don't check the inactive set here. We want to keep this block for at
// least one interval.
t.mu.Lock()
@ -96,33 +96,43 @@ func (t *TimedCacheBS) Put(b blocks.Block) error {
return t.active.Put(b)
}
func (t *TimedCacheBS) PutMany(bs []blocks.Block) error {
func (t *TimedCacheBlockstore) PutMany(bs []blocks.Block) error {
t.mu.Lock()
defer t.mu.Unlock()
return t.active.PutMany(bs)
}
func (t *TimedCacheBS) Get(k cid.Cid) (blocks.Block, error) {
func (t *TimedCacheBlockstore) View(k cid.Cid, callback func([]byte) error) error {
t.mu.RLock()
defer t.mu.RUnlock()
err := t.active.View(k, callback)
if err == ErrNotFound {
err = t.inactive.View(k, callback)
}
return err
}
func (t *TimedCacheBlockstore) Get(k cid.Cid) (blocks.Block, error) {
t.mu.RLock()
defer t.mu.RUnlock()
b, err := t.active.Get(k)
if err == blockstore.ErrNotFound {
if err == ErrNotFound {
b, err = t.inactive.Get(k)
}
return b, err
}
func (t *TimedCacheBS) GetSize(k cid.Cid) (int, error) {
func (t *TimedCacheBlockstore) GetSize(k cid.Cid) (int, error) {
t.mu.RLock()
defer t.mu.RUnlock()
size, err := t.active.GetSize(k)
if err == blockstore.ErrNotFound {
if err == ErrNotFound {
size, err = t.inactive.GetSize(k)
}
return size, err
}
func (t *TimedCacheBS) Has(k cid.Cid) (bool, error) {
func (t *TimedCacheBlockstore) Has(k cid.Cid) (bool, error) {
t.mu.RLock()
defer t.mu.RUnlock()
if has, err := t.active.Has(k); err != nil {
@ -133,17 +143,23 @@ func (t *TimedCacheBS) Has(k cid.Cid) (bool, error) {
return t.inactive.Has(k)
}
func (t *TimedCacheBS) HashOnRead(_ bool) {
func (t *TimedCacheBlockstore) HashOnRead(_ bool) {
// no-op
}
func (t *TimedCacheBS) DeleteBlock(k cid.Cid) error {
func (t *TimedCacheBlockstore) DeleteBlock(k cid.Cid) error {
t.mu.Lock()
defer t.mu.Unlock()
return multierr.Combine(t.active.DeleteBlock(k), t.inactive.DeleteBlock(k))
}
func (t *TimedCacheBS) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
func (t *TimedCacheBlockstore) DeleteMany(ks []cid.Cid) error {
t.mu.Lock()
defer t.mu.Unlock()
return multierr.Combine(t.active.DeleteMany(ks), t.inactive.DeleteMany(ks))
}
func (t *TimedCacheBlockstore) AllKeysChan(_ context.Context) (<-chan cid.Cid, error) {
t.mu.RLock()
defer t.mu.RUnlock()

6
lib/timedbs/timedbs_test.go → blockstore/timed_test.go
View File

@ -1,4 +1,4 @@
package timedbs
package blockstore
import (
"context"
@ -12,8 +12,8 @@ import (
"github.com/ipfs/go-cid"
)
func TestTimedBSSimple(t *testing.T) {
tc := NewTimedCacheBS(10 * time.Millisecond)
func TestTimedCacheBlockstoreSimple(t *testing.T) {
tc := NewTimedCacheBlockstore(10 * time.Millisecond)
mClock := clock.NewMock()
mClock.Set(time.Now())
tc.clock = mClock

119
blockstore/union.go Normal file
View File

@ -0,0 +1,119 @@
package blockstore
import (
"context"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
)
type unionBlockstore []Blockstore
// Union returns an unioned blockstore.
//
// * Reads return from the first blockstore that has the value, querying in the
// supplied order.
// * Writes (puts and deltes) are broadcast to all stores.
//
func Union(stores ...Blockstore) Blockstore {
return unionBlockstore(stores)
}
func (m unionBlockstore) Has(cid cid.Cid) (has bool, err error) {
for _, bs := range m {
if has, err = bs.Has(cid); has || err != nil {
break
}
}
return has, err
}
func (m unionBlockstore) Get(cid cid.Cid) (blk blocks.Block, err error) {
for _, bs := range m {
if blk, err = bs.Get(cid); err == nil || err != ErrNotFound {
break
}
}
return blk, err
}
func (m unionBlockstore) View(cid cid.Cid, callback func([]byte) error) (err error) {
for _, bs := range m {
if err = bs.View(cid, callback); err == nil || err != ErrNotFound {
break
}
}
return err
}
func (m unionBlockstore) GetSize(cid cid.Cid) (size int, err error) {
for _, bs := range m {
if size, err = bs.GetSize(cid); err == nil || err != ErrNotFound {
break
}
}
return size, err
}
func (m unionBlockstore) Put(block blocks.Block) (err error) {
for _, bs := range m {
if err = bs.Put(block); err != nil {
break
}
}
return err
}
func (m unionBlockstore) PutMany(blks []blocks.Block) (err error) {
for _, bs := range m {
if err = bs.PutMany(blks); err != nil {
break
}
}
return err
}
func (m unionBlockstore) DeleteBlock(cid cid.Cid) (err error) {
for _, bs := range m {
if err = bs.DeleteBlock(cid); err != nil {
break
}
}
return err
}
func (m unionBlockstore) DeleteMany(cids []cid.Cid) (err error) {
for _, bs := range m {
if err = bs.DeleteMany(cids); err != nil {
break
}
}
return err
}
func (m unionBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
// this does not deduplicate; this interface needs to be revisited.
outCh := make(chan cid.Cid)
go func() {
defer close(outCh)
for _, bs := range m {
ch, err := bs.AllKeysChan(ctx)
if err != nil {
return
}
for cid := range ch {
outCh <- cid
}
}
}()
return outCh, nil
}
func (m unionBlockstore) HashOnRead(enabled bool) {
for _, bs := range m {
bs.HashOnRead(enabled)
}
}

102
blockstore/union_test.go Normal file
View File

@ -0,0 +1,102 @@
package blockstore
import (
"context"
"testing"
blocks "github.com/ipfs/go-block-format"
"github.com/stretchr/testify/require"
)
var (
b0 = blocks.NewBlock([]byte("abc"))
b1 = blocks.NewBlock([]byte("foo"))
b2 = blocks.NewBlock([]byte("bar"))
)
func TestUnionBlockstore_Get(t *testing.T) {
m1 := NewMemory()
m2 := NewMemory()
_ = m1.Put(b1)
_ = m2.Put(b2)
u := Union(m1, m2)
v1, err := u.Get(b1.Cid())
require.NoError(t, err)
require.Equal(t, b1.RawData(), v1.RawData())
v2, err := u.Get(b2.Cid())
require.NoError(t, err)
require.Equal(t, b2.RawData(), v2.RawData())
}
func TestUnionBlockstore_Put_PutMany_Delete_AllKeysChan(t *testing.T) {
m1 := NewMemory()
m2 := NewMemory()
u := Union(m1, m2)
err := u.Put(b0)
require.NoError(t, err)
var has bool
// write was broadcasted to all stores.
has, _ = m1.Has(b0.Cid())
require.True(t, has)
has, _ = m2.Has(b0.Cid())
require.True(t, has)
has, _ = u.Has(b0.Cid())
require.True(t, has)
// put many.
err = u.PutMany([]blocks.Block{b1, b2})
require.NoError(t, err)
// write was broadcasted to all stores.
has, _ = m1.Has(b1.Cid())
require.True(t, has)
has, _ = m1.Has(b2.Cid())
require.True(t, has)
has, _ = m2.Has(b1.Cid())
require.True(t, has)
has, _ = m2.Has(b2.Cid())
require.True(t, has)
// also in the union store.
has, _ = u.Has(b1.Cid())
require.True(t, has)
has, _ = u.Has(b2.Cid())
require.True(t, has)
// deleted from all stores.
err = u.DeleteBlock(b1.Cid())
require.NoError(t, err)
has, _ = u.Has(b1.Cid())
require.False(t, has)
has, _ = m1.Has(b1.Cid())
require.False(t, has)
has, _ = m2.Has(b1.Cid())
require.False(t, has)
// check that AllKeysChan returns b0 and b2, twice (once per backing store)
ch, err := u.AllKeysChan(context.Background())
require.NoError(t, err)
var i int
for range ch {
i++
}
require.Equal(t, 4, i)
}

1
build/tools.go
View File

@ -4,5 +4,6 @@ package build
import (
_ "github.com/GeertJohan/go.rice/rice"
_ "github.com/golang/mock/mockgen"
_ "github.com/whyrusleeping/bencher"
)

70
build/version.go
View File

@ -1,11 +1,5 @@
package build
import (
"fmt"
"golang.org/x/xerrors"
)
var CurrentCommit string
var BuildType int
@ -40,67 +34,3 @@ const BuildVersion = "1.5.0"
func UserVersion() string {
return BuildVersion + buildType() + CurrentCommit
}
type Version uint32
func newVer(major, minor, patch uint8) Version {
return Version(uint32(major)<<16 | uint32(minor)<<8 | uint32(patch))
}
// Ints returns (major, minor, patch) versions
func (ve Version) Ints() (uint32, uint32, uint32) {
v := uint32(ve)
return (v & majorOnlyMask) >> 16, (v & minorOnlyMask) >> 8, v & patchOnlyMask
}
func (ve Version) String() string {
vmj, vmi, vp := ve.Ints()
return fmt.Sprintf("%d.%d.%d", vmj, vmi, vp)
}
func (ve Version) EqMajorMinor(v2 Version) bool {
return ve&minorMask == v2&minorMask
}
type NodeType int
const (
NodeUnknown NodeType = iota
NodeFull
NodeMiner
NodeWorker
)
var RunningNodeType NodeType
func VersionForType(nodeType NodeType) (Version, error) {
switch nodeType {
case NodeFull:
return FullAPIVersion, nil
case NodeMiner:
return MinerAPIVersion, nil
case NodeWorker:
return WorkerAPIVersion, nil
default:
return Version(0), xerrors.Errorf("unknown node type %d", nodeType)
}
}
// semver versions of the rpc api exposed
var (
FullAPIVersion = newVer(1, 1, 0)
MinerAPIVersion = newVer(1, 0, 1)
WorkerAPIVersion = newVer(1, 0, 0)
)
//nolint:varcheck,deadcode
const (
majorMask = 0xff0000
minorMask = 0xffff00
patchMask = 0xffffff
majorOnlyMask = 0xff0000
minorOnlyMask = 0x00ff00
patchOnlyMask = 0x0000ff
)

4
chain/actors/adt/diff_adt_test.go
View File

@ -16,7 +16,7 @@ import (
builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
bstore "github.com/filecoin-project/lotus/blockstore"
)
func TestDiffAdtArray(t *testing.T) {
@ -295,7 +295,7 @@ func (t *TestDiffArray) Remove(key uint64, val *typegen.Deferred) error {
func newContextStore() Store {
ctx := context.Background()
bs := bstore.NewTemporarySync()
bs := bstore.NewMemorySync()
store := cbornode.NewCborStore(bs)
return WrapStore(ctx, store)
}

2
chain/events/state/mock/api.go
View File

@ -5,8 +5,8 @@ import (
"sync"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
)

7
chain/events/state/predicates.go
View File

@ -3,6 +3,7 @@ package state
import (
"context"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/go-address"
@ -10,7 +11,7 @@ import (
"github.com/filecoin-project/go-state-types/big"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/filecoin-project/lotus/api/apibstore"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/actors/adt"
init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init"
"github.com/filecoin-project/lotus/chain/actors/builtin/market"
@ -23,7 +24,7 @@ type UserData interface{}
// ChainAPI abstracts out calls made by this class to external APIs
type ChainAPI interface {
apibstore.ChainIO
api.ChainIO
StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error)
}
@ -36,7 +37,7 @@ type StatePredicates struct {
func NewStatePredicates(api ChainAPI) *StatePredicates {
return &StatePredicates{
api: api,
cst: cbor.NewCborStore(apibstore.NewAPIBlockstore(api)),
cst: cbor.NewCborStore(blockstore.NewAPIBlockstore(api)),
}
}

6
chain/events/state/predicates_test.go
View File

@ -23,9 +23,9 @@ import (
adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt"
tutils "github.com/filecoin-project/specs-actors/v2/support/testing"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/actors/builtin/market"
"github.com/filecoin-project/lotus/chain/types"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
)
var dummyCid cid.Cid
@ -36,7 +36,7 @@ func init() {
func TestMarketPredicates(t *testing.T) {
ctx := context.Background()
bs := bstore.NewTemporarySync()
bs := bstore.NewMemorySync()
store := adt2.WrapStore(ctx, cbornode.NewCborStore(bs))
oldDeal1 := &market2.DealState{
@ -334,7 +334,7 @@ func TestMarketPredicates(t *testing.T) {
func TestMinerSectorChange(t *testing.T) {
ctx := context.Background()
bs := bstore.NewTemporarySync()
bs := bstore.NewMemorySync()
store := adt2.WrapStore(ctx, cbornode.NewCborStore(bs))
nextID := uint64(0)

4
chain/gen/gen.go
View File

@ -27,6 +27,7 @@ import (
proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/policy"
"github.com/filecoin-project/lotus/chain/beacon"
@ -40,7 +41,6 @@ import (
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/journal"
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/lotus/lib/sigs"
"github.com/filecoin-project/lotus/node/repo"
)
@ -125,7 +125,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
return nil, xerrors.Errorf("failed to get metadata datastore: %w", err)
}
bs, err := lr.Blockstore(context.TODO(), repo.BlockstoreChain)
bs, err := lr.Blockstore(context.TODO(), repo.UniversalBlockstore)
if err != nil {
return nil, err
}

2
chain/gen/genesis/f00_system.go
View File

@ -8,8 +8,8 @@ import (
"github.com/filecoin-project/specs-actors/actors/builtin"
cbor "github.com/ipfs/go-ipld-cbor"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/types"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
)
func SetupSystemActor(bs bstore.Blockstore) (*types.Actor, error) {

2
chain/gen/genesis/f01_init.go
View File

@ -16,9 +16,9 @@ import (
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/genesis"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
)
func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesis.Actor, rootVerifier genesis.Actor, remainder genesis.Actor) (int64, *types.Actor, map[address.Address]address.Address, error) {

2
chain/gen/genesis/f02_reward.go
View File

@ -9,9 +9,9 @@ import (
reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward"
cbor "github.com/ipfs/go-ipld-cbor"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
)
func SetupRewardActor(bs bstore.Blockstore, qaPower big.Int) (*types.Actor, error) {

2
chain/gen/genesis/f03_cron.go
View File

@ -7,8 +7,8 @@ import (
"github.com/filecoin-project/specs-actors/actors/builtin/cron"
cbor "github.com/ipfs/go-ipld-cbor"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/types"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
)
func SetupCronActor(bs bstore.Blockstore) (*types.Actor, error) {

2
chain/gen/genesis/f04_power.go
View File

@ -9,8 +9,8 @@ import (
power0 "github.com/filecoin-project/specs-actors/actors/builtin/power"
cbor "github.com/ipfs/go-ipld-cbor"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/types"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
)
func SetupStoragePowerActor(bs bstore.Blockstore) (*types.Actor, error) {

2
chain/gen/genesis/f05_market.go
View File

@ -8,8 +8,8 @@ import (
"github.com/filecoin-project/specs-actors/actors/util/adt"
cbor "github.com/ipfs/go-ipld-cbor"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/types"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
)
func SetupStorageMarketActor(bs bstore.Blockstore) (*types.Actor, error) {

2
chain/gen/genesis/f06_vreg.go
View File

@ -10,8 +10,8 @@ import (
verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
"github.com/filecoin-project/specs-actors/actors/util/adt"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/types"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
)
var RootVerifierID address.Address

64
chain/gen/genesis/genesis.go
View File

@ -7,6 +7,7 @@ import (
"fmt"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/journal"
"github.com/ipfs/go-cid"
@ -26,13 +27,13 @@ import (
verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
adt0 "github.com/filecoin-project/specs-actors/actors/util/adt"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/state"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/genesis"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/lotus/lib/sigs"
)
@ -233,14 +234,37 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
}
vregroot, err := address.NewIDAddress(80)
switch template.VerifregRootKey.Type {
case genesis.TAccount:
var ainfo genesis.AccountMeta
if err := json.Unmarshal(template.VerifregRootKey.Meta, &ainfo); err != nil {
return nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
}
st, err := cst.Put(ctx, &account0.State{Address: ainfo.Owner})
if err != nil {
return nil, nil, err
}
if err = createMultisigAccount(ctx, bs, cst, state, vregroot, template.VerifregRootKey, keyIDs); err != nil {
_, ok := keyIDs[ainfo.Owner]
if ok {
return nil, nil, fmt.Errorf("rootkey account has already been declared, cannot be assigned 80: %s", ainfo.Owner)
}
err = state.SetActor(builtin.RootVerifierAddress, &types.Actor{
Code: builtin0.AccountActorCodeID,
Balance: template.VerifregRootKey.Balance,
Head: st,
})
if err != nil {
return nil, nil, xerrors.Errorf("setting verifreg rootkey account: %w", err)
}
case genesis.TMultisig:
if err = createMultisigAccount(ctx, bs, cst, state, builtin.RootVerifierAddress, template.VerifregRootKey, keyIDs); err != nil {
return nil, nil, xerrors.Errorf("failed to set up verified registry signer: %w", err)
}
default:
return nil, nil, xerrors.Errorf("unknown account type for verifreg rootkey: %w", err)
}
// Setup the first verifier as ID-address 81
// TODO: remove this
@ -300,8 +324,36 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
template.RemainderAccount.Balance = remainingFil
if err := createMultisigAccount(ctx, bs, cst, state, builtin.ReserveAddress, template.RemainderAccount, keyIDs); err != nil {
return nil, nil, xerrors.Errorf("failed to set up remainder account: %w", err)
switch template.RemainderAccount.Type {
case genesis.TAccount:
var ainfo genesis.AccountMeta
if err := json.Unmarshal(template.RemainderAccount.Meta, &ainfo); err != nil {
return nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
}
st, err := cst.Put(ctx, &account0.State{Address: ainfo.Owner})
if err != nil {
return nil, nil, err
}
_, ok := keyIDs[ainfo.Owner]
if ok {
return nil, nil, fmt.Errorf("remainder account has already been declared, cannot be assigned 90: %s", ainfo.Owner)
}
err = state.SetActor(builtin.ReserveAddress, &types.Actor{
Code: builtin0.AccountActorCodeID,
Balance: template.RemainderAccount.Balance,
Head: st,
})
if err != nil {
return nil, nil, xerrors.Errorf("setting remainder account: %w", err)
}
case genesis.TMultisig:
if err = createMultisigAccount(ctx, bs, cst, state, builtin.ReserveAddress, template.RemainderAccount, keyIDs); err != nil {
return nil, nil, xerrors.Errorf("failed to set up remainder: %w", err)
}
default:
return nil, nil, xerrors.Errorf("unknown account type for remainder: %w", err)
}
return state, keyIDs, nil
@ -406,7 +458,7 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci
StateBase: stateroot,
Epoch: 0,
Rand: &fakeRand{},
Bstore: cs.Blockstore(),
Bstore: cs.StateBlockstore(),
Syscalls: mkFakedSigSyscalls(cs.VMSys()),
CircSupplyCalc: nil,
NtwkVersion: genesisNetworkVersion,

2
chain/gen/genesis/miners.go
View File

@ -70,7 +70,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
StateBase: sroot,
Epoch: 0,
Rand: &fakeRand{},
Bstore: cs.Blockstore(),
Bstore: cs.StateBlockstore(),
Syscalls: mkFakedSigSyscalls(cs.VMSys()),
CircSupplyCalc: csc,
NtwkVersion: genesisNetworkVersion,

2
chain/gen/mining.go
View File

@ -79,7 +79,7 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w api.WalletA
}
}
store := sm.ChainStore().Store(ctx)
store := sm.ChainStore().ActorStore(ctx)
blsmsgroot, err := toArray(store, blsMsgCids)
if err != nil {
return nil, xerrors.Errorf("building bls amt: %w", err)

4
chain/stmgr/call.go
View File

@ -59,7 +59,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
StateBase: bstate,
Epoch: bheight,
Rand: store.NewChainRand(sm.cs, ts.Cids()),
Bstore: sm.cs.Blockstore(),
Bstore: sm.cs.StateBlockstore(),
Syscalls: sm.cs.VMSys(),
CircSupplyCalc: sm.GetVMCirculatingSupply,
NtwkVersion: sm.GetNtwkVersion,
@ -174,7 +174,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
StateBase: state,
Epoch: ts.Height() + 1,
Rand: r,
Bstore: sm.cs.Blockstore(),
Bstore: sm.cs.StateBlockstore(),
Syscalls: sm.cs.VMSys(),
CircSupplyCalc: sm.GetVMCirculatingSupply,
NtwkVersion: sm.GetNtwkVersion,

29
chain/stmgr/forks.go
View File

@ -15,6 +15,7 @@ import (
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/network"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/adt"
"github.com/filecoin-project/lotus/chain/actors/builtin"
@ -24,8 +25,6 @@ import (
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/lotus/lib/bufbstore"
builtin0 "github.com/filecoin-project/specs-actors/actors/builtin"
miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
multisig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig"
@ -505,7 +504,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
}
case builtin0.StorageMinerActorCodeID:
var st miner0.State
if err := sm.ChainStore().Store(ctx).Get(ctx, act.Head, &st); err != nil {
if err := sm.ChainStore().ActorStore(ctx).Get(ctx, act.Head, &st); err != nil {
return xerrors.Errorf("failed to load miner state: %w", err)
}
@ -549,7 +548,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
return cid.Undef, xerrors.Errorf("failed to load power actor: %w", err)
}
cst := cbor.NewCborStore(sm.ChainStore().Blockstore())
cst := cbor.NewCborStore(sm.ChainStore().StateBlockstore())
if err := cst.Get(ctx, powAct.Head, &ps); err != nil {
return cid.Undef, xerrors.Errorf("failed to get power actor state: %w", err)
}
@ -583,7 +582,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
}
case builtin0.StorageMinerActorCodeID:
var st miner0.State
if err := sm.ChainStore().Store(ctx).Get(ctx, act.Head, &st); err != nil {
if err := sm.ChainStore().ActorStore(ctx).Get(ctx, act.Head, &st); err != nil {
return xerrors.Errorf("failed to load miner state: %w", err)
}
@ -592,7 +591,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
return xerrors.Errorf("failed to get miner info: %w", err)
}
sectorsArr, err := adt0.AsArray(sm.ChainStore().Store(ctx), st.Sectors)
sectorsArr, err := adt0.AsArray(sm.ChainStore().ActorStore(ctx), st.Sectors)
if err != nil {
return xerrors.Errorf("failed to load sectors array: %w", err)
}
@ -612,11 +611,11 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
lbact, err := lbtree.GetActor(addr)
if err == nil {
var lbst miner0.State
if err := sm.ChainStore().Store(ctx).Get(ctx, lbact.Head, &lbst); err != nil {
if err := sm.ChainStore().ActorStore(ctx).Get(ctx, lbact.Head, &lbst); err != nil {
return xerrors.Errorf("failed to load miner state: %w", err)
}
lbsectors, err := adt0.AsArray(sm.ChainStore().Store(ctx), lbst.Sectors)
lbsectors, err := adt0.AsArray(sm.ChainStore().ActorStore(ctx), lbst.Sectors)
if err != nil {
return xerrors.Errorf("failed to load lb sectors array: %w", err)
}
@ -712,7 +711,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
}
func UpgradeIgnition(ctx context.Context, sm *StateManager, _ MigrationCache, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
store := sm.cs.Store(ctx)
store := sm.cs.ActorStore(ctx)
if build.UpgradeLiftoffHeight <= epoch {
return cid.Undef, xerrors.Errorf("liftoff height must be beyond ignition height")
@ -768,7 +767,7 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, _ MigrationCache, cb
func UpgradeRefuel(ctx context.Context, sm *StateManager, _ MigrationCache, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
store := sm.cs.Store(ctx)
store := sm.cs.ActorStore(ctx)
tree, err := sm.StateTree(root)
if err != nil {
return cid.Undef, xerrors.Errorf("getting state tree: %w", err)
@ -793,7 +792,7 @@ func UpgradeRefuel(ctx context.Context, sm *StateManager, _ MigrationCache, cb E
}
func UpgradeActorsV2(ctx context.Context, sm *StateManager, _ MigrationCache, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
buf := bufbstore.NewTieredBstore(sm.cs.Blockstore(), bstore.NewTemporarySync())
buf := blockstore.NewTieredBstore(sm.cs.StateBlockstore(), blockstore.NewMemorySync())
store := store.ActorStore(ctx, buf)
info, err := store.Put(ctx, new(types.StateInfo0))
@ -844,7 +843,7 @@ func UpgradeLiftoff(ctx context.Context, sm *StateManager, _ MigrationCache, cb
return cid.Undef, xerrors.Errorf("getting state tree: %w", err)
}
err = setNetworkName(ctx, sm.cs.Store(ctx), tree, "mainnet")
err = setNetworkName(ctx, sm.cs.ActorStore(ctx), tree, "mainnet")
if err != nil {
return cid.Undef, xerrors.Errorf("setting network name: %w", err)
}
@ -853,7 +852,7 @@ func UpgradeLiftoff(ctx context.Context, sm *StateManager, _ MigrationCache, cb
}
func UpgradeCalico(ctx context.Context, sm *StateManager, _ MigrationCache, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
store := sm.cs.Store(ctx)
store := sm.cs.ActorStore(ctx)
var stateRoot types.StateRoot
if err := store.Get(ctx, root, &stateRoot); err != nil {
return cid.Undef, xerrors.Errorf("failed to decode state root: %w", err)
@ -1010,7 +1009,7 @@ func upgradeActorsV3Common(
root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet,
config nv10.Config,
) (cid.Cid, error) {
buf := bufbstore.NewTieredBstore(sm.cs.Blockstore(), bstore.NewTemporarySync())
buf := blockstore.NewTieredBstore(sm.cs.StateBlockstore(), blockstore.NewMemorySync())
store := store.ActorStore(ctx, buf)
// Load the state root.
@ -1240,7 +1239,7 @@ func resetGenesisMsigs0(ctx context.Context, sm *StateManager, store adt0.Store,
return xerrors.Errorf("getting genesis tipset: %w", err)
}
cst := cbor.NewCborStore(sm.cs.Blockstore())
cst := cbor.NewCborStore(sm.cs.StateBlockstore())
genesisTree, err := state.LoadStateTree(cst, gts.ParentState())
if err != nil {
return xerrors.Errorf("loading state tree: %w", err)

2
chain/stmgr/forks_test.go
View File

@ -125,7 +125,7 @@ func TestForkHeightTriggers(t *testing.T) {
Height: testForkHeight,
Migration: func(ctx context.Context, sm *StateManager, cache MigrationCache, cb ExecCallback,
root cid.Cid, height abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
cst := ipldcbor.NewCborStore(sm.ChainStore().Blockstore())
cst := ipldcbor.NewCborStore(sm.ChainStore().StateBlockstore())
st, err := sm.StateTree(root)
if err != nil {

4
chain/stmgr/read.go
View File

@ -22,7 +22,7 @@ func (sm *StateManager) ParentStateTsk(tsk types.TipSetKey) (*state.StateTree, e
}
func (sm *StateManager) ParentState(ts *types.TipSet) (*state.StateTree, error) {
cst := cbor.NewCborStore(sm.cs.Blockstore())
cst := cbor.NewCborStore(sm.cs.StateBlockstore())
state, err := state.LoadStateTree(cst, sm.parentState(ts))
if err != nil {
return nil, xerrors.Errorf("load state tree: %w", err)
@ -32,7 +32,7 @@ func (sm *StateManager) ParentState(ts *types.TipSet) (*state.StateTree, error)
}
func (sm *StateManager) StateTree(st cid.Cid) (*state.StateTree, error) {
cst := cbor.NewCborStore(sm.cs.Blockstore())
cst := cbor.NewCborStore(sm.cs.StateBlockstore())
state, err := state.LoadStateTree(cst, st)
if err != nil {
return nil, xerrors.Errorf("load state tree: %w", err)

22
chain/stmgr/stmgr.go
View File

@ -286,7 +286,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp
StateBase: base,
Epoch: epoch,
Rand: r,
Bstore: sm.cs.Blockstore(),
Bstore: sm.cs.StateBlockstore(),
Syscalls: sm.cs.VMSys(),
CircSupplyCalc: sm.GetVMCirculatingSupply,
NtwkVersion: sm.GetNtwkVersion,
@ -430,7 +430,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp
return cid.Cid{}, cid.Cid{}, err
}
rectarr := blockadt.MakeEmptyArray(sm.cs.Store(ctx))
rectarr := blockadt.MakeEmptyArray(sm.cs.ActorStore(ctx))
for i, receipt := range receipts {
if err := rectarr.Set(uint64(i), receipt); err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("failed to build receipts amt: %w", err)
@ -515,7 +515,7 @@ func (sm *StateManager) ResolveToKeyAddress(ctx context.Context, addr address.Ad
ts = sm.cs.GetHeaviestTipSet()
}
cst := cbor.NewCborStore(sm.cs.Blockstore())
cst := cbor.NewCborStore(sm.cs.StateBlockstore())
// First try to resolve the actor in the parent state, so we don't have to compute anything.
tree, err := state.LoadStateTree(cst, ts.ParentState())
@ -556,7 +556,7 @@ func (sm *StateManager) GetBlsPublicKey(ctx context.Context, addr address.Addres
}
func (sm *StateManager) LookupID(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) {
cst := cbor.NewCborStore(sm.cs.Blockstore())
cst := cbor.NewCborStore(sm.cs.StateBlockstore())
state, err := state.LoadStateTree(cst, sm.parentState(ts))
if err != nil {
return address.Undef, xerrors.Errorf("load state tree: %w", err)
@ -882,7 +882,7 @@ func (sm *StateManager) MarketBalance(ctx context.Context, addr address.Address,
return api.MarketBalance{}, err
}
mstate, err := market.Load(sm.cs.Store(ctx), act)
mstate, err := market.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return api.MarketBalance{}, err
}
@ -966,7 +966,7 @@ func (sm *StateManager) setupGenesisVestingSchedule(ctx context.Context) error {
return xerrors.Errorf("getting genesis tipset state: %w", err)
}
cst := cbor.NewCborStore(sm.cs.Blockstore())
cst := cbor.NewCborStore(sm.cs.StateBlockstore())
sTree, err := state.LoadStateTree(cst, st)
if err != nil {
return xerrors.Errorf("loading state tree: %w", err)
@ -1325,7 +1325,7 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha
unCirc = big.Add(unCirc, actor.Balance)
case a == market.Address:
mst, err := market.Load(sm.cs.Store(ctx), actor)
mst, err := market.Load(sm.cs.ActorStore(ctx), actor)
if err != nil {
return err
}
@ -1342,7 +1342,7 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha
circ = big.Add(circ, actor.Balance)
case builtin.IsStorageMinerActor(actor.Code):
mst, err := miner.Load(sm.cs.Store(ctx), actor)
mst, err := miner.Load(sm.cs.ActorStore(ctx), actor)
if err != nil {
return err
}
@ -1359,7 +1359,7 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha
}
case builtin.IsMultisigActor(actor.Code):
mst, err := multisig.Load(sm.cs.Store(ctx), actor)
mst, err := multisig.Load(sm.cs.ActorStore(ctx), actor)
if err != nil {
return err
}
@ -1413,7 +1413,7 @@ func (sm *StateManager) GetPaychState(ctx context.Context, addr address.Address,
return nil, nil, err
}
actState, err := paych.Load(sm.cs.Store(ctx), act)
actState, err := paych.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return nil, nil, err
}
@ -1431,7 +1431,7 @@ func (sm *StateManager) GetMarketState(ctx context.Context, ts *types.TipSet) (m
return nil, err
}
actState, err := market.Load(sm.cs.Store(ctx), act)
actState, err := market.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return nil, err
}

32
chain/stmgr/utils.go
View File

@ -48,7 +48,7 @@ func GetNetworkName(ctx context.Context, sm *StateManager, st cid.Cid) (dtypes.N
if err != nil {
return "", err
}
ias, err := init_.Load(sm.cs.Store(ctx), act)
ias, err := init_.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return "", err
}
@ -65,7 +65,7 @@ func GetMinerWorkerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr
if err != nil {
return address.Undef, xerrors.Errorf("(get sset) failed to load miner actor: %w", err)
}
mas, err := miner.Load(sm.cs.Store(ctx), act)
mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return address.Undef, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err)
}
@ -75,7 +75,7 @@ func GetMinerWorkerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr
return address.Undef, xerrors.Errorf("failed to load actor info: %w", err)
}
return vm.ResolveToKeyAddr(state, sm.cs.Store(ctx), info.Worker)
return vm.ResolveToKeyAddr(state, sm.cs.ActorStore(ctx), info.Worker)
}
func GetPower(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (power.Claim, power.Claim, bool, error) {
@ -88,7 +88,7 @@ func GetPowerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr addres
return power.Claim{}, power.Claim{}, false, xerrors.Errorf("(get sset) failed to load power actor state: %w", err)
}
pas, err := power.Load(sm.cs.Store(ctx), act)
pas, err := power.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return power.Claim{}, power.Claim{}, false, err
}
@ -123,7 +123,7 @@ func PreCommitInfo(ctx context.Context, sm *StateManager, maddr address.Address,
return nil, xerrors.Errorf("(get sset) failed to load miner actor: %w", err)
}
mas, err := miner.Load(sm.cs.Store(ctx), act)
mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return nil, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err)
}
@ -137,7 +137,7 @@ func MinerSectorInfo(ctx context.Context, sm *StateManager, maddr address.Addres
return nil, xerrors.Errorf("(get sset) failed to load miner actor: %w", err)
}
mas, err := miner.Load(sm.cs.Store(ctx), act)
mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return nil, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err)
}
@ -151,7 +151,7 @@ func GetSectorsForWinningPoSt(ctx context.Context, nv network.Version, pv ffiwra
return nil, xerrors.Errorf("failed to load miner actor: %w", err)
}
mas, err := miner.Load(sm.cs.Store(ctx), act)
mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return nil, xerrors.Errorf("failed to load miner actor state: %w", err)
}
@ -249,7 +249,7 @@ func GetMinerSlashed(ctx context.Context, sm *StateManager, ts *types.TipSet, ma
return false, xerrors.Errorf("failed to load power actor: %w", err)
}
spas, err := power.Load(sm.cs.Store(ctx), act)
spas, err := power.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return false, xerrors.Errorf("failed to load power actor state: %w", err)
}
@ -272,7 +272,7 @@ func GetStorageDeal(ctx context.Context, sm *StateManager, dealID abi.DealID, ts
return nil, xerrors.Errorf("failed to load market actor: %w", err)
}
state, err := market.Load(sm.cs.Store(ctx), act)
state, err := market.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return nil, xerrors.Errorf("failed to load market actor state: %w", err)
}
@ -320,7 +320,7 @@ func ListMinerActors(ctx context.Context, sm *StateManager, ts *types.TipSet) ([
return nil, xerrors.Errorf("failed to load power actor: %w", err)
}
powState, err := power.Load(sm.cs.Store(ctx), act)
powState, err := power.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return nil, xerrors.Errorf("failed to load power actor state: %w", err)
}
@ -353,7 +353,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
StateBase: base,
Epoch: height,
Rand: r,
Bstore: sm.cs.Blockstore(),
Bstore: sm.cs.StateBlockstore(),
Syscalls: sm.cs.VMSys(),
CircSupplyCalc: sm.GetVMCirculatingSupply,
NtwkVersion: sm.GetNtwkVersion,
@ -474,7 +474,7 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule
return nil, xerrors.Errorf("failed to load miner actor: %w", err)
}
mas, err := miner.Load(sm.cs.Store(ctx), act)
mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return nil, xerrors.Errorf("failed to load miner actor state: %w", err)
}
@ -623,7 +623,7 @@ func minerHasMinPower(ctx context.Context, sm *StateManager, addr address.Addres
return false, xerrors.Errorf("loading power actor state: %w", err)
}
ps, err := power.Load(sm.cs.Store(ctx), pact)
ps, err := power.Load(sm.cs.ActorStore(ctx), pact)
if err != nil {
return false, err
}
@ -654,7 +654,7 @@ func MinerEligibleToMine(ctx context.Context, sm *StateManager, addr address.Add
return false, xerrors.Errorf("loading power actor state: %w", err)
}
pstate, err := power.Load(sm.cs.Store(ctx), pact)
pstate, err := power.Load(sm.cs.ActorStore(ctx), pact)
if err != nil {
return false, err
}
@ -664,7 +664,7 @@ func MinerEligibleToMine(ctx context.Context, sm *StateManager, addr address.Add
return false, xerrors.Errorf("loading miner actor state: %w", err)
}
mstate, err := miner.Load(sm.cs.Store(ctx), mact)
mstate, err := miner.Load(sm.cs.ActorStore(ctx), mact)
if err != nil {
return false, err
}
@ -696,7 +696,7 @@ func MinerEligibleToMine(ctx context.Context, sm *StateManager, addr address.Add
}
func CheckTotalFIL(ctx context.Context, sm *StateManager, ts *types.TipSet) (abi.TokenAmount, error) {
str, err := state.LoadStateTree(sm.ChainStore().Store(ctx), ts.ParentState())
str, err := state.LoadStateTree(sm.ChainStore().ActorStore(ctx), ts.ParentState())
if err != nil {
return abi.TokenAmount{}, err
}

4
chain/store/index_test.go
View File

@ -6,10 +6,10 @@ import (
"testing"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types/mock"
"github.com/filecoin-project/lotus/lib/blockstore"
datastore "github.com/ipfs/go-datastore"
syncds "github.com/ipfs/go-datastore/sync"
"github.com/stretchr/testify/assert"
@ -30,7 +30,7 @@ func TestIndexSeeks(t *testing.T) {
ctx := context.TODO()
nbs := blockstore.NewTemporarySync()
nbs := blockstore.NewMemorySync()
cs := store.NewChainStore(nbs, nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil, nil)
defer cs.Close() //nolint:errcheck

139
chain/store/store.go
View File

@ -23,12 +23,12 @@ import (
blockadt "github.com/filecoin-project/specs-actors/actors/util/adt"
"github.com/filecoin-project/lotus/api"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/adt"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/journal"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/lotus/metrics"
"go.opencensus.io/stats"
@ -81,7 +81,7 @@ func init() {
}
// ReorgNotifee represents a callback that gets called upon reorgs.
type ReorgNotifee func(rev, app []*types.TipSet) error
type ReorgNotifee = func(rev, app []*types.TipSet) error
// Journal event types.
const (
@ -107,11 +107,11 @@ type HeadChangeEvt struct {
// 1. a tipset cache
// 2. a block => messages references cache.
type ChainStore struct {
bs bstore.Blockstore
localbs bstore.Blockstore
ds dstore.Batching
chainBlockstore bstore.Blockstore
stateBlockstore bstore.Blockstore
metadataDs dstore.Batching
localviewer bstore.Viewer
chainLocalBlockstore bstore.Blockstore
heaviestLk sync.Mutex
heaviest *types.TipSet
@ -139,32 +139,31 @@ type ChainStore struct {
wg sync.WaitGroup
}
// localbs is guaranteed to fail Get* if requested block isn't stored locally
func NewChainStore(bs bstore.Blockstore, localbs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder, j journal.Journal) *ChainStore {
mmCache, _ := lru.NewARC(DefaultMsgMetaCacheSize)
tsCache, _ := lru.NewARC(DefaultTipSetCacheSize)
func NewChainStore(chainBs bstore.Blockstore, stateBs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder, j journal.Journal) *ChainStore {
c, _ := lru.NewARC(DefaultMsgMetaCacheSize)
tsc, _ := lru.NewARC(DefaultTipSetCacheSize)
if j == nil {
j = journal.NilJournal()
}
ctx, cancel := context.WithCancel(context.Background())
// unwraps the fallback store in case one is configured.
// some methods _need_ to operate on a local blockstore only.
localbs, _ := bstore.UnwrapFallbackStore(chainBs)
cs := &ChainStore{
bs: bs,
localbs: localbs,
ds: ds,
chainBlockstore: chainBs,
stateBlockstore: stateBs,
chainLocalBlockstore: localbs,
metadataDs: ds,
bestTips: pubsub.New(64),
tipsets: make(map[abi.ChainEpoch][]cid.Cid),
mmCache: mmCache,
tsCache: tsCache,
mmCache: c,
tsCache: tsc,
vmcalls: vmcalls,
cancelFn: cancel,
journal: j,
}
if v, ok := localbs.(bstore.Viewer); ok {
cs.localviewer = v
}
cs.evtTypes = [1]journal.EventType{
evtTypeHeadChange: j.RegisterEventType("sync", "head_change"),
}
@ -216,7 +215,7 @@ func (cs *ChainStore) Close() error {
}
func (cs *ChainStore) Load() error {
head, err := cs.ds.Get(chainHeadKey)
head, err := cs.metadataDs.Get(chainHeadKey)
if err == dstore.ErrNotFound {
log.Warn("no previous chain state found")
return nil
@ -246,7 +245,7 @@ func (cs *ChainStore) writeHead(ts *types.TipSet) error {
return xerrors.Errorf("failed to marshal tipset: %w", err)
}
if err := cs.ds.Put(chainHeadKey, data); err != nil {
if err := cs.metadataDs.Put(chainHeadKey, data); err != nil {
return xerrors.Errorf("failed to write chain head to datastore: %w", err)
}
@ -306,13 +305,13 @@ func (cs *ChainStore) SubscribeHeadChanges(f ReorgNotifee) {
func (cs *ChainStore) IsBlockValidated(ctx context.Context, blkid cid.Cid) (bool, error) {
key := blockValidationCacheKeyPrefix.Instance(blkid.String())
return cs.ds.Has(key)
return cs.metadataDs.Has(key)
}
func (cs *ChainStore) MarkBlockAsValidated(ctx context.Context, blkid cid.Cid) error {
key := blockValidationCacheKeyPrefix.Instance(blkid.String())
if err := cs.ds.Put(key, []byte{0}); err != nil {
if err := cs.metadataDs.Put(key, []byte{0}); err != nil {
return xerrors.Errorf("cache block validation: %w", err)
}
@ -322,7 +321,7 @@ func (cs *ChainStore) MarkBlockAsValidated(ctx context.Context, blkid cid.Cid) e
func (cs *ChainStore) UnmarkBlockAsValidated(ctx context.Context, blkid cid.Cid) error {
key := blockValidationCacheKeyPrefix.Instance(blkid.String())
if err := cs.ds.Delete(key); err != nil {
if err := cs.metadataDs.Delete(key); err != nil {
return xerrors.Errorf("removing from valid block cache: %w", err)
}
@ -339,7 +338,7 @@ func (cs *ChainStore) SetGenesis(b *types.BlockHeader) error {
return err
}
return cs.ds.Put(dstore.NewKey("0"), b.Cid().Bytes())
return cs.metadataDs.Put(dstore.NewKey("0"), b.Cid().Bytes())
}
func (cs *ChainStore) PutTipSet(ctx context.Context, ts *types.TipSet) error {
@ -594,7 +593,7 @@ func (cs *ChainStore) takeHeaviestTipSet(ctx context.Context, ts *types.TipSet)
// FlushValidationCache removes all results of block validation from the
// chain metadata store. Usually the first step after a new chain import.
func (cs *ChainStore) FlushValidationCache() error {
return FlushValidationCache(cs.ds)
return FlushValidationCache(cs.metadataDs)
}
func FlushValidationCache(ds datastore.Batching) error {
@ -653,7 +652,7 @@ func (cs *ChainStore) SetHead(ts *types.TipSet) error {
// Contains returns whether our BlockStore has all blocks in the supplied TipSet.
func (cs *ChainStore) Contains(ts *types.TipSet) (bool, error) {
for _, c := range ts.Cids() {
has, err := cs.bs.Has(c)
has, err := cs.chainBlockstore.Has(c)
if err != nil {
return false, err
}
@ -668,16 +667,8 @@ func (cs *ChainStore) Contains(ts *types.TipSet) (bool, error) {
// GetBlock fetches a BlockHeader with the supplied CID. It returns
// blockstore.ErrNotFound if the block was not found in the BlockStore.
func (cs *ChainStore) GetBlock(c cid.Cid) (*types.BlockHeader, error) {
if cs.localviewer == nil {
sb, err := cs.localbs.Get(c)
if err != nil {
return nil, err
}
return types.DecodeBlock(sb.RawData())
}
var blk *types.BlockHeader
err := cs.localviewer.View(c, func(b []byte) (err error) {
err := cs.chainLocalBlockstore.View(c, func(b []byte) (err error) {
blk, err = types.DecodeBlock(b)
return err
})
@ -851,7 +842,7 @@ func (cs *ChainStore) PersistBlockHeaders(b ...*types.BlockHeader) error {
end = len(b)
}
err = multierr.Append(err, cs.bs.PutMany(sbs[start:end]))
err = multierr.Append(err, cs.chainLocalBlockstore.PutMany(sbs[start:end]))
}
return err
@ -875,7 +866,7 @@ func PutMessage(bs bstore.Blockstore, m storable) (cid.Cid, error) {
}
func (cs *ChainStore) PutMessage(m storable) (cid.Cid, error) {
return PutMessage(cs.bs, m)
return PutMessage(cs.chainBlockstore, m)
}
func (cs *ChainStore) expandTipset(b *types.BlockHeader) (*types.TipSet, error) {
@ -936,7 +927,7 @@ func (cs *ChainStore) AddBlock(ctx context.Context, b *types.BlockHeader) error
}
func (cs *ChainStore) GetGenesis() (*types.BlockHeader, error) {
data, err := cs.ds.Get(dstore.NewKey("0"))
data, err := cs.metadataDs.Get(dstore.NewKey("0"))
if err != nil {
return nil, err
}
@ -962,17 +953,8 @@ func (cs *ChainStore) GetCMessage(c cid.Cid) (types.ChainMsg, error) {
}
func (cs *ChainStore) GetMessage(c cid.Cid) (*types.Message, error) {
if cs.localviewer == nil {
sb, err := cs.localbs.Get(c)
if err != nil {
log.Errorf("get message get failed: %s: %s", c, err)
return nil, err
}
return types.DecodeMessage(sb.RawData())
}
var msg *types.Message
err := cs.localviewer.View(c, func(b []byte) (err error) {
err := cs.chainLocalBlockstore.View(c, func(b []byte) (err error) {
msg, err = types.DecodeMessage(b)
return err
})
@ -980,17 +962,8 @@ func (cs *ChainStore) GetMessage(c cid.Cid) (*types.Message, error) {
}
func (cs *ChainStore) GetSignedMessage(c cid.Cid) (*types.SignedMessage, error) {
if cs.localviewer == nil {
sb, err := cs.localbs.Get(c)
if err != nil {
log.Errorf("get message get failed: %s: %s", c, err)
return nil, err
}
return types.DecodeSignedMessage(sb.RawData())
}
var msg *types.SignedMessage
err := cs.localviewer.View(c, func(b []byte) (err error) {
err := cs.chainLocalBlockstore.View(c, func(b []byte) (err error) {
msg, err = types.DecodeSignedMessage(b)
return err
})
@ -1000,7 +973,7 @@ func (cs *ChainStore) GetSignedMessage(c cid.Cid) (*types.SignedMessage, error)
func (cs *ChainStore) readAMTCids(root cid.Cid) ([]cid.Cid, error) {
ctx := context.TODO()
// block headers use adt0, for now.
a, err := blockadt.AsArray(cs.Store(ctx), root)
a, err := blockadt.AsArray(cs.ActorStore(ctx), root)
if err != nil {
return nil, xerrors.Errorf("amt load: %w", err)
}
@ -1124,7 +1097,7 @@ func (cs *ChainStore) ReadMsgMetaCids(mmc cid.Cid) ([]cid.Cid, []cid.Cid, error)
return mmcids.bls, mmcids.secpk, nil
}
cst := cbor.NewCborStore(cs.localbs)
cst := cbor.NewCborStore(cs.chainLocalBlockstore)
var msgmeta types.MsgMeta
if err := cst.Get(context.TODO(), mmc, &msgmeta); err != nil {
return nil, nil, xerrors.Errorf("failed to load msgmeta (%s): %w", mmc, err)
@ -1194,7 +1167,7 @@ func (cs *ChainStore) MessagesForBlock(b *types.BlockHeader) ([]*types.Message,
func (cs *ChainStore) GetParentReceipt(b *types.BlockHeader, i int) (*types.MessageReceipt, error) {
ctx := context.TODO()
// block headers use adt0, for now.
a, err := blockadt.AsArray(cs.Store(ctx), b.ParentMessageReceipts)
a, err := blockadt.AsArray(cs.ActorStore(ctx), b.ParentMessageReceipts)
if err != nil {
return nil, xerrors.Errorf("amt load: %w", err)
}
@ -1237,16 +1210,26 @@ func (cs *ChainStore) LoadSignedMessagesFromCids(cids []cid.Cid) ([]*types.Signe
return msgs, nil
}
func (cs *ChainStore) Blockstore() bstore.Blockstore {
return cs.bs
// ChainBlockstore returns the chain blockstore. Currently the chain and state
// // stores are both backed by the same physical store, albeit with different
// // caching policies, but in the future they will segregate.
func (cs *ChainStore) ChainBlockstore() bstore.Blockstore {
return cs.chainBlockstore
}
// StateBlockstore returns the state blockstore. Currently the chain and state
// stores are both backed by the same physical store, albeit with different
// caching policies, but in the future they will segregate.
func (cs *ChainStore) StateBlockstore() bstore.Blockstore {
return cs.stateBlockstore
}
func ActorStore(ctx context.Context, bs bstore.Blockstore) adt.Store {
return adt.WrapStore(ctx, cbor.NewCborStore(bs))
}
func (cs *ChainStore) Store(ctx context.Context) adt.Store {
return ActorStore(ctx, cs.bs)
func (cs *ChainStore) ActorStore(ctx context.Context) adt.Store {
return ActorStore(ctx, cs.stateBlockstore)
}
func (cs *ChainStore) VMSys() vm.SyscallBuilder {
@ -1444,8 +1427,9 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo
return xerrors.Errorf("failed to write car header: %s", err)
}
return cs.WalkSnapshot(ctx, ts, inclRecentRoots, skipOldMsgs, func(c cid.Cid) error {
blk, err := cs.bs.Get(c)
unionBs := bstore.Union(cs.stateBlockstore, cs.chainBlockstore)
return cs.WalkSnapshot(ctx, ts, inclRecentRoots, skipOldMsgs, true, func(c cid.Cid) error {
blk, err := unionBs.Get(c)
if err != nil {
return xerrors.Errorf("writing object to car, bs.Get: %w", err)
}
@ -1458,7 +1442,7 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo
})
}
func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, skipOldMsgs bool, cb func(cid.Cid) error) error {
func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, skipOldMsgs, skipMsgReceipts bool, cb func(cid.Cid) error) error {
if ts == nil {
ts = cs.GetHeaviestTipSet()
}
@ -1478,7 +1462,7 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
return err
}
data, err := cs.bs.Get(blk)
data, err := cs.chainBlockstore.Get(blk)
if err != nil {
return xerrors.Errorf("getting block: %w", err)
}
@ -1498,7 +1482,7 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
var cids []cid.Cid
if !skipOldMsgs || b.Height > ts.Height()-inclRecentRoots {
if walked.Visit(b.Messages) {
mcids, err := recurseLinks(cs.bs, walked, b.Messages, []cid.Cid{b.Messages})
mcids, err := recurseLinks(cs.chainBlockstore, walked, b.Messages, []cid.Cid{b.Messages})
if err != nil {
return xerrors.Errorf("recursing messages failed: %w", err)
}
@ -1519,13 +1503,17 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
if b.Height == 0 || b.Height > ts.Height()-inclRecentRoots {
if walked.Visit(b.ParentStateRoot) {
cids, err := recurseLinks(cs.bs, walked, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot})
cids, err := recurseLinks(cs.stateBlockstore, walked, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot})
if err != nil {
return xerrors.Errorf("recursing genesis state failed: %w", err)
}
out = append(out, cids...)
}
if !skipMsgReceipts && walked.Visit(b.ParentMessageReceipts) {
out = append(out, b.ParentMessageReceipts)
}
}
for _, c := range out {
@ -1561,7 +1549,12 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
}
func (cs *ChainStore) Import(r io.Reader) (*types.TipSet, error) {
header, err := car.LoadCar(cs.Blockstore(), r)
// TODO: writing only to the state blockstore is incorrect.
// At this time, both the state and chain blockstores are backed by the
// universal store. When we physically segregate the stores, we will need
// to route state objects to the state blockstore, and chain objects to
// the chain blockstore.
header, err := car.LoadCar(cs.StateBlockstore(), r)
if err != nil {
return nil, xerrors.Errorf("loadcar failed: %w", err)
}

8
chain/store/store_test.go
View File

@ -11,12 +11,12 @@ import (
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/actors/policy"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/lotus/node/repo"
)
@ -52,7 +52,7 @@ func BenchmarkGetRandomness(b *testing.B) {
b.Fatal(err)
}
bs, err := lr.Blockstore(context.TODO(), repo.BlockstoreChain)
bs, err := lr.Blockstore(context.TODO(), repo.UniversalBlockstore)
if err != nil {
b.Fatal(err)
}
@ -104,7 +104,7 @@ func TestChainExportImport(t *testing.T) {
t.Fatal(err)
}
nbs := blockstore.NewTemporary()
nbs := blockstore.NewMemory()
cs := store.NewChainStore(nbs, nbs, datastore.NewMapDatastore(), nil, nil)
defer cs.Close() //nolint:errcheck
@ -139,7 +139,7 @@ func TestChainExportImportFull(t *testing.T) {
t.Fatal(err)
}
nbs := blockstore.NewTemporary()
nbs := blockstore.NewMemory()
cs := store.NewChainStore(nbs, nbs, datastore.NewMapDatastore(), nil, nil)
defer cs.Close() //nolint:errcheck

4
chain/store/weight.go
View File

@ -28,7 +28,7 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn
tpow := big2.Zero()
{
cst := cbor.NewCborStore(cs.Blockstore())
cst := cbor.NewCborStore(cs.StateBlockstore())
state, err := state.LoadStateTree(cst, ts.ParentState())
if err != nil {
return types.NewInt(0), xerrors.Errorf("load state tree: %w", err)
@ -39,7 +39,7 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn
return types.NewInt(0), xerrors.Errorf("get power actor: %w", err)
}
powState, err := power.Load(cs.Store(ctx), act)
powState, err := power.Load(cs.ActorStore(ctx), act)
if err != nil {
return types.NewInt(0), xerrors.Errorf("failed to load power actor state: %w", err)
}

6
chain/sub/incoming.go
View File

@ -7,13 +7,13 @@ import (
"time"
address "github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain"
"github.com/filecoin-project/lotus/chain/messagepool"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/lotus/lib/sigs"
"github.com/filecoin-project/lotus/metrics"
"github.com/filecoin-project/lotus/node/impl/client"
@ -101,7 +101,7 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha
[]tag.Mutator{tag.Insert(metrics.MinerID, blk.Header.Miner.String())},
metrics.BlockDelay.M(delay),
)
log.Warnf("Received block with large delay %d from miner %s", delay, blk.Header.Miner)
log.Warnw("received block with large delay from miner", "block", blk.Cid(), "delay", delay, "miner", blk.Header.Miner)
}
if s.InformNewBlock(msg.ReceivedFrom, &types.FullBlock{
@ -392,7 +392,7 @@ func (bv *BlockValidator) isChainNearSynced() bool {
func (bv *BlockValidator) validateMsgMeta(ctx context.Context, msg *types.BlockMsg) error {
// TODO there has to be a simpler way to do this without the blockstore dance
// block headers use adt0
store := blockadt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewTemporary()))
store := blockadt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewMemory()))
bmArr := blockadt.MakeEmptyArray(store)
smArr := blockadt.MakeEmptyArray(store)

18
chain/sync.go
View File

@ -44,6 +44,7 @@ import (
proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof"
"github.com/filecoin-project/lotus/api"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/builtin/power"
"github.com/filecoin-project/lotus/chain/beacon"
@ -54,7 +55,6 @@ import (
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/lotus/lib/sigs"
"github.com/filecoin-project/lotus/metrics"
)
@ -321,7 +321,7 @@ func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error {
// We use a temporary bstore here to avoid writing intermediate pieces
// into the blockstore.
blockstore := bstore.NewTemporary()
blockstore := bstore.NewMemory()
cst := cbor.NewCborStore(blockstore)
var bcids, scids []cid.Cid
@ -354,7 +354,7 @@ func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error {
}
// Finally, flush.
return vm.Copy(context.TODO(), blockstore, syncer.store.Blockstore(), smroot)
return vm.Copy(context.TODO(), blockstore, syncer.store.ChainBlockstore(), smroot)
}
func (syncer *Syncer) LocalPeer() peer.ID {
@ -640,7 +640,7 @@ func (syncer *Syncer) minerIsValid(ctx context.Context, maddr address.Address, b
return xerrors.Errorf("failed to load power actor: %w", err)
}
powState, err := power.Load(syncer.store.Store(ctx), act)
powState, err := power.Load(syncer.store.ActorStore(ctx), act)
if err != nil {
return xerrors.Errorf("failed to load power actor state: %w", err)
}
@ -1055,7 +1055,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock
return err
}
st, err := state.LoadStateTree(syncer.store.Store(ctx), stateroot)
st, err := state.LoadStateTree(syncer.store.ActorStore(ctx), stateroot)
if err != nil {
return xerrors.Errorf("failed to load base state tree: %w", err)
}
@ -1102,7 +1102,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock
}
// Validate message arrays in a temporary blockstore.
tmpbs := bstore.NewTemporary()
tmpbs := bstore.NewMemory()
tmpstore := blockadt.WrapStore(ctx, cbor.NewCborStore(tmpbs))
bmArr := blockadt.MakeEmptyArray(tmpstore)
@ -1172,7 +1172,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock
}
// Finally, flush.
return vm.Copy(ctx, tmpbs, syncer.store.Blockstore(), mrcid)
return vm.Copy(ctx, tmpbs, syncer.store.ChainBlockstore(), mrcid)
}
func (syncer *Syncer) verifyBlsAggregate(ctx context.Context, sig *crypto.Signature, msgs []cid.Cid, pubks [][]byte) error {
@ -1553,7 +1553,7 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS
for bsi := 0; bsi < len(bstout); bsi++ {
// temp storage so we don't persist data we dont want to
bs := bstore.NewTemporary()
bs := bstore.NewMemory()
blks := cbor.NewCborStore(bs)
this := headers[i-bsi]
@ -1574,7 +1574,7 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS
return err
}
if err := copyBlockstore(ctx, bs, syncer.store.Blockstore()); err != nil {
if err := copyBlockstore(ctx, bs, syncer.store.ChainBlockstore()); err != nil {
return xerrors.Errorf("message processing failed: %w", err)
}
}

10
chain/vm/vm.go
View File

@ -28,6 +28,7 @@ import (
"github.com/filecoin-project/go-state-types/exitcode"
"github.com/filecoin-project/go-state-types/network"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/adt"
"github.com/filecoin-project/lotus/chain/actors/aerrors"
@ -36,9 +37,6 @@ import (
"github.com/filecoin-project/lotus/chain/actors/builtin/reward"
"github.com/filecoin-project/lotus/chain/state"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/blockstore"
bstore "github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/lotus/lib/bufbstore"
)
const MaxCallDepth = 4096
@ -208,7 +206,7 @@ type VM struct {
cstate *state.StateTree
base cid.Cid
cst *cbor.BasicIpldStore
buf *bufbstore.BufferedBS
buf *blockstore.BufferedBlockstore
blockHeight abi.ChainEpoch
areg *ActorRegistry
rand Rand
@ -224,7 +222,7 @@ type VMOpts struct {
StateBase cid.Cid
Epoch abi.ChainEpoch
Rand Rand
Bstore bstore.Blockstore
Bstore blockstore.Blockstore
Syscalls SyscallBuilder
CircSupplyCalc CircSupplyCalculator
NtwkVersion NtwkVersionGetter // TODO: stebalien: In what cases do we actually need this? It seems like even when creating new networks we want to use the 'global'/build-default version getter
@ -233,7 +231,7 @@ type VMOpts struct {
}
func NewVM(ctx context.Context, opts *VMOpts) (*VM, error) {
buf := bufbstore.NewBufferedBstore(opts.Bstore)
buf := blockstore.NewBuffered(opts.Bstore)
cst := cbor.NewCborStore(buf)
state, err := state.LoadStateTree(cst, opts.StateBase)
if err != nil {

3
cli/auth.go
View File

@ -9,6 +9,7 @@ import (
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/filecoin-project/lotus/api/apistruct"
cliutil "github.com/filecoin-project/lotus/cli/util"
"github.com/filecoin-project/lotus/node/repo"
)
@ -127,7 +128,7 @@ var authApiInfoToken = &cli.Command{
// TODO: Log in audit log when it is implemented
fmt.Printf("%s=%s:%s\n", envForRepo(t), string(token), ainfo.Addr)
fmt.Printf("%s=%s:%s\n", cliutil.EnvForRepo(t), string(token), ainfo.Addr)
return nil
},
}

5
cli/backup.go
View File

@ -51,7 +51,10 @@ func BackupCmd(repoFlag string, rt repo.RepoType, getApi BackupApiFn) *cli.Comma
return xerrors.Errorf("getting metadata datastore: %w", err)
}
bds := backupds.Wrap(mds)
bds, err := backupds.Wrap(mds, backupds.NoLogdir)
if err != nil {
return err
}
fpath, err := homedir.Expand(cctx.Args().First())
if err != nil {

5
cli/chain.go
View File

@ -642,7 +642,10 @@ var chainListCmd = &cli.Command{
gasUsed += r.GasUsed
}
fmt.Printf("\ttipset: \t%d msgs, %d / %d (%0.2f%%)\n", len(msgs), gasUsed, limitSum, 100*float64(gasUsed)/float64(limitSum))
gasEfficiency := 100 * float64(gasUsed) / float64(limitSum)
gasCapacity := 100 * float64(limitSum) / float64(build.BlockGasLimit)
fmt.Printf("\ttipset: \t%d msgs, %d (%0.2f%%) / %d (%0.2f%%)\n", len(msgs), gasUsed, gasEfficiency, limitSum, gasCapacity)
}
fmt.Println()
}

26
cli/client.go
View File

@ -1195,6 +1195,11 @@ var clientListAsksCmd = &cli.Command{
&cli.BoolFlag{
Name: "by-ping",
},
&cli.StringFlag{
Name: "output-format",
Value: "text",
Usage: "Either 'text' or 'csv'",
},
},
Action: func(cctx *cli.Context) error {
api, closer, err := GetFullNodeAPI(cctx)
@ -1214,11 +1219,16 @@ var clientListAsksCmd = &cli.Command{
return asks[i].Ping < asks[j].Ping
})
}
pfmt := "%s: min:%s max:%s price:%s/GiB/Epoch verifiedPrice:%s/GiB/Epoch ping:%s\n"
if cctx.String("output-format") == "csv" {
fmt.Printf("Miner,Min,Max,Price,VerifiedPrice,Ping\n")
pfmt = "%s,%s,%s,%s,%s,%s\n"
}
for _, a := range asks {
ask := a.Ask
fmt.Printf("%s: min:%s max:%s price:%s/GiB/Epoch verifiedPrice:%s/GiB/Epoch ping:%s\n", ask.Miner,
fmt.Printf(pfmt, ask.Miner,
types.SizeStr(types.NewInt(uint64(ask.MinPieceSize))),
types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize))),
types.FIL(ask.Price),
@ -1237,7 +1247,13 @@ type QueriedAsk struct {
}
func GetAsks(ctx context.Context, api lapi.FullNode) ([]QueriedAsk, error) {
isTTY := true
if fileInfo, _ := os.Stdout.Stat(); (fileInfo.Mode() & os.ModeCharDevice) == 0 {
isTTY = false
}
if isTTY {
color.Blue(".. getting miner list")
}
miners, err := api.StateListMiners(ctx, types.EmptyTSK)
if err != nil {
return nil, xerrors.Errorf("getting miner list: %w", err)
@ -1282,14 +1298,18 @@ loop:
for {
select {
case <-time.After(150 * time.Millisecond):
if isTTY {
fmt.Printf("\r* Found %d miners with power", atomic.LoadInt64(&found))
}
case <-done:
break loop
}
}
if isTTY {
fmt.Printf("\r* Found %d miners with power\n", atomic.LoadInt64(&found))
color.Blue(".. querying asks")
}
var asks []QueriedAsk
var queried, got int64
@ -1349,12 +1369,16 @@ loop2:
for {
select {
case <-time.After(150 * time.Millisecond):
if isTTY {
fmt.Printf("\r* Queried %d asks, got %d responses", atomic.LoadInt64(&queried), atomic.LoadInt64(&got))
}
case <-done:
break loop2
}
}
if isTTY {
fmt.Printf("\r* Queried %d asks, got %d responses\n", atomic.LoadInt64(&queried), atomic.LoadInt64(&got))
}
sort.Slice(asks, func(i, j int) bool {
return asks[i].Ask.Price.LessThan(asks[j].Ask.Price)

269
cli/cmd.go
View File

@ -1,34 +1,17 @@
package cli
import (
"context"
"fmt"
"net/http"
"net/url"
"os"
"os/signal"
"strings"
"syscall"
logging "github.com/ipfs/go-log/v2"
"github.com/mitchellh/go-homedir"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-jsonrpc"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/client"
cliutil "github.com/filecoin-project/lotus/cli/util"
"github.com/filecoin-project/lotus/node/repo"
)
var log = logging.Logger("cli")
const (
metadataTraceContext = "traceContext"
)
// custom CLI error
type ErrCmdFailed struct {
@ -46,253 +29,31 @@ func NewCliError(s string) error {
// ApiConnector returns API instance
type ApiConnector func() api.FullNode
// The flag passed on the command line with the listen address of the API
// server (only used by the tests)
func flagForAPI(t repo.RepoType) string {
switch t {
case repo.FullNode:
return "api-url"
case repo.StorageMiner:
return "miner-api-url"
case repo.Worker:
return "worker-api-url"
default:
panic(fmt.Sprintf("Unknown repo type: %v", t))
}
}
func flagForRepo(t repo.RepoType) string {
switch t {
case repo.FullNode:
return "repo"
case repo.StorageMiner:
return "miner-repo"
case repo.Worker:
return "worker-repo"
default:
panic(fmt.Sprintf("Unknown repo type: %v", t))
}
}
func envForRepo(t repo.RepoType) string {
switch t {
case repo.FullNode:
return "FULLNODE_API_INFO"
case repo.StorageMiner:
return "MINER_API_INFO"
case repo.Worker:
return "WORKER_API_INFO"
default:
panic(fmt.Sprintf("Unknown repo type: %v", t))
}
}
// TODO remove after deprecation period
func envForRepoDeprecation(t repo.RepoType) string {
switch t {
case repo.FullNode:
return "FULLNODE_API_INFO"
case repo.StorageMiner:
return "STORAGE_API_INFO"
case repo.Worker:
return "WORKER_API_INFO"
default:
panic(fmt.Sprintf("Unknown repo type: %v", t))
}
}
func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (cliutil.APIInfo, error) {
// Check if there was a flag passed with the listen address of the API
// server (only used by the tests)
apiFlag := flagForAPI(t)
if ctx.IsSet(apiFlag) {
strma := ctx.String(apiFlag)
strma = strings.TrimSpace(strma)
return cliutil.APIInfo{Addr: strma}, nil
func GetFullNodeServices(ctx *cli.Context) (ServicesAPI, error) {
if tn, ok := ctx.App.Metadata["test-services"]; ok {
return tn.(ServicesAPI), nil
}
envKey := envForRepo(t)
env, ok := os.LookupEnv(envKey)
if !ok {
// TODO remove after deprecation period
envKey = envForRepoDeprecation(t)
env, ok = os.LookupEnv(envKey)
if ok {
log.Warnf("Use deprecation env(%s) value, please use env(%s) instead.", envKey, envForRepo(t))
}
}
if ok {
return cliutil.ParseApiInfo(env), nil
}
repoFlag := flagForRepo(t)
p, err := homedir.Expand(ctx.String(repoFlag))
api, c, err := GetFullNodeAPI(ctx)
if err != nil {
return cliutil.APIInfo{}, xerrors.Errorf("could not expand home dir (%s): %w", repoFlag, err)
return nil, err
}
r, err := repo.NewFS(p)
if err != nil {
return cliutil.APIInfo{}, xerrors.Errorf("could not open repo at path: %s; %w", p, err)
}
ma, err := r.APIEndpoint()
if err != nil {
return cliutil.APIInfo{}, xerrors.Errorf("could not get api endpoint: %w", err)
}
token, err := r.APIToken()
if err != nil {
log.Warnf("Couldn't load CLI token, capabilities may be limited: %v", err)
}
return cliutil.APIInfo{
Addr: ma.String(),
Token: token,
}, nil
return &ServicesImpl{api: api, closer: c}, nil
}
func GetRawAPI(ctx *cli.Context, t repo.RepoType) (string, http.Header, error) {
ainfo, err := GetAPIInfo(ctx, t)
if err != nil {
return "", nil, xerrors.Errorf("could not get API info: %w", err)
}
var GetAPIInfo = cliutil.GetAPIInfo
var GetRawAPI = cliutil.GetRawAPI
var GetAPI = cliutil.GetAPI
addr, err := ainfo.DialArgs()
if err != nil {
return "", nil, xerrors.Errorf("could not get DialArgs: %w", err)
}
var DaemonContext = cliutil.DaemonContext
var ReqContext = cliutil.ReqContext
return addr, ainfo.AuthHeader(), nil
}
var GetFullNodeAPI = cliutil.GetFullNodeAPI
var GetGatewayAPI = cliutil.GetGatewayAPI
func GetAPI(ctx *cli.Context) (api.Common, jsonrpc.ClientCloser, error) {
ti, ok := ctx.App.Metadata["repoType"]
if !ok {
log.Errorf("unknown repo type, are you sure you want to use GetAPI?")
ti = repo.FullNode
}
t, ok := ti.(repo.RepoType)
if !ok {
log.Errorf("repoType type does not match the type of repo.RepoType")
}
if tn, ok := ctx.App.Metadata["testnode-storage"]; ok {
return tn.(api.StorageMiner), func() {}, nil
}
if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
return tn.(api.FullNode), func() {}, nil
}
addr, headers, err := GetRawAPI(ctx, t)
if err != nil {
return nil, nil, err
}
return client.NewCommonRPC(ctx.Context, addr, headers)
}
func GetFullNodeAPI(ctx *cli.Context) (api.FullNode, jsonrpc.ClientCloser, error) {
if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
return tn.(api.FullNode), func() {}, nil
}
addr, headers, err := GetRawAPI(ctx, repo.FullNode)
if err != nil {
return nil, nil, err
}
return client.NewFullNodeRPC(ctx.Context, addr, headers)
}
type GetStorageMinerOptions struct {
PreferHttp bool
}
type GetStorageMinerOption func(*GetStorageMinerOptions)
func StorageMinerUseHttp(opts *GetStorageMinerOptions) {
opts.PreferHttp = true
}
func GetStorageMinerAPI(ctx *cli.Context, opts ...GetStorageMinerOption) (api.StorageMiner, jsonrpc.ClientCloser, error) {
var options GetStorageMinerOptions
for _, opt := range opts {
opt(&options)
}
if tn, ok := ctx.App.Metadata["testnode-storage"]; ok {
return tn.(api.StorageMiner), func() {}, nil
}
addr, headers, err := GetRawAPI(ctx, repo.StorageMiner)
if err != nil {
return nil, nil, err
}
if options.PreferHttp {
u, err := url.Parse(addr)
if err != nil {
return nil, nil, xerrors.Errorf("parsing miner api URL: %w", err)
}
switch u.Scheme {
case "ws":
u.Scheme = "http"
case "wss":
u.Scheme = "https"
}
addr = u.String()
}
return client.NewStorageMinerRPC(ctx.Context, addr, headers)
}
func GetWorkerAPI(ctx *cli.Context) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
addr, headers, err := GetRawAPI(ctx, repo.Worker)
if err != nil {
return nil, nil, err
}
return client.NewWorkerRPC(ctx.Context, addr, headers)
}
func GetGatewayAPI(ctx *cli.Context) (api.GatewayAPI, jsonrpc.ClientCloser, error) {
addr, headers, err := GetRawAPI(ctx, repo.FullNode)
if err != nil {
return nil, nil, err
}
return client.NewGatewayRPC(ctx.Context, addr, headers)
}
func DaemonContext(cctx *cli.Context) context.Context {
if mtCtx, ok := cctx.App.Metadata[metadataTraceContext]; ok {
return mtCtx.(context.Context)
}
return context.Background()
}
// ReqContext returns context for cli execution. Calling it for the first time
// installs SIGTERM handler that will close returned context.
// Not safe for concurrent execution.
func ReqContext(cctx *cli.Context) context.Context {
tCtx := DaemonContext(cctx)
ctx, done := context.WithCancel(tCtx)
sigChan := make(chan os.Signal, 2)
go func() {
<-sigChan
done()
}()
signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP)
return ctx
}
var GetStorageMinerAPI = cliutil.GetStorageMinerAPI
var GetWorkerAPI = cliutil.GetWorkerAPI
var CommonCommands = []*cli.Command{
netCmd,

8
cli/multisig.go
View File

@ -29,7 +29,7 @@ import (
init2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init"
msig2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig"
"github.com/filecoin-project/lotus/api/apibstore"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/adt"
"github.com/filecoin-project/lotus/chain/actors/builtin/multisig"
@ -202,7 +202,7 @@ var msigInspectCmd = &cli.Command{
defer closer()
ctx := ReqContext(cctx)
store := adt.WrapStore(ctx, cbor.NewCborStore(apibstore.NewAPIBlockstore(api)))
store := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewAPIBlockstore(api)))
maddr, err := address.NewFromString(cctx.Args().First())
if err != nil {
@ -1275,7 +1275,7 @@ var msigLockApproveCmd = &cli.Command{
params, actErr := actors.SerializeParams(&msig2.LockBalanceParams{
StartEpoch: abi.ChainEpoch(start),
UnlockDuration: abi.ChainEpoch(duration),
Amount: abi.NewTokenAmount(amount.Int64()),
Amount: big.Int(amount),
})
if actErr != nil {
@ -1367,7 +1367,7 @@ var msigLockCancelCmd = &cli.Command{
params, actErr := actors.SerializeParams(&msig2.LockBalanceParams{
StartEpoch: abi.ChainEpoch(start),
UnlockDuration: abi.ChainEpoch(duration),
Amount: abi.NewTokenAmount(amount.Int64()),
Amount: big.Int(amount),
})
if actErr != nil {

32
cli/net.go
View File

@ -48,6 +48,11 @@ var NetPeers = &cli.Command{
Aliases: []string{"a"},
Usage: "Print agent name",
},
&cli.BoolFlag{
Name: "extended",
Aliases: []string{"x"},
Usage: "Print extended peer information in json",
},
},
Action: func(cctx *cli.Context) error {
api, closer, err := GetAPI(cctx)
@ -65,6 +70,30 @@ var NetPeers = &cli.Command{
return strings.Compare(string(peers[i].ID), string(peers[j].ID)) > 0
})
if cctx.Bool("extended") {
// deduplicate
seen := make(map[peer.ID]struct{})
for _, peer := range peers {
_, dup := seen[peer.ID]
if dup {
continue
}
seen[peer.ID] = struct{}{}
info, err := api.NetPeerInfo(ctx, peer.ID)
if err != nil {
log.Warnf("error getting extended peer info: %s", err)
} else {
bytes, err := json.Marshal(&info)
if err != nil {
log.Warnf("error marshalling extended peer info: %s", err)
} else {
fmt.Println(string(bytes))
}
}
}
} else {
for _, peer := range peers {
var agent string
if cctx.Bool("agent") {
@ -75,9 +104,9 @@ var NetPeers = &cli.Command{
agent = ", " + agent
}
}
fmt.Printf("%s, %s%s\n", peer.ID, peer.Addrs, agent)
}
}
return nil
},
@ -89,6 +118,7 @@ var netScores = &cli.Command{
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "extended",
Aliases: []string{"x"},
Usage: "print extended peer scores in json",
},
},

4
cli/paych_test.go
View File

@ -20,8 +20,8 @@ import (
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/lotus/api/apibstore"
"github.com/filecoin-project/lotus/api/test"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/events"
"github.com/filecoin-project/lotus/chain/types"
@ -400,7 +400,7 @@ func getPaychState(ctx context.Context, t *testing.T, node test.TestNode, chAddr
act, err := node.StateGetActor(ctx, chAddr, types.EmptyTSK)
require.NoError(t, err)
store := cbor.NewCborStore(apibstore.NewAPIBlockstore(node))
store := cbor.NewCborStore(blockstore.NewAPIBlockstore(node))
chState, err := paych.Load(adt.WrapStore(ctx, store), act)
require.NoError(t, err)

122
cli/send.go
View File

@ -1,22 +1,17 @@
package cli
import (
"bytes"
"context"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"reflect"
"github.com/urfave/cli/v2"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
)
@ -72,15 +67,16 @@ var sendCmd = &cli.Command{
return ShowHelp(cctx, fmt.Errorf("'send' expects two arguments, target and amount"))
}
api, closer, err := GetFullNodeAPI(cctx)
srv, err := GetFullNodeServices(cctx)
if err != nil {
return err
}
defer closer()
defer srv.Close() //nolint:errcheck
ctx := ReqContext(cctx)
var params SendParams
toAddr, err := address.NewFromString(cctx.Args().Get(0))
params.To, err = address.NewFromString(cctx.Args().Get(0))
if err != nil {
return ShowHelp(cctx, fmt.Errorf("failed to parse target address: %w", err))
}
@ -89,123 +85,75 @@ var sendCmd = &cli.Command{
if err != nil {
return ShowHelp(cctx, fmt.Errorf("failed to parse amount: %w", err))
}
params.Val = abi.TokenAmount(val)
var fromAddr address.Address
if from := cctx.String("from"); from == "" {
defaddr, err := api.WalletDefaultAddress(ctx)
if err != nil {
return err
}
fromAddr = defaddr
} else {
if from := cctx.String("from"); from != "" {
addr, err := address.NewFromString(from)
if err != nil {
return err
}
fromAddr = addr
params.From = addr
}
if cctx.IsSet("gas-premium") {
gp, err := types.BigFromString(cctx.String("gas-premium"))
if err != nil {
return err
}
params.GasPremium = &gp
}
if cctx.IsSet("gas-feecap") {
gfc, err := types.BigFromString(cctx.String("gas-feecap"))
if err != nil {
return err
}
params.GasFeeCap = &gfc
}
method := abi.MethodNum(cctx.Uint64("method"))
if cctx.IsSet("gas-limit") {
limit := cctx.Int64("gas-limit")
params.GasLimit = &limit
}
params.Method = abi.MethodNum(cctx.Uint64("method"))
var params []byte
if cctx.IsSet("params-json") {
decparams, err := decodeTypedParams(ctx, api, toAddr, method, cctx.String("params-json"))
decparams, err := srv.DecodeTypedParamsFromJSON(ctx, params.To, params.Method, cctx.String("params-json"))
if err != nil {
return fmt.Errorf("failed to decode json params: %w", err)
}
params = decparams
params.Params = decparams
}
if cctx.IsSet("params-hex") {
if params != nil {
if params.Params != nil {
return fmt.Errorf("can only specify one of 'params-json' and 'params-hex'")
}
decparams, err := hex.DecodeString(cctx.String("params-hex"))
if err != nil {
return fmt.Errorf("failed to decode hex params: %w", err)
}
params = decparams
params.Params = decparams
}
msg := &types.Message{
From: fromAddr,
To: toAddr,
Value: types.BigInt(val),
GasPremium: gp,
GasFeeCap: gfc,
GasLimit: cctx.Int64("gas-limit"),
Method: method,
Params: params,
}
if !cctx.Bool("force") {
// Funds insufficient check
fromBalance, err := api.WalletBalance(ctx, msg.From)
if err != nil {
return err
}
totalCost := types.BigAdd(types.BigMul(msg.GasFeeCap, types.NewInt(uint64(msg.GasLimit))), msg.Value)
if fromBalance.LessThan(totalCost) {
fmt.Printf("WARNING: From balance %s less than total cost %s\n", types.FIL(fromBalance), types.FIL(totalCost))
return fmt.Errorf("--force must be specified for this action to have an effect; you have been warned")
}
}
params.Force = cctx.Bool("force")
if cctx.IsSet("nonce") {
msg.Nonce = cctx.Uint64("nonce")
sm, err := api.WalletSignMessage(ctx, fromAddr, msg)
if err != nil {
return err
n := cctx.Uint64("nonce")
params.Nonce = &n
}
_, err = api.MpoolPush(ctx, sm)
msgCid, err := srv.Send(ctx, params)
if err != nil {
return err
if errors.Is(err, ErrSendBalanceTooLow) {
return fmt.Errorf("--force must be specified for this action to have an effect; you have been warned: %w", err)
}
fmt.Println(sm.Cid())
} else {
sm, err := api.MpoolPushMessage(ctx, msg, nil)
if err != nil {
return err
}
fmt.Println(sm.Cid())
return xerrors.Errorf("executing send: %w", err)
}
fmt.Fprintf(cctx.App.Writer, "%s\n", msgCid)
return nil
},
}
func decodeTypedParams(ctx context.Context, fapi api.FullNode, to address.Address, method abi.MethodNum, paramstr string) ([]byte, error) {
act, err := fapi.StateGetActor(ctx, to, types.EmptyTSK)
if err != nil {
return nil, err
}
methodMeta, found := stmgr.MethodsMap[act.Code][method]
if !found {
return nil, fmt.Errorf("method %d not found on actor %s", method, act.Code)
}
p := reflect.New(methodMeta.Params.Elem()).Interface().(cbg.CBORMarshaler)
if err := json.Unmarshal([]byte(paramstr), p); err != nil {
return nil, fmt.Errorf("unmarshaling input into params type: %w", err)
}
buf := new(bytes.Buffer)
if err := p.MarshalCBOR(buf); err != nil {
return nil, err
}
return buf.Bytes(), nil
}

128
cli/send_test.go Normal file
View File

@ -0,0 +1,128 @@
package cli
import (
"bytes"
"errors"
"testing"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
types "github.com/filecoin-project/lotus/chain/types"
gomock "github.com/golang/mock/gomock"
cid "github.com/ipfs/go-cid"
"github.com/stretchr/testify/assert"
ucli "github.com/urfave/cli/v2"
)
var arbtCid = (&types.Message{
From: mustAddr(address.NewIDAddress(2)),
To: mustAddr(address.NewIDAddress(1)),
Value: types.NewInt(1000),
}).Cid()
func mustAddr(a address.Address, err error) address.Address {
if err != nil {
panic(err)
}
return a
}
func newMockApp(t *testing.T, cmd *ucli.Command) (*ucli.App, *MockServicesAPI, *bytes.Buffer, func()) {
app := ucli.NewApp()
app.Commands = ucli.Commands{cmd}
app.Setup()
mockCtrl := gomock.NewController(t)
mockSrvcs := NewMockServicesAPI(mockCtrl)
app.Metadata["test-services"] = mockSrvcs
buf := &bytes.Buffer{}
app.Writer = buf
return app, mockSrvcs, buf, mockCtrl.Finish
}
func TestSendCLI(t *testing.T) {
oneFil := abi.TokenAmount(types.MustParseFIL("1"))
t.Run("simple", func(t *testing.T) {
app, mockSrvcs, buf, done := newMockApp(t, sendCmd)
defer done()
gomock.InOrder(
mockSrvcs.EXPECT().Send(gomock.Any(), SendParams{
To: mustAddr(address.NewIDAddress(1)),
Val: oneFil,
}).Return(arbtCid, nil),
mockSrvcs.EXPECT().Close(),
)
err := app.Run([]string{"lotus", "send", "t01", "1"})
assert.NoError(t, err)
assert.EqualValues(t, arbtCid.String()+"\n", buf.String())
})
t.Run("ErrSendBalanceTooLow", func(t *testing.T) {
app, mockSrvcs, _, done := newMockApp(t, sendCmd)
defer done()
gomock.InOrder(
mockSrvcs.EXPECT().Send(gomock.Any(), SendParams{
To: mustAddr(address.NewIDAddress(1)),
Val: oneFil,
}).Return(cid.Undef, ErrSendBalanceTooLow),
mockSrvcs.EXPECT().Close(),
)
err := app.Run([]string{"lotus", "send", "t01", "1"})
assert.ErrorIs(t, err, ErrSendBalanceTooLow)
})
t.Run("generic-err-is-forwarded", func(t *testing.T) {
app, mockSrvcs, _, done := newMockApp(t, sendCmd)
defer done()
errMark := errors.New("something")
gomock.InOrder(
mockSrvcs.EXPECT().Send(gomock.Any(), SendParams{
To: mustAddr(address.NewIDAddress(1)),
Val: oneFil,
}).Return(cid.Undef, errMark),
mockSrvcs.EXPECT().Close(),
)
err := app.Run([]string{"lotus", "send", "t01", "1"})
assert.ErrorIs(t, err, errMark)
})
t.Run("from-specific", func(t *testing.T) {
app, mockSrvcs, buf, done := newMockApp(t, sendCmd)
defer done()
gomock.InOrder(
mockSrvcs.EXPECT().Send(gomock.Any(), SendParams{
To: mustAddr(address.NewIDAddress(1)),
From: mustAddr(address.NewIDAddress(2)),
Val: oneFil,
}).Return(arbtCid, nil),
mockSrvcs.EXPECT().Close(),
)
err := app.Run([]string{"lotus", "send", "--from=t02", "t01", "1"})
assert.NoError(t, err)
assert.EqualValues(t, arbtCid.String()+"\n", buf.String())
})
t.Run("nonce-specific", func(t *testing.T) {
app, mockSrvcs, buf, done := newMockApp(t, sendCmd)
defer done()
zero := uint64(0)
gomock.InOrder(
mockSrvcs.EXPECT().Send(gomock.Any(), SendParams{
To: mustAddr(address.NewIDAddress(1)),
Nonce: &zero,
Val: oneFil,
}).Return(arbtCid, nil),
mockSrvcs.EXPECT().Close(),
)
err := app.Run([]string{"lotus", "send", "--nonce=0", "t01", "1"})
assert.NoError(t, err)
assert.EqualValues(t, arbtCid.String()+"\n", buf.String())
})
}

166
cli/services.go Normal file
View File

@ -0,0 +1,166 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-jsonrpc"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/stmgr"
types "github.com/filecoin-project/lotus/chain/types"
cid "github.com/ipfs/go-cid"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
)
//go:generate go run github.com/golang/mock/mockgen -destination=servicesmock_test.go -package=cli -self_package github.com/filecoin-project/lotus/cli . ServicesAPI
type ServicesAPI interface {
// Sends executes a send given SendParams
Send(ctx context.Context, params SendParams) (cid.Cid, error)
// DecodeTypedParamsFromJSON takes in information needed to identify a method and converts JSON
// parameters to bytes of their CBOR encoding
DecodeTypedParamsFromJSON(ctx context.Context, to address.Address, method abi.MethodNum, paramstr string) ([]byte, error)
// Close ends the session of services and disconnects from RPC, using Services after Close is called
// most likely will result in an error
// Should not be called concurrently
Close() error
}
type ServicesImpl struct {
api api.FullNode
closer jsonrpc.ClientCloser
}
func (s *ServicesImpl) Close() error {
if s.closer == nil {
return xerrors.Errorf("Services already closed")
}
s.closer()
s.closer = nil
return nil
}
func (s *ServicesImpl) DecodeTypedParamsFromJSON(ctx context.Context, to address.Address, method abi.MethodNum, paramstr string) ([]byte, error) {
act, err := s.api.StateGetActor(ctx, to, types.EmptyTSK)
if err != nil {
return nil, err
}
methodMeta, found := stmgr.MethodsMap[act.Code][method]
if !found {
return nil, fmt.Errorf("method %d not found on actor %s", method, act.Code)
}
p := reflect.New(methodMeta.Params.Elem()).Interface().(cbg.CBORMarshaler)
if err := json.Unmarshal([]byte(paramstr), p); err != nil {
return nil, fmt.Errorf("unmarshaling input into params type: %w", err)
}
buf := new(bytes.Buffer)
if err := p.MarshalCBOR(buf); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
type SendParams struct {
To address.Address
From address.Address
Val abi.TokenAmount
GasPremium *abi.TokenAmount
GasFeeCap *abi.TokenAmount
GasLimit *int64
Nonce *uint64
Method abi.MethodNum
Params []byte
Force bool
}
// This is specialised Send for Send command
// There might be room for generic Send that other commands can use to send their messages
// We will see
var ErrSendBalanceTooLow = errors.New("balance too low")
func (s *ServicesImpl) Send(ctx context.Context, params SendParams) (cid.Cid, error) {
if params.From == address.Undef {
defaddr, err := s.api.WalletDefaultAddress(ctx)
if err != nil {
return cid.Undef, err
}
params.From = defaddr
}
msg := &types.Message{
From: params.From,
To: params.To,
Value: params.Val,
Method: params.Method,
Params: params.Params,
}
if params.GasPremium != nil {
msg.GasPremium = *params.GasPremium
} else {
msg.GasPremium = types.NewInt(0)
}
if params.GasFeeCap != nil {
msg.GasFeeCap = *params.GasFeeCap
} else {
msg.GasFeeCap = types.NewInt(0)
}
if params.GasLimit != nil {
msg.GasLimit = *params.GasLimit
} else {
msg.GasLimit = 0
}
if !params.Force {
// Funds insufficient check
fromBalance, err := s.api.WalletBalance(ctx, msg.From)
if err != nil {
return cid.Undef, err
}
totalCost := types.BigAdd(types.BigMul(msg.GasFeeCap, types.NewInt(uint64(msg.GasLimit))), msg.Value)
if fromBalance.LessThan(totalCost) {
return cid.Undef, xerrors.Errorf("From balance %s less than total cost %s: %w", types.FIL(fromBalance), types.FIL(totalCost), ErrSendBalanceTooLow)
}
}
if params.Nonce != nil {
msg.Nonce = *params.Nonce
sm, err := s.api.WalletSignMessage(ctx, params.From, msg)
if err != nil {
return cid.Undef, err
}
_, err = s.api.MpoolPush(ctx, sm)
if err != nil {
return cid.Undef, err
}
return sm.Cid(), nil
}
sm, err := s.api.MpoolPushMessage(ctx, msg, nil)
if err != nil {
return cid.Undef, err
}
return sm.Cid(), nil
}

266
cli/services_send_test.go Normal file
View File

@ -0,0 +1,266 @@
package cli
import (
"context"
"fmt"
"testing"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/mocks"
types "github.com/filecoin-project/lotus/chain/types"
gomock "github.com/golang/mock/gomock"
cid "github.com/ipfs/go-cid"
"github.com/stretchr/testify/assert"
)
type markerKeyType struct{}
var markerKey = markerKeyType{}
type contextMatcher struct {
marker *int
}
// Matches returns whether x is a match.
func (cm contextMatcher) Matches(x interface{}) bool {
ctx, ok := x.(context.Context)
if !ok {
return false
}
maybeMarker, ok := ctx.Value(markerKey).(*int)
if !ok {
return false
}
return cm.marker == maybeMarker
}
func (cm contextMatcher) String() string {
return fmt.Sprintf("Context with Value(%v/%T, %p)", markerKey, markerKey, cm.marker)
}
func ContextWithMarker(ctx context.Context) (context.Context, gomock.Matcher) {
marker := new(int)
outCtx := context.WithValue(ctx, markerKey, marker)
return outCtx, contextMatcher{marker: marker}
}
func setupMockSrvcs(t *testing.T) (*ServicesImpl, *mocks.MockFullNode) {
mockCtrl := gomock.NewController(t)
mockApi := mocks.NewMockFullNode(mockCtrl)
srvcs := &ServicesImpl{
api: mockApi,
closer: mockCtrl.Finish,
}
return srvcs, mockApi
}
func fakeSign(msg *types.Message) *types.SignedMessage {
return &types.SignedMessage{
Message: *msg,
Signature: crypto.Signature{Type: crypto.SigTypeSecp256k1, Data: make([]byte, 32)},
}
}
func makeMessageSigner() (*cid.Cid, interface{}) {
smCid := cid.Undef
return &smCid,
func(_ context.Context, msg *types.Message, _ *api.MessageSendSpec) (*types.SignedMessage, error) {
sm := fakeSign(msg)
smCid = sm.Cid()
return sm, nil
}
}
type MessageMatcher SendParams
var _ gomock.Matcher = MessageMatcher{}
// Matches returns whether x is a match.
func (mm MessageMatcher) Matches(x interface{}) bool {
m, ok := x.(*types.Message)
if !ok {
return false
}
if mm.From != address.Undef && mm.From != m.From {
return false
}
if mm.To != address.Undef && mm.To != m.To {
return false
}
if types.BigCmp(mm.Val, m.Value) != 0 {
return false
}
if mm.Nonce != nil && *mm.Nonce != m.Nonce {
return false
}
if mm.GasPremium != nil && big.Cmp(*mm.GasPremium, m.GasPremium) != 0 {
return false
}
if mm.GasPremium == nil && m.GasPremium.Sign() != 0 {
return false
}
if mm.GasFeeCap != nil && big.Cmp(*mm.GasFeeCap, m.GasFeeCap) != 0 {
return false
}
if mm.GasFeeCap == nil && m.GasFeeCap.Sign() != 0 {
return false
}
if mm.GasLimit != nil && *mm.GasLimit != m.GasLimit {
return false
}
if mm.GasLimit == nil && m.GasLimit != 0 {
return false
}
// handle rest of options
return true
}
// String describes what the matcher matches.
func (mm MessageMatcher) String() string {
return fmt.Sprintf("%#v", SendParams(mm))
}
func TestSendService(t *testing.T) {
addrGen := address.NewForTestGetter()
a1 := addrGen()
a2 := addrGen()
const balance = 10000
params := SendParams{
From: a1,
To: a2,
Val: types.NewInt(balance - 100),
}
ctx, ctxM := ContextWithMarker(context.Background())
t.Run("happy", func(t *testing.T) {
params := params
srvcs, mockApi := setupMockSrvcs(t)
defer srvcs.Close() //nolint:errcheck
msgCid, sign := makeMessageSigner()
gomock.InOrder(
mockApi.EXPECT().WalletBalance(ctxM, params.From).Return(types.NewInt(balance), nil),
mockApi.EXPECT().MpoolPushMessage(ctxM, MessageMatcher(params), nil).DoAndReturn(sign),
)
c, err := srvcs.Send(ctx, params)
assert.NoError(t, err)
assert.Equal(t, *msgCid, c)
})
t.Run("balance-too-low", func(t *testing.T) {
params := params
srvcs, mockApi := setupMockSrvcs(t)
defer srvcs.Close() //nolint:errcheck
gomock.InOrder(
mockApi.EXPECT().WalletBalance(ctxM, a1).Return(types.NewInt(balance-200), nil),
// no MpoolPushMessage
)
c, err := srvcs.Send(ctx, params)
assert.Equal(t, c, cid.Undef)
assert.ErrorIs(t, err, ErrSendBalanceTooLow)
})
t.Run("force", func(t *testing.T) {
params := params
params.Force = true
srvcs, mockApi := setupMockSrvcs(t)
defer srvcs.Close() //nolint:errcheck
msgCid, sign := makeMessageSigner()
gomock.InOrder(
mockApi.EXPECT().WalletBalance(ctxM, a1).Return(types.NewInt(balance-200), nil).AnyTimes(),
mockApi.EXPECT().MpoolPushMessage(ctxM, MessageMatcher(params), nil).DoAndReturn(sign),
)
c, err := srvcs.Send(ctx, params)
assert.NoError(t, err)
assert.Equal(t, *msgCid, c)
})
t.Run("default-from", func(t *testing.T) {
params := params
params.From = address.Undef
mm := MessageMatcher(params)
mm.From = a1
srvcs, mockApi := setupMockSrvcs(t)
defer srvcs.Close() //nolint:errcheck
msgCid, sign := makeMessageSigner()
gomock.InOrder(
mockApi.EXPECT().WalletDefaultAddress(ctxM).Return(a1, nil),
mockApi.EXPECT().WalletBalance(ctxM, a1).Return(types.NewInt(balance), nil),
mockApi.EXPECT().MpoolPushMessage(ctxM, mm, nil).DoAndReturn(sign),
)
c, err := srvcs.Send(ctx, params)
assert.NoError(t, err)
assert.Equal(t, *msgCid, c)
})
t.Run("set-nonce", func(t *testing.T) {
params := params
n := uint64(5)
params.Nonce = &n
mm := MessageMatcher(params)
srvcs, mockApi := setupMockSrvcs(t)
defer srvcs.Close() //nolint:errcheck
_, _ = mm, mockApi
var sm *types.SignedMessage
gomock.InOrder(
mockApi.EXPECT().WalletBalance(ctxM, a1).Return(types.NewInt(balance), nil),
mockApi.EXPECT().WalletSignMessage(ctxM, a1, mm).DoAndReturn(
func(_ context.Context, _ address.Address, msg *types.Message) (*types.SignedMessage, error) {
sm = fakeSign(msg)
// now we expect MpoolPush with that SignedMessage
mockApi.EXPECT().MpoolPush(ctxM, sm).Return(sm.Cid(), nil)
return sm, nil
}),
)
c, err := srvcs.Send(ctx, params)
assert.NoError(t, err)
assert.Equal(t, sm.Cid(), c)
})
t.Run("gas-params", func(t *testing.T) {
params := params
limit := int64(1)
params.GasLimit = &limit
gfc := big.NewInt(100)
params.GasFeeCap = &gfc
gp := big.NewInt(10)
params.GasPremium = &gp
srvcs, mockApi := setupMockSrvcs(t)
defer srvcs.Close() //nolint:errcheck
msgCid, sign := makeMessageSigner()
gomock.InOrder(
mockApi.EXPECT().WalletBalance(ctxM, params.From).Return(types.NewInt(balance), nil),
mockApi.EXPECT().MpoolPushMessage(ctxM, MessageMatcher(params), nil).DoAndReturn(sign),
)
c, err := srvcs.Send(ctx, params)
assert.NoError(t, err)
assert.Equal(t, *msgCid, c)
})
}

81
cli/servicesmock_test.go Normal file
View File

@ -0,0 +1,81 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: github.com/filecoin-project/lotus/cli (interfaces: ServicesAPI)
// Package cli is a generated GoMock package.
package cli
import (
context "context"
go_address "github.com/filecoin-project/go-address"
abi "github.com/filecoin-project/go-state-types/abi"
gomock "github.com/golang/mock/gomock"
go_cid "github.com/ipfs/go-cid"
reflect "reflect"
)
// MockServicesAPI is a mock of ServicesAPI interface
type MockServicesAPI struct {
ctrl *gomock.Controller
recorder *MockServicesAPIMockRecorder
}
// MockServicesAPIMockRecorder is the mock recorder for MockServicesAPI
type MockServicesAPIMockRecorder struct {
mock *MockServicesAPI
}
// NewMockServicesAPI creates a new mock instance
func NewMockServicesAPI(ctrl *gomock.Controller) *MockServicesAPI {
mock := &MockServicesAPI{ctrl: ctrl}
mock.recorder = &MockServicesAPIMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockServicesAPI) EXPECT() *MockServicesAPIMockRecorder {
return m.recorder
}
// Close mocks base method
func (m *MockServicesAPI) Close() error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Close")
ret0, _ := ret[0].(error)
return ret0
}
// Close indicates an expected call of Close
func (mr *MockServicesAPIMockRecorder) Close() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockServicesAPI)(nil).Close))
}
// DecodeTypedParamsFromJSON mocks base method
func (m *MockServicesAPI) DecodeTypedParamsFromJSON(arg0 context.Context, arg1 go_address.Address, arg2 abi.MethodNum, arg3 string) ([]byte, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "DecodeTypedParamsFromJSON", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].([]byte)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// DecodeTypedParamsFromJSON indicates an expected call of DecodeTypedParamsFromJSON
func (mr *MockServicesAPIMockRecorder) DecodeTypedParamsFromJSON(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DecodeTypedParamsFromJSON", reflect.TypeOf((*MockServicesAPI)(nil).DecodeTypedParamsFromJSON), arg0, arg1, arg2, arg3)
}
// Send mocks base method
func (m *MockServicesAPI) Send(arg0 context.Context, arg1 SendParams) (go_cid.Cid, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Send", arg0, arg1)
ret0, _ := ret[0].(go_cid.Cid)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// Send indicates an expected call of Send
func (mr *MockServicesAPIMockRecorder) Send(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockServicesAPI)(nil).Send), arg0, arg1)
}

4
cli/state.go
View File

@ -34,7 +34,7 @@ import (
"github.com/filecoin-project/lotus/api"
lapi "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/apibstore"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/state"
"github.com/filecoin-project/lotus/chain/stmgr"
@ -1010,7 +1010,7 @@ var stateComputeStateCmd = &cli.Command{
}
if cctx.Bool("html") {
st, err := state.LoadStateTree(cbor.NewCborStore(apibstore.NewAPIBlockstore(api)), stout.Root)
st, err := state.LoadStateTree(cbor.NewCborStore(blockstore.NewAPIBlockstore(api)), stout.Root)
if err != nil {
return xerrors.Errorf("loading state tree: %w", err)
}

274
cli/util/api.go Normal file
View File

@ -0,0 +1,274 @@
package cliutil
import (
"context"
"fmt"
"net/http"
"net/url"
"os"
"os/signal"
"strings"
"syscall"
"github.com/mitchellh/go-homedir"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-jsonrpc"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/client"
"github.com/filecoin-project/lotus/node/repo"
)
const (
metadataTraceContext = "traceContext"
)
// The flag passed on the command line with the listen address of the API
// server (only used by the tests)
func flagForAPI(t repo.RepoType) string {
switch t {
case repo.FullNode:
return "api-url"
case repo.StorageMiner:
return "miner-api-url"
case repo.Worker:
return "worker-api-url"
default:
panic(fmt.Sprintf("Unknown repo type: %v", t))
}
}
func flagForRepo(t repo.RepoType) string {
switch t {
case repo.FullNode:
return "repo"
case repo.StorageMiner:
return "miner-repo"
case repo.Worker:
return "worker-repo"
default:
panic(fmt.Sprintf("Unknown repo type: %v", t))
}
}
func EnvForRepo(t repo.RepoType) string {
switch t {
case repo.FullNode:
return "FULLNODE_API_INFO"
case repo.StorageMiner:
return "MINER_API_INFO"
case repo.Worker:
return "WORKER_API_INFO"
default:
panic(fmt.Sprintf("Unknown repo type: %v", t))
}
}
// TODO remove after deprecation period
func envForRepoDeprecation(t repo.RepoType) string {
switch t {
case repo.FullNode:
return "FULLNODE_API_INFO"
case repo.StorageMiner:
return "STORAGE_API_INFO"
case repo.Worker:
return "WORKER_API_INFO"
default:
panic(fmt.Sprintf("Unknown repo type: %v", t))
}
}
func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) {
// Check if there was a flag passed with the listen address of the API
// server (only used by the tests)
apiFlag := flagForAPI(t)
if ctx.IsSet(apiFlag) {
strma := ctx.String(apiFlag)
strma = strings.TrimSpace(strma)
return APIInfo{Addr: strma}, nil
}
envKey := EnvForRepo(t)
env, ok := os.LookupEnv(envKey)
if !ok {
// TODO remove after deprecation period
envKey = envForRepoDeprecation(t)
env, ok = os.LookupEnv(envKey)
if ok {
log.Warnf("Use deprecation env(%s) value, please use env(%s) instead.", envKey, EnvForRepo(t))
}
}
if ok {
return ParseApiInfo(env), nil
}
repoFlag := flagForRepo(t)
p, err := homedir.Expand(ctx.String(repoFlag))
if err != nil {
return APIInfo{}, xerrors.Errorf("could not expand home dir (%s): %w", repoFlag, err)
}
r, err := repo.NewFS(p)
if err != nil {
return APIInfo{}, xerrors.Errorf("could not open repo at path: %s; %w", p, err)
}
ma, err := r.APIEndpoint()
if err != nil {
return APIInfo{}, xerrors.Errorf("could not get api endpoint: %w", err)
}
token, err := r.APIToken()
if err != nil {
log.Warnf("Couldn't load CLI token, capabilities may be limited: %v", err)
}
return APIInfo{
Addr: ma.String(),
Token: token,
}, nil
}
func GetRawAPI(ctx *cli.Context, t repo.RepoType) (string, http.Header, error) {
ainfo, err := GetAPIInfo(ctx, t)
if err != nil {
return "", nil, xerrors.Errorf("could not get API info: %w", err)
}
addr, err := ainfo.DialArgs()
if err != nil {
return "", nil, xerrors.Errorf("could not get DialArgs: %w", err)
}
return addr, ainfo.AuthHeader(), nil
}
func GetAPI(ctx *cli.Context) (api.Common, jsonrpc.ClientCloser, error) {
ti, ok := ctx.App.Metadata["repoType"]
if !ok {
log.Errorf("unknown repo type, are you sure you want to use GetAPI?")
ti = repo.FullNode
}
t, ok := ti.(repo.RepoType)
if !ok {
log.Errorf("repoType type does not match the type of repo.RepoType")
}
if tn, ok := ctx.App.Metadata["testnode-storage"]; ok {
return tn.(api.StorageMiner), func() {}, nil
}
if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
return tn.(api.FullNode), func() {}, nil
}
addr, headers, err := GetRawAPI(ctx, t)
if err != nil {
return nil, nil, err
}
return client.NewCommonRPC(ctx.Context, addr, headers)
}
func GetFullNodeAPI(ctx *cli.Context) (api.FullNode, jsonrpc.ClientCloser, error) {
if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
return tn.(api.FullNode), func() {}, nil
}
addr, headers, err := GetRawAPI(ctx, repo.FullNode)
if err != nil {
return nil, nil, err
}
return client.NewFullNodeRPC(ctx.Context, addr, headers)
}
type GetStorageMinerOptions struct {
PreferHttp bool
}
type GetStorageMinerOption func(*GetStorageMinerOptions)
func StorageMinerUseHttp(opts *GetStorageMinerOptions) {
opts.PreferHttp = true
}
func GetStorageMinerAPI(ctx *cli.Context, opts ...GetStorageMinerOption) (api.StorageMiner, jsonrpc.ClientCloser, error) {
var options GetStorageMinerOptions
for _, opt := range opts {
opt(&options)
}
if tn, ok := ctx.App.Metadata["testnode-storage"]; ok {
return tn.(api.StorageMiner), func() {}, nil
}
addr, headers, err := GetRawAPI(ctx, repo.StorageMiner)
if err != nil {
return nil, nil, err
}
if options.PreferHttp {
u, err := url.Parse(addr)
if err != nil {
return nil, nil, xerrors.Errorf("parsing miner api URL: %w", err)
}
switch u.Scheme {
case "ws":
u.Scheme = "http"
case "wss":
u.Scheme = "https"
}
addr = u.String()
}
return client.NewStorageMinerRPC(ctx.Context, addr, headers)
}
func GetWorkerAPI(ctx *cli.Context) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
addr, headers, err := GetRawAPI(ctx, repo.Worker)
if err != nil {
return nil, nil, err
}
return client.NewWorkerRPC(ctx.Context, addr, headers)
}
func GetGatewayAPI(ctx *cli.Context) (api.GatewayAPI, jsonrpc.ClientCloser, error) {
addr, headers, err := GetRawAPI(ctx, repo.FullNode)
if err != nil {
return nil, nil, err
}
return client.NewGatewayRPC(ctx.Context, addr, headers)
}
func DaemonContext(cctx *cli.Context) context.Context {
if mtCtx, ok := cctx.App.Metadata[metadataTraceContext]; ok {
return mtCtx.(context.Context)
}
return context.Background()
}
// ReqContext returns context for cli execution. Calling it for the first time
// installs SIGTERM handler that will close returned context.
// Not safe for concurrent execution.
func ReqContext(cctx *cli.Context) context.Context {
tCtx := DaemonContext(cctx)
ctx, done := context.WithCancel(tCtx)
sigChan := make(chan os.Signal, 2)
go func() {
<-sigChan
done()
}()
signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP)
return ctx
}

37
cmd/lotus-bench/import.go
View File

@ -20,18 +20,17 @@ import (
"github.com/cockroachdb/pebble"
"github.com/cockroachdb/pebble/bloom"
"github.com/ipfs/go-cid"
metricsi "github.com/ipfs/go-metrics-interface"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/blockstore"
badgerbs "github.com/filecoin-project/lotus/blockstore/badger"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/lib/blockstore"
badgerbs "github.com/filecoin-project/lotus/lib/blockstore/badger"
_ "github.com/filecoin-project/lotus/lib/sigs/bls"
_ "github.com/filecoin-project/lotus/lib/sigs/secp"
"github.com/filecoin-project/lotus/node/repo"
@ -204,7 +203,7 @@ var importBenchCmd = &cli.Command{
case cctx.Bool("use-native-badger"):
log.Info("using native badger")
var opts badgerbs.Options
if opts, err = repo.BadgerBlockstoreOptions(repo.BlockstoreChain, tdir, false); err != nil {
if opts, err = repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, tdir, false); err != nil {
return err
}
opts.SyncWrites = false
@ -229,21 +228,13 @@ var importBenchCmd = &cli.Command{
if ds != nil {
ds = measure.New("dsbench", ds)
defer ds.Close() //nolint:errcheck
bs = blockstore.NewBlockstore(ds)
bs = blockstore.FromDatastore(ds)
}
if c, ok := bs.(io.Closer); ok {
defer c.Close() //nolint:errcheck
}
ctx := metricsi.CtxScope(context.Background(), "lotus")
cacheOpts := blockstore.DefaultCacheOpts()
cacheOpts.HasBloomFilterSize = 0
bs, err = blockstore.CachedBlockstore(ctx, bs, cacheOpts)
if err != nil {
return err
}
var verifier ffiwrapper.Verifier = ffiwrapper.ProofVerifier
if cctx.IsSet("syscall-cache") {
scds, err := badger.NewDatastore(cctx.String("syscall-cache"), &badger.DefaultOptions)
@ -267,6 +258,15 @@ var importBenchCmd = &cli.Command{
stm := stmgr.NewStateManager(cs)
var carFile *os.File
// open the CAR file if one is provided.
if path := cctx.String("car"); path != "" {
var err error
if carFile, err = os.Open(path); err != nil {
return xerrors.Errorf("failed to open provided CAR file: %w", err)
}
}
startTime := time.Now()
// register a gauge that reports how long since the measurable
@ -308,18 +308,7 @@ var importBenchCmd = &cli.Command{
writeProfile("allocs")
}()
var carFile *os.File
// open the CAR file if one is provided.
if path := cctx.String("car"); path != "" {
var err error
if carFile, err = os.Open(path); err != nil {
return xerrors.Errorf("failed to open provided CAR file: %w", err)
}
}
var head *types.TipSet
// --- IMPORT ---
if !cctx.Bool("no-import") {
if cctx.Bool("global-profile") {

6
cmd/lotus-chainwatch/processor/miner.go
View File

@ -15,7 +15,7 @@ import (
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/apibstore"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/power"
"github.com/filecoin-project/lotus/chain/events/state"
@ -202,7 +202,7 @@ func (p *Processor) processMiners(ctx context.Context, minerTips map[types.TipSe
log.Debugw("Processed Miners", "duration", time.Since(start).String())
}()
stor := store.ActorStore(ctx, apibstore.NewAPIBlockstore(p.node))
stor := store.ActorStore(ctx, blockstore.NewAPIBlockstore(p.node))
var out []minerActorInfo
// TODO add parallel calls if this becomes slow
@ -649,7 +649,7 @@ func (p *Processor) getMinerStateAt(ctx context.Context, maddr address.Address,
if err != nil {
return nil, err
}
return miner.Load(store.ActorStore(ctx, apibstore.NewAPIBlockstore(p.node)), prevActor)
return miner.Load(store.ActorStore(ctx, blockstore.NewAPIBlockstore(p.node)), prevActor)
}
func (p *Processor) getMinerPreCommitChanges(ctx context.Context, m minerActorInfo) (*miner.PreCommitChanges, error) {

4
cmd/lotus-gateway/api.go
View File

@ -34,7 +34,7 @@ var (
// gatewayDepsAPI defines the API methods that the GatewayAPI depends on
// (to make it easy to mock for tests)
type gatewayDepsAPI interface {
Version(context.Context) (api.Version, error)
Version(context.Context) (api.APIVersion, error)
ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error)
ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error)
ChainGetNode(ctx context.Context, p string) (*api.IpldObject, error)
@ -130,7 +130,7 @@ func (a *GatewayAPI) checkTimestamp(at time.Time) error {
return nil
}
func (a *GatewayAPI) Version(ctx context.Context) (api.Version, error) {
func (a *GatewayAPI) Version(ctx context.Context) (api.APIVersion, error) {
return a.api.Version(ctx)
}

2
cmd/lotus-gateway/main.go
View File

@ -77,7 +77,7 @@ var runCmd = &cli.Command{
// Register all metric views
if err := view.Register(
metrics.DefaultViews...,
metrics.ChainNodeViews...,
); err != nil {
log.Fatalf("Cannot register the view: %v", err)
}

9
cmd/lotus-seal-worker/main.go
View File

@ -31,6 +31,7 @@ import (
"github.com/filecoin-project/lotus/api/apistruct"
"github.com/filecoin-project/lotus/build"
lcli "github.com/filecoin-project/lotus/cli"
cliutil "github.com/filecoin-project/lotus/cli/util"
sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
@ -49,7 +50,7 @@ const FlagWorkerRepo = "worker-repo"
const FlagWorkerRepoDeprecation = "workerrepo"
func main() {
build.RunningNodeType = build.NodeWorker
api.RunningNodeType = api.NodeWorker
lotuslog.SetupLogLevels()
@ -183,7 +184,7 @@ var runCmd = &cli.Command{
var closer func()
var err error
for {
nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx, lcli.StorageMinerUseHttp)
nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx, cliutil.StorageMinerUseHttp)
if err == nil {
_, err = nodeApi.Version(ctx)
if err == nil {
@ -210,8 +211,8 @@ var runCmd = &cli.Command{
if err != nil {
return err
}
if v.APIVersion != build.MinerAPIVersion {
return xerrors.Errorf("lotus-miner API version doesn't match: expected: %s", api.Version{APIVersion: build.MinerAPIVersion})
if v.APIVersion != api.MinerAPIVersion {
return xerrors.Errorf("lotus-miner API version doesn't match: expected: %s", api.APIVersion{APIVersion: api.MinerAPIVersion})
}
log.Infof("Remote version %s", v)

6
cmd/lotus-seal-worker/rpc.go
View File

@ -8,7 +8,7 @@ import (
"github.com/mitchellh/go-homedir"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/api"
sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
@ -23,8 +23,8 @@ type worker struct {
disabled int64
}
func (w *worker) Version(context.Context) (build.Version, error) {
return build.WorkerAPIVersion, nil
func (w *worker) Version(context.Context) (api.Version, error) {
return api.WorkerAPIVersion, nil
}
func (w *worker) StorageAddLocal(ctx context.Context, path string) error {

14
cmd/lotus-seal-worker/storage.go
View File

@ -6,6 +6,7 @@ import (
"os"
"path/filepath"
"github.com/docker/go-units"
"github.com/google/uuid"
"github.com/mitchellh/go-homedir"
"github.com/urfave/cli/v2"
@ -46,6 +47,10 @@ var storageAttachCmd = &cli.Command{
Name: "store",
Usage: "(for init) use path for long-term storage",
},
&cli.StringFlag{
Name: "max-storage",
Usage: "(for init) limit storage space for sectors (expensive for very large paths!)",
},
},
Action: func(cctx *cli.Context) error {
nodeApi, closer, err := lcli.GetWorkerAPI(cctx)
@ -79,11 +84,20 @@ var storageAttachCmd = &cli.Command{
return err
}
var maxStor int64
if cctx.IsSet("max-storage") {
maxStor, err = units.RAMInBytes(cctx.String("max-storage"))
if err != nil {
return xerrors.Errorf("parsing max-storage: %w", err)
}
}
cfg := &stores.LocalStorageMeta{
ID: stores.ID(uuid.New().String()),
Weight: cctx.Uint64("weight"),
CanSeal: cctx.Bool("seal"),
CanStore: cctx.Bool("store"),
MaxStorage: uint64(maxStor),
}
if !(cfg.CanStore || cfg.CanSeal) {

200
cmd/lotus-seed/genesis.go
View File

@ -9,10 +9,10 @@ import (
"strconv"
"strings"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/journal"
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/lotus/node/modules/testing"
"github.com/google/uuid"
"github.com/mitchellh/go-homedir"
@ -37,6 +37,8 @@ var genesisCmd = &cli.Command{
genesisNewCmd,
genesisAddMinerCmd,
genesisAddMsigsCmd,
genesisSetVRKCmd,
genesisSetRemainderCmd,
genesisCarCmd,
},
}
@ -309,6 +311,200 @@ func parseMultisigCsv(csvf string) ([]GenAccountEntry, error) {
return entries, nil
}
var genesisSetVRKCmd = &cli.Command{
Name: "set-vrk",
Usage: "Set the verified registry's root key",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "multisig",
Usage: "CSV file to parse the multisig that will be set as the root key",
},
&cli.StringFlag{
Name: "account",
Usage: "pubkey address that will be set as the root key (must NOT be declared anywhere else, since it must be given ID 80)",
},
},
Action: func(cctx *cli.Context) error {
if cctx.Args().Len() != 1 {
return fmt.Errorf("must specify template file")
}
genf, err := homedir.Expand(cctx.Args().First())
if err != nil {
return err
}
csvf, err := homedir.Expand(cctx.Args().Get(1))
if err != nil {
return err
}
var template genesis.Template
b, err := ioutil.ReadFile(genf)
if err != nil {
return xerrors.Errorf("read genesis template: %w", err)
}
if err := json.Unmarshal(b, &template); err != nil {
return xerrors.Errorf("unmarshal genesis template: %w", err)
}
if cctx.IsSet("account") {
addr, err := address.NewFromString(cctx.String("account"))
if err != nil {
return err
}
am := genesis.AccountMeta{Owner: addr}
template.VerifregRootKey = genesis.Actor{
Type: genesis.TAccount,
Balance: big.Zero(),
Meta: am.ActorMeta(),
}
} else if cctx.IsSet("multisig") {
entries, err := parseMultisigCsv(csvf)
if err != nil {
return xerrors.Errorf("parsing multisig csv file: %w", err)
}
if len(entries) == 0 {
return xerrors.Errorf("no msig entries in csv file: %w", err)
}
e := entries[0]
if len(e.Addresses) != e.N {
return fmt.Errorf("entry had mismatch between 'N' and number of addresses")
}
msig := &genesis.MultisigMeta{
Signers: e.Addresses,
Threshold: e.M,
VestingDuration: monthsToBlocks(e.VestingMonths),
VestingStart: 0,
}
act := genesis.Actor{
Type: genesis.TMultisig,
Balance: abi.TokenAmount(e.Amount),
Meta: msig.ActorMeta(),
}
template.VerifregRootKey = act
} else {
return xerrors.Errorf("must include either --account or --multisig flag")
}
b, err = json.MarshalIndent(&template, "", " ")
if err != nil {
return err
}
if err := ioutil.WriteFile(genf, b, 0644); err != nil {
return err
}
return nil
},
}
var genesisSetRemainderCmd = &cli.Command{
Name: "set-remainder",
Usage: "Set the remainder actor",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "multisig",
Usage: "CSV file to parse the multisig that will be set as the remainder actor",
},
&cli.StringFlag{
Name: "account",
Usage: "pubkey address that will be set as the remainder key (must NOT be declared anywhere else, since it must be given ID 90)",
},
},
Action: func(cctx *cli.Context) error {
if cctx.Args().Len() != 1 {
return fmt.Errorf("must specify template file")
}
genf, err := homedir.Expand(cctx.Args().First())
if err != nil {
return err
}
csvf, err := homedir.Expand(cctx.Args().Get(1))
if err != nil {
return err
}
var template genesis.Template
b, err := ioutil.ReadFile(genf)
if err != nil {
return xerrors.Errorf("read genesis template: %w", err)
}
if err := json.Unmarshal(b, &template); err != nil {
return xerrors.Errorf("unmarshal genesis template: %w", err)
}
if cctx.IsSet("account") {
addr, err := address.NewFromString(cctx.String("account"))
if err != nil {
return err
}
am := genesis.AccountMeta{Owner: addr}
template.RemainderAccount = genesis.Actor{
Type: genesis.TAccount,
Balance: big.Zero(),
Meta: am.ActorMeta(),
}
} else if cctx.IsSet("multisig") {
entries, err := parseMultisigCsv(csvf)
if err != nil {
return xerrors.Errorf("parsing multisig csv file: %w", err)
}
if len(entries) == 0 {
return xerrors.Errorf("no msig entries in csv file: %w", err)
}
e := entries[0]
if len(e.Addresses) != e.N {
return fmt.Errorf("entry had mismatch between 'N' and number of addresses")
}
msig := &genesis.MultisigMeta{
Signers: e.Addresses,
Threshold: e.M,
VestingDuration: monthsToBlocks(e.VestingMonths),
VestingStart: 0,
}
act := genesis.Actor{
Type: genesis.TMultisig,
Balance: abi.TokenAmount(e.Amount),
Meta: msig.ActorMeta(),
}
template.RemainderAccount = act
} else {
return xerrors.Errorf("must include either --account or --multisig flag")
}
b, err = json.MarshalIndent(&template, "", " ")
if err != nil {
return err
}
if err := ioutil.WriteFile(genf, b, 0644); err != nil {
return err
}
return nil
},
}
var genesisCarCmd = &cli.Command{
Name: "car",
Description: "write genesis car file",
@ -327,7 +523,7 @@ var genesisCarCmd = &cli.Command{
}
ofile := c.String("out")
jrnl := journal.NilJournal()
bstor := blockstore.NewTemporarySync()
bstor := blockstore.NewMemorySync()
sbldr := vm.Syscalls(ffiwrapper.ProofVerifier)
_, err := testing.MakeGenesis(ofile, c.Args().First())(bstor, sbldr, jrnl)()
return err

4
cmd/lotus-shed/balances.go
View File

@ -175,7 +175,7 @@ var chainBalanceStateCmd = &cli.Command{
defer lkrepo.Close() //nolint:errcheck
bs, err := lkrepo.Blockstore(ctx, repo.BlockstoreChain)
bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore)
if err != nil {
return fmt.Errorf("failed to open blockstore: %w", err)
}
@ -396,7 +396,7 @@ var chainPledgeCmd = &cli.Command{
defer lkrepo.Close() //nolint:errcheck
bs, err := lkrepo.Blockstore(ctx, repo.BlockstoreChain)
bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore)
if err != nil {
return xerrors.Errorf("failed to open blockstore: %w", err)
}

2
cmd/lotus-shed/consensus.go
View File

@ -36,7 +36,7 @@ type consensusItem struct {
targetTipset *types.TipSet
headTipset *types.TipSet
peerID peer.ID
version api.Version
version api.APIVersion
api api.FullNode
}

4
cmd/lotus-shed/datastore.go
View File

@ -319,7 +319,7 @@ var datastoreRewriteCmd = &cli.Command{
)
// open the destination (to) store.
opts, err := repo.BadgerBlockstoreOptions(repo.BlockstoreChain, toPath, false)
opts, err := repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, toPath, false)
if err != nil {
return xerrors.Errorf("failed to get badger options: %w", err)
}
@ -329,7 +329,7 @@ var datastoreRewriteCmd = &cli.Command{
}
// open the source (from) store.
opts, err = repo.BadgerBlockstoreOptions(repo.BlockstoreChain, fromPath, true)
opts, err = repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, fromPath, true)
if err != nil {
return xerrors.Errorf("failed to get badger options: %w", err)
}

2
cmd/lotus-shed/export.go
View File

@ -72,7 +72,7 @@ var exportChainCmd = &cli.Command{
defer fi.Close() //nolint:errcheck
bs, err := lr.Blockstore(ctx, repo.BlockstoreChain)
bs, err := lr.Blockstore(ctx, repo.UniversalBlockstore)
if err != nil {
return fmt.Errorf("failed to open blockstore: %w", err)
}

4
cmd/lotus-shed/genesis-verify.go
View File

@ -17,6 +17,7 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/adt"
"github.com/filecoin-project/lotus/chain/actors/builtin/account"
@ -26,7 +27,6 @@ import (
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/blockstore"
)
type addrInfo struct {
@ -50,7 +50,7 @@ var genesisVerifyCmd = &cli.Command{
if !cctx.Args().Present() {
return fmt.Errorf("must pass genesis car file")
}
bs := blockstore.NewBlockstore(datastore.NewMapDatastore())
bs := blockstore.FromDatastore(datastore.NewMapDatastore())
cs := store.NewChainStore(bs, bs, datastore.NewMapDatastore(), nil, nil)
defer cs.Close() //nolint:errcheck

Some files were not shown because too many files have changed in this diff Show More