Merge pull request #5679 from filecoin-project/next
1.5.1 staging branch
This commit is contained in:
commit
1673362795
@ -2,6 +2,7 @@ version: 2.1
|
||||
orbs:
|
||||
go: gotest/tools@0.0.13
|
||||
aws-cli: circleci/aws-cli@1.3.2
|
||||
packer: salaxander/packer@0.0.3
|
||||
|
||||
executors:
|
||||
golang:
|
||||
@ -277,6 +278,11 @@ jobs:
|
||||
- install-deps
|
||||
- prepare
|
||||
- run: make calibnet
|
||||
- run: mkdir linux-calibnet && mv lotus lotus-miner lotus-worker linux-calibnet
|
||||
- persist_to_workspace:
|
||||
root: "."
|
||||
paths:
|
||||
- linux-calibnet
|
||||
build-lotus-soup:
|
||||
description: |
|
||||
Compile `lotus-soup` Testground test plan
|
||||
@ -289,7 +295,7 @@ jobs:
|
||||
- run: cd extern/filecoin-ffi && make
|
||||
- run:
|
||||
name: "go get lotus@master"
|
||||
command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master
|
||||
command: cd testplans/lotus-soup && go mod edit -replace=github.com/filecoin-project/lotus=../..
|
||||
- run:
|
||||
name: "build lotus-soup testplan"
|
||||
command: pushd testplans/lotus-soup && go build -tags=testground .
|
||||
@ -583,6 +589,22 @@ jobs:
|
||||
docker push $<<parameters.account-url>>/<<parameters.repo>>:${tag}
|
||||
done
|
||||
|
||||
publish-packer:
|
||||
description: build and push AWS IAM and DigitalOcean droplet.
|
||||
executor:
|
||||
name: packer/default
|
||||
packer-version: 1.6.6
|
||||
steps:
|
||||
- checkout
|
||||
- attach_workspace:
|
||||
at: "."
|
||||
- packer/build:
|
||||
template: tools/packer/lotus.pkr.hcl
|
||||
args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG"
|
||||
- packer/build:
|
||||
template: tools/packer/lotus.pkr.hcl
|
||||
args: "-var ci_workspace_bins=./linux-calibnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG"
|
||||
|
||||
workflows:
|
||||
version: 2.1
|
||||
ci:
|
||||
@ -683,3 +705,15 @@ workflows:
|
||||
path: .
|
||||
repo: lotus-dev
|
||||
tag: '${CIRCLE_SHA1:0:8}'
|
||||
- publish-packer:
|
||||
requires:
|
||||
- build-all
|
||||
- build-ntwk-calibration
|
||||
filters:
|
||||
branches:
|
||||
ignore:
|
||||
- /.*/
|
||||
tags:
|
||||
only:
|
||||
- /^v\d+\.\d+\.\d+$/
|
||||
|
||||
|
@ -11,8 +11,6 @@ import (
|
||||
"github.com/libp2p/go-libp2p-core/network"
|
||||
"github.com/libp2p/go-libp2p-core/peer"
|
||||
protocol "github.com/libp2p/go-libp2p-core/protocol"
|
||||
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
)
|
||||
|
||||
type Common interface {
|
||||
@ -33,6 +31,7 @@ type Common interface {
|
||||
NetPubsubScores(context.Context) ([]PubsubScore, error)
|
||||
NetAutoNatStatus(context.Context) (NatInfo, error)
|
||||
NetAgentVersion(ctx context.Context, p peer.ID) (string, error)
|
||||
NetPeerInfo(context.Context, peer.ID) (*ExtendedPeerInfo, error)
|
||||
|
||||
// NetBandwidthStats returns statistics about the nodes total bandwidth
|
||||
// usage and current rate across all peers and protocols.
|
||||
@ -57,7 +56,7 @@ type Common interface {
|
||||
ID(context.Context) (peer.ID, error)
|
||||
|
||||
// Version provides information about API provider
|
||||
Version(context.Context) (Version, error)
|
||||
Version(context.Context) (APIVersion, error)
|
||||
|
||||
LogList(context.Context) ([]string, error)
|
||||
LogSetLevel(context.Context, string, string) error
|
||||
@ -71,15 +70,15 @@ type Common interface {
|
||||
Closing(context.Context) (<-chan struct{}, error)
|
||||
}
|
||||
|
||||
// Version provides various build-time information
|
||||
type Version struct {
|
||||
// APIVersion provides various build-time information
|
||||
type APIVersion struct {
|
||||
Version string
|
||||
|
||||
// APIVersion is a binary encoded semver version of the remote implementing
|
||||
// this api
|
||||
//
|
||||
// See APIVersion in build/version.go
|
||||
APIVersion build.Version
|
||||
APIVersion Version
|
||||
|
||||
// TODO: git commit / os / genesis cid?
|
||||
|
||||
@ -87,7 +86,7 @@ type Version struct {
|
||||
BlockDelay uint64
|
||||
}
|
||||
|
||||
func (v Version) String() string {
|
||||
func (v APIVersion) String() string {
|
||||
return fmt.Sprintf("%s+api%s", v.Version, v.APIVersion.String())
|
||||
}
|
||||
|
||||
|
@ -32,6 +32,14 @@ import (
|
||||
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||
)
|
||||
|
||||
//go:generate go run github.com/golang/mock/mockgen -destination=mocks/mock_full.go -package=mocks . FullNode
|
||||
|
||||
// ChainIO abstracts operations for accessing raw IPLD objects.
|
||||
type ChainIO interface {
|
||||
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
|
||||
ChainHasObj(context.Context, cid.Cid) (bool, error)
|
||||
}
|
||||
|
||||
// FullNode API is a low-level interface to the Filecoin network full node
|
||||
type FullNode interface {
|
||||
Common
|
||||
@ -862,6 +870,8 @@ const (
|
||||
|
||||
func (v SyncStateStage) String() string {
|
||||
switch v {
|
||||
case StageIdle:
|
||||
return "idle"
|
||||
case StageHeaders:
|
||||
return "header sync"
|
||||
case StagePersistHeaders:
|
||||
|
@ -36,7 +36,7 @@ type StorageMiner interface {
|
||||
MiningBase(context.Context) (*types.TipSet, error)
|
||||
|
||||
// Temp api for testing
|
||||
PledgeSector(context.Context) error
|
||||
PledgeSector(context.Context) (abi.SectorID, error)
|
||||
|
||||
// Get the status of a given sector by ID
|
||||
SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (SectorInfo, error)
|
||||
@ -238,6 +238,9 @@ type AddressConfig struct {
|
||||
PreCommitControl []address.Address
|
||||
CommitControl []address.Address
|
||||
TerminateControl []address.Address
|
||||
|
||||
DisableOwnerFallback bool
|
||||
DisableWorkerFallback bool
|
||||
}
|
||||
|
||||
// PendingDealInfo has info about pending deals and when they are due to be
|
||||
|
@ -9,12 +9,10 @@ import (
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
|
||||
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
)
|
||||
|
||||
type WorkerAPI interface {
|
||||
Version(context.Context) (build.Version, error)
|
||||
Version(context.Context) (Version, error)
|
||||
// TODO: Info() (name, ...) ?
|
||||
|
||||
TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) // TaskType -> Weight
|
||||
|
@ -1,68 +0,0 @@
|
||||
package apibstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/lotus/lib/blockstore"
|
||||
)
|
||||
|
||||
type ChainIO interface {
|
||||
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
|
||||
ChainHasObj(context.Context, cid.Cid) (bool, error)
|
||||
}
|
||||
|
||||
type apiBStore struct {
|
||||
api ChainIO
|
||||
}
|
||||
|
||||
func NewAPIBlockstore(cio ChainIO) blockstore.Blockstore {
|
||||
return &apiBStore{
|
||||
api: cio,
|
||||
}
|
||||
}
|
||||
|
||||
func (a *apiBStore) DeleteBlock(cid.Cid) error {
|
||||
return xerrors.New("not supported")
|
||||
}
|
||||
|
||||
func (a *apiBStore) Has(c cid.Cid) (bool, error) {
|
||||
return a.api.ChainHasObj(context.TODO(), c)
|
||||
}
|
||||
|
||||
func (a *apiBStore) Get(c cid.Cid) (blocks.Block, error) {
|
||||
bb, err := a.api.ChainReadObj(context.TODO(), c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return blocks.NewBlockWithCid(bb, c)
|
||||
}
|
||||
|
||||
func (a *apiBStore) GetSize(c cid.Cid) (int, error) {
|
||||
bb, err := a.api.ChainReadObj(context.TODO(), c)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(bb), nil
|
||||
}
|
||||
|
||||
func (a *apiBStore) Put(blocks.Block) error {
|
||||
return xerrors.New("not supported")
|
||||
}
|
||||
|
||||
func (a *apiBStore) PutMany([]blocks.Block) error {
|
||||
return xerrors.New("not supported")
|
||||
}
|
||||
|
||||
func (a *apiBStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
return nil, xerrors.New("not supported")
|
||||
}
|
||||
|
||||
func (a *apiBStore) HashOnRead(enabled bool) {
|
||||
return
|
||||
}
|
||||
|
||||
var _ blockstore.Blockstore = &apiBStore{}
|
@ -33,7 +33,6 @@ import (
|
||||
"github.com/filecoin-project/specs-storage/storage"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
@ -60,12 +59,13 @@ type CommonStruct struct {
|
||||
NetBandwidthStatsByPeer func(ctx context.Context) (map[string]metrics.Stats, error) `perm:"read"`
|
||||
NetBandwidthStatsByProtocol func(ctx context.Context) (map[protocol.ID]metrics.Stats, error) `perm:"read"`
|
||||
NetAgentVersion func(ctx context.Context, p peer.ID) (string, error) `perm:"read"`
|
||||
NetPeerInfo func(context.Context, peer.ID) (*api.ExtendedPeerInfo, error) `perm:"read"`
|
||||
NetBlockAdd func(ctx context.Context, acl api.NetBlockList) error `perm:"admin"`
|
||||
NetBlockRemove func(ctx context.Context, acl api.NetBlockList) error `perm:"admin"`
|
||||
NetBlockList func(ctx context.Context) (api.NetBlockList, error) `perm:"read"`
|
||||
|
||||
ID func(context.Context) (peer.ID, error) `perm:"read"`
|
||||
Version func(context.Context) (api.Version, error) `perm:"read"`
|
||||
ID func(context.Context) (peer.ID, error) `perm:"read"`
|
||||
Version func(context.Context) (api.APIVersion, error) `perm:"read"`
|
||||
|
||||
LogList func(context.Context) ([]string, error) `perm:"write"`
|
||||
LogSetLevel func(context.Context, string, string) error `perm:"write"`
|
||||
@ -304,7 +304,7 @@ type StorageMinerStruct struct {
|
||||
MarketPendingDeals func(ctx context.Context) (api.PendingDealInfo, error) `perm:"write"`
|
||||
MarketPublishPendingDeals func(ctx context.Context) error `perm:"admin"`
|
||||
|
||||
PledgeSector func(context.Context) error `perm:"write"`
|
||||
PledgeSector func(context.Context) (abi.SectorID, error) `perm:"write"`
|
||||
|
||||
SectorsStatus func(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (api.SectorInfo, error) `perm:"read"`
|
||||
SectorsList func(context.Context) ([]abi.SectorNumber, error) `perm:"read"`
|
||||
@ -389,7 +389,7 @@ type WorkerStruct struct {
|
||||
Internal struct {
|
||||
// TODO: lower perms
|
||||
|
||||
Version func(context.Context) (build.Version, error) `perm:"admin"`
|
||||
Version func(context.Context) (api.Version, error) `perm:"admin"`
|
||||
|
||||
TaskTypes func(context.Context) (map[sealtasks.TaskType]struct{}, error) `perm:"admin"`
|
||||
Paths func(context.Context) ([]stores.StoragePath, error) `perm:"admin"`
|
||||
@ -540,13 +540,17 @@ func (c *CommonStruct) NetAgentVersion(ctx context.Context, p peer.ID) (string,
|
||||
return c.Internal.NetAgentVersion(ctx, p)
|
||||
}
|
||||
|
||||
func (c *CommonStruct) NetPeerInfo(ctx context.Context, p peer.ID) (*api.ExtendedPeerInfo, error) {
|
||||
return c.Internal.NetPeerInfo(ctx, p)
|
||||
}
|
||||
|
||||
// ID implements API.ID
|
||||
func (c *CommonStruct) ID(ctx context.Context) (peer.ID, error) {
|
||||
return c.Internal.ID(ctx)
|
||||
}
|
||||
|
||||
// Version implements API.Version
|
||||
func (c *CommonStruct) Version(ctx context.Context) (api.Version, error) {
|
||||
func (c *CommonStruct) Version(ctx context.Context) (api.APIVersion, error) {
|
||||
return c.Internal.Version(ctx)
|
||||
}
|
||||
|
||||
@ -1274,7 +1278,7 @@ func (c *StorageMinerStruct) ActorAddressConfig(ctx context.Context) (api.Addres
|
||||
return c.Internal.ActorAddressConfig(ctx)
|
||||
}
|
||||
|
||||
func (c *StorageMinerStruct) PledgeSector(ctx context.Context) error {
|
||||
func (c *StorageMinerStruct) PledgeSector(ctx context.Context) (abi.SectorID, error) {
|
||||
return c.Internal.PledgeSector(ctx)
|
||||
}
|
||||
|
||||
@ -1610,7 +1614,7 @@ func (c *StorageMinerStruct) CheckProvable(ctx context.Context, pp abi.Registere
|
||||
|
||||
// WorkerStruct
|
||||
|
||||
func (w *WorkerStruct) Version(ctx context.Context) (build.Version, error) {
|
||||
func (w *WorkerStruct) Version(ctx context.Context) (api.Version, error) {
|
||||
return w.Internal.Version(ctx)
|
||||
}
|
||||
|
||||
|
@ -113,7 +113,7 @@ func init() {
|
||||
addExample(network.Connected)
|
||||
addExample(dtypes.NetworkName("lotus"))
|
||||
addExample(api.SyncStateStage(1))
|
||||
addExample(build.FullAPIVersion)
|
||||
addExample(api.FullAPIVersion)
|
||||
addExample(api.PCHInbound)
|
||||
addExample(time.Minute)
|
||||
addExample(datatransfer.TransferID(3))
|
||||
@ -123,6 +123,8 @@ func init() {
|
||||
addExample(retrievalmarket.DealStatusNew)
|
||||
addExample(network.ReachabilityPublic)
|
||||
addExample(build.NewestNetworkVersion)
|
||||
addExample(map[string]int{"name": 42})
|
||||
addExample(map[string]time.Time{"name": time.Unix(1615243938, 0).UTC()})
|
||||
addExample(&types.ExecutionTrace{
|
||||
Msg: exampleValue("init", reflect.TypeOf(&types.Message{}), nil).(*types.Message),
|
||||
MsgRct: exampleValue("init", reflect.TypeOf(&types.MessageReceipt{}), nil).(*types.MessageReceipt),
|
||||
|
2987
api/mocks/mock_full.go
Normal file
2987
api/mocks/mock_full.go
Normal file
File diff suppressed because it is too large
Load Diff
@ -15,7 +15,7 @@ import (
|
||||
cbor "github.com/ipfs/go-ipld-cbor"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/api/apibstore"
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/actors/adt"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin"
|
||||
@ -132,7 +132,7 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) {
|
||||
t.Fatal("Unable to settle payment channel")
|
||||
}
|
||||
|
||||
creatorStore := adt.WrapStore(ctx, cbor.NewCborStore(apibstore.NewAPIBlockstore(paymentCreator)))
|
||||
creatorStore := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewAPIBlockstore(paymentCreator)))
|
||||
|
||||
// wait for the receiver to submit their vouchers
|
||||
ev := events.NewEvents(ctx, paymentCreator)
|
||||
|
@ -6,7 +6,6 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/network"
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
@ -75,23 +74,9 @@ func testTapeFix(t *testing.T, b APIBuilder, blocktime time.Duration, after bool
|
||||
<-done
|
||||
}()
|
||||
|
||||
err = miner.PledgeSector(ctx)
|
||||
sid, err := miner.PledgeSector(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Wait till done.
|
||||
var sectorNo abi.SectorNumber
|
||||
for {
|
||||
s, err := miner.SectorsList(ctx) // Note - the test builder doesn't import genesis sectors into FSM
|
||||
require.NoError(t, err)
|
||||
fmt.Printf("Sectors: %d\n", len(s))
|
||||
if len(s) == 1 {
|
||||
sectorNo = s[0]
|
||||
break
|
||||
}
|
||||
|
||||
build.Clock.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
|
||||
fmt.Printf("All sectors is fsm\n")
|
||||
|
||||
// If before, we expect the precommit to fail
|
||||
@ -103,7 +88,7 @@ func testTapeFix(t *testing.T, b APIBuilder, blocktime time.Duration, after bool
|
||||
}
|
||||
|
||||
for {
|
||||
st, err := miner.SectorsStatus(ctx, sectorNo, false)
|
||||
st, err := miner.SectorsStatus(ctx, sid.Number, false)
|
||||
require.NoError(t, err)
|
||||
if st.State == successState {
|
||||
break
|
||||
|
@ -155,13 +155,13 @@ var MineNext = miner.MineReq{
|
||||
}
|
||||
|
||||
func (ts *testSuite) testVersion(t *testing.T) {
|
||||
build.RunningNodeType = build.NodeFull
|
||||
api.RunningNodeType = api.NodeFull
|
||||
|
||||
ctx := context.Background()
|
||||
apis, _ := ts.makeNodes(t, OneFull, OneMiner)
|
||||
api := apis[0]
|
||||
napi := apis[0]
|
||||
|
||||
v, err := api.Version(ctx)
|
||||
v, err := napi.Version(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
@ -162,7 +162,7 @@ func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n,
|
||||
log.Errorf("WAIT")
|
||||
}
|
||||
log.Errorf("PLEDGING %d", i)
|
||||
err := miner.PledgeSector(ctx)
|
||||
_, err := miner.PledgeSector(ctx)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
|
16
api/types.go
16
api/types.go
@ -3,6 +3,7 @@ package api
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
datatransfer "github.com/filecoin-project/go-data-transfer"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
@ -99,3 +100,18 @@ type NetBlockList struct {
|
||||
IPAddrs []string
|
||||
IPSubnets []string
|
||||
}
|
||||
|
||||
type ExtendedPeerInfo struct {
|
||||
ID peer.ID
|
||||
Agent string
|
||||
Addrs []string
|
||||
Protocols []string
|
||||
ConnMgrMeta *ConnMgrInfo
|
||||
}
|
||||
|
||||
type ConnMgrInfo struct {
|
||||
FirstSeen time.Time
|
||||
Value int
|
||||
Tags map[string]int
|
||||
Conns map[string]time.Time
|
||||
}
|
||||
|
71
api/version.go
Normal file
71
api/version.go
Normal file
@ -0,0 +1,71 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
xerrors "golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
type Version uint32
|
||||
|
||||
func newVer(major, minor, patch uint8) Version {
|
||||
return Version(uint32(major)<<16 | uint32(minor)<<8 | uint32(patch))
|
||||
}
|
||||
|
||||
// Ints returns (major, minor, patch) versions
|
||||
func (ve Version) Ints() (uint32, uint32, uint32) {
|
||||
v := uint32(ve)
|
||||
return (v & majorOnlyMask) >> 16, (v & minorOnlyMask) >> 8, v & patchOnlyMask
|
||||
}
|
||||
|
||||
func (ve Version) String() string {
|
||||
vmj, vmi, vp := ve.Ints()
|
||||
return fmt.Sprintf("%d.%d.%d", vmj, vmi, vp)
|
||||
}
|
||||
|
||||
func (ve Version) EqMajorMinor(v2 Version) bool {
|
||||
return ve&minorMask == v2&minorMask
|
||||
}
|
||||
|
||||
type NodeType int
|
||||
|
||||
const (
|
||||
NodeUnknown NodeType = iota
|
||||
|
||||
NodeFull
|
||||
NodeMiner
|
||||
NodeWorker
|
||||
)
|
||||
|
||||
var RunningNodeType NodeType
|
||||
|
||||
func VersionForType(nodeType NodeType) (Version, error) {
|
||||
switch nodeType {
|
||||
case NodeFull:
|
||||
return FullAPIVersion, nil
|
||||
case NodeMiner:
|
||||
return MinerAPIVersion, nil
|
||||
case NodeWorker:
|
||||
return WorkerAPIVersion, nil
|
||||
default:
|
||||
return Version(0), xerrors.Errorf("unknown node type %d", nodeType)
|
||||
}
|
||||
}
|
||||
|
||||
// semver versions of the rpc api exposed
|
||||
var (
|
||||
FullAPIVersion = newVer(1, 1, 0)
|
||||
MinerAPIVersion = newVer(1, 0, 1)
|
||||
WorkerAPIVersion = newVer(1, 0, 0)
|
||||
)
|
||||
|
||||
//nolint:varcheck,deadcode
|
||||
const (
|
||||
majorMask = 0xff0000
|
||||
minorMask = 0xffff00
|
||||
patchMask = 0xffffff
|
||||
|
||||
majorOnlyMask = 0xff0000
|
||||
minorOnlyMask = 0x00ff00
|
||||
patchOnlyMask = 0x0000ff
|
||||
)
|
66
blockstore/api.go
Normal file
66
blockstore/api.go
Normal file
@ -0,0 +1,66 @@
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
type ChainIO interface {
|
||||
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
|
||||
ChainHasObj(context.Context, cid.Cid) (bool, error)
|
||||
}
|
||||
|
||||
type apiBlockstore struct {
|
||||
api ChainIO
|
||||
}
|
||||
|
||||
// This blockstore is adapted in the constructor.
|
||||
var _ BasicBlockstore = (*apiBlockstore)(nil)
|
||||
|
||||
func NewAPIBlockstore(cio ChainIO) Blockstore {
|
||||
bs := &apiBlockstore{api: cio}
|
||||
return Adapt(bs) // return an adapted blockstore.
|
||||
}
|
||||
|
||||
func (a *apiBlockstore) DeleteBlock(cid.Cid) error {
|
||||
return xerrors.New("not supported")
|
||||
}
|
||||
|
||||
func (a *apiBlockstore) Has(c cid.Cid) (bool, error) {
|
||||
return a.api.ChainHasObj(context.TODO(), c)
|
||||
}
|
||||
|
||||
func (a *apiBlockstore) Get(c cid.Cid) (blocks.Block, error) {
|
||||
bb, err := a.api.ChainReadObj(context.TODO(), c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return blocks.NewBlockWithCid(bb, c)
|
||||
}
|
||||
|
||||
func (a *apiBlockstore) GetSize(c cid.Cid) (int, error) {
|
||||
bb, err := a.api.ChainReadObj(context.TODO(), c)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(bb), nil
|
||||
}
|
||||
|
||||
func (a *apiBlockstore) Put(blocks.Block) error {
|
||||
return xerrors.New("not supported")
|
||||
}
|
||||
|
||||
func (a *apiBlockstore) PutMany([]blocks.Block) error {
|
||||
return xerrors.New("not supported")
|
||||
}
|
||||
|
||||
func (a *apiBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
return nil, xerrors.New("not supported")
|
||||
}
|
||||
|
||||
func (a *apiBlockstore) HashOnRead(enabled bool) {
|
||||
return
|
||||
}
|
@ -16,7 +16,7 @@ import (
|
||||
logger "github.com/ipfs/go-log/v2"
|
||||
pool "github.com/libp2p/go-buffer-pool"
|
||||
|
||||
"github.com/filecoin-project/lotus/lib/blockstore"
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -110,10 +110,7 @@ func Open(opts Options) (*Blockstore, error) {
|
||||
return nil, fmt.Errorf("failed to open badger blockstore: %w", err)
|
||||
}
|
||||
|
||||
bs := &Blockstore{
|
||||
DB: db,
|
||||
}
|
||||
|
||||
bs := &Blockstore{DB: db}
|
||||
if p := opts.Prefix; p != "" {
|
||||
bs.prefixing = true
|
||||
bs.prefix = []byte(p)
|
||||
@ -134,6 +131,25 @@ func (b *Blockstore) Close() error {
|
||||
return b.DB.Close()
|
||||
}
|
||||
|
||||
// CollectGarbage runs garbage collection on the value log
|
||||
func (b *Blockstore) CollectGarbage() error {
|
||||
if atomic.LoadInt64(&b.state) != stateOpen {
|
||||
return ErrBlockstoreClosed
|
||||
}
|
||||
|
||||
var err error
|
||||
for err == nil {
|
||||
err = b.DB.RunValueLogGC(0.125)
|
||||
}
|
||||
|
||||
if err == badger.ErrNoRewrite {
|
||||
// not really an error in this case
|
||||
return nil
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// View implements blockstore.Viewer, which leverages zero-copy read-only
|
||||
// access to values.
|
||||
func (b *Blockstore) View(cid cid.Cid, fn func([]byte) error) error {
|
||||
@ -321,6 +337,44 @@ func (b *Blockstore) DeleteBlock(cid cid.Cid) error {
|
||||
})
|
||||
}
|
||||
|
||||
func (b *Blockstore) DeleteMany(cids []cid.Cid) error {
|
||||
if atomic.LoadInt64(&b.state) != stateOpen {
|
||||
return ErrBlockstoreClosed
|
||||
}
|
||||
|
||||
batch := b.DB.NewWriteBatch()
|
||||
defer batch.Cancel()
|
||||
|
||||
// toReturn tracks the byte slices to return to the pool, if we're using key
|
||||
// prefixing. we can't return each slice to the pool after each Set, because
|
||||
// badger holds on to the slice.
|
||||
var toReturn [][]byte
|
||||
if b.prefixing {
|
||||
toReturn = make([][]byte, 0, len(cids))
|
||||
defer func() {
|
||||
for _, b := range toReturn {
|
||||
KeyPool.Put(b)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
for _, cid := range cids {
|
||||
k, pooled := b.PooledStorageKey(cid)
|
||||
if pooled {
|
||||
toReturn = append(toReturn, k)
|
||||
}
|
||||
if err := batch.Delete(k); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err := batch.Flush()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("failed to delete blocks from badger blockstore: %w", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// AllKeysChan implements Blockstore.AllKeysChan.
|
||||
func (b *Blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
if atomic.LoadInt64(&b.state) != stateOpen {
|
@ -6,8 +6,9 @@ import (
|
||||
"testing"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
blockstore "github.com/ipfs/go-ipfs-blockstore"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
)
|
||||
|
||||
func TestBadgerBlockstore(t *testing.T) {
|
||||
@ -60,8 +61,8 @@ func TestStorageKey(t *testing.T) {
|
||||
require.Equal(t, k3, k2)
|
||||
}
|
||||
|
||||
func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (bs blockstore.Blockstore, path string) {
|
||||
return func(tb testing.TB) (bs blockstore.Blockstore, path string) {
|
||||
func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) {
|
||||
return func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) {
|
||||
tb.Helper()
|
||||
|
||||
path, err := ioutil.TempDir("", "")
|
||||
@ -82,8 +83,8 @@ func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (
|
||||
}
|
||||
}
|
||||
|
||||
func openBlockstore(optsSupplier func(path string) Options) func(tb testing.TB, path string) (bs blockstore.Blockstore, err error) {
|
||||
return func(tb testing.TB, path string) (bs blockstore.Blockstore, err error) {
|
||||
func openBlockstore(optsSupplier func(path string) Options) func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) {
|
||||
return func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) {
|
||||
tb.Helper()
|
||||
return Open(optsSupplier(path))
|
||||
}
|
@ -8,18 +8,19 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/filecoin-project/lotus/lib/blockstore"
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
u "github.com/ipfs/go-ipfs-util"
|
||||
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TODO: move this to go-ipfs-blockstore.
|
||||
type Suite struct {
|
||||
NewBlockstore func(tb testing.TB) (bs blockstore.Blockstore, path string)
|
||||
OpenBlockstore func(tb testing.TB, path string) (bs blockstore.Blockstore, err error)
|
||||
NewBlockstore func(tb testing.TB) (bs blockstore.BasicBlockstore, path string)
|
||||
OpenBlockstore func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error)
|
||||
}
|
||||
|
||||
func (s *Suite) RunTests(t *testing.T, prefix string) {
|
||||
@ -290,7 +291,7 @@ func (s *Suite) TestDelete(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
func insertBlocks(t *testing.T, bs blockstore.Blockstore, count int) []cid.Cid {
|
||||
func insertBlocks(t *testing.T, bs blockstore.BasicBlockstore, count int) []cid.Cid {
|
||||
keys := make([]cid.Cid, count)
|
||||
for i := 0; i < count; i++ {
|
||||
block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
|
95
blockstore/blockstore.go
Normal file
95
blockstore/blockstore.go
Normal file
@ -0,0 +1,95 @@
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
cid "github.com/ipfs/go-cid"
|
||||
ds "github.com/ipfs/go-datastore"
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
|
||||
blockstore "github.com/ipfs/go-ipfs-blockstore"
|
||||
)
|
||||
|
||||
var log = logging.Logger("blockstore")
|
||||
|
||||
var ErrNotFound = blockstore.ErrNotFound
|
||||
|
||||
// Blockstore is the blockstore interface used by Lotus. It is the union
|
||||
// of the basic go-ipfs blockstore, with other capabilities required by Lotus,
|
||||
// e.g. View or Sync.
|
||||
type Blockstore interface {
|
||||
blockstore.Blockstore
|
||||
blockstore.Viewer
|
||||
BatchDeleter
|
||||
}
|
||||
|
||||
// BasicBlockstore is an alias to the original IPFS Blockstore.
|
||||
type BasicBlockstore = blockstore.Blockstore
|
||||
|
||||
type Viewer = blockstore.Viewer
|
||||
|
||||
type BatchDeleter interface {
|
||||
DeleteMany(cids []cid.Cid) error
|
||||
}
|
||||
|
||||
// WrapIDStore wraps the underlying blockstore in an "identity" blockstore.
|
||||
// The ID store filters out all puts for blocks with CIDs using the "identity"
|
||||
// hash function. It also extracts inlined blocks from CIDs using the identity
|
||||
// hash function and returns them on get/has, ignoring the contents of the
|
||||
// blockstore.
|
||||
func WrapIDStore(bstore blockstore.Blockstore) Blockstore {
|
||||
if is, ok := bstore.(*idstore); ok {
|
||||
// already wrapped
|
||||
return is
|
||||
}
|
||||
|
||||
if bs, ok := bstore.(Blockstore); ok {
|
||||
// we need to wrap our own because we don't want to neuter the DeleteMany method
|
||||
// the underlying blockstore has implemented an (efficient) DeleteMany
|
||||
return NewIDStore(bs)
|
||||
}
|
||||
|
||||
// The underlying blockstore does not implement DeleteMany, so we need to shim it.
|
||||
// This is less efficient as it'll iterate and perform single deletes.
|
||||
return NewIDStore(Adapt(bstore))
|
||||
}
|
||||
|
||||
// FromDatastore creates a new blockstore backed by the given datastore.
|
||||
func FromDatastore(dstore ds.Batching) Blockstore {
|
||||
return WrapIDStore(blockstore.NewBlockstore(dstore))
|
||||
}
|
||||
|
||||
type adaptedBlockstore struct {
|
||||
blockstore.Blockstore
|
||||
}
|
||||
|
||||
var _ Blockstore = (*adaptedBlockstore)(nil)
|
||||
|
||||
func (a *adaptedBlockstore) View(cid cid.Cid, callback func([]byte) error) error {
|
||||
blk, err := a.Get(cid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return callback(blk.RawData())
|
||||
}
|
||||
|
||||
func (a *adaptedBlockstore) DeleteMany(cids []cid.Cid) error {
|
||||
for _, cid := range cids {
|
||||
err := a.DeleteBlock(cid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Adapt adapts a standard blockstore to a Lotus blockstore by
|
||||
// enriching it with the extra methods that Lotus requires (e.g. View, Sync).
|
||||
//
|
||||
// View proxies over to Get and calls the callback with the value supplied by Get.
|
||||
// Sync noops.
|
||||
func Adapt(bs blockstore.Blockstore) Blockstore {
|
||||
if ret, ok := bs.(Blockstore); ok {
|
||||
return ret
|
||||
}
|
||||
return &adaptedBlockstore{bs}
|
||||
}
|
174
blockstore/buffered.go
Normal file
174
blockstore/buffered.go
Normal file
@ -0,0 +1,174 @@
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
block "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
// buflog is a logger for the buffered blockstore. It is subscoped from the
|
||||
// blockstore logger.
|
||||
var buflog = log.Named("buf")
|
||||
|
||||
type BufferedBlockstore struct {
|
||||
read Blockstore
|
||||
write Blockstore
|
||||
}
|
||||
|
||||
func NewBuffered(base Blockstore) *BufferedBlockstore {
|
||||
var buf Blockstore
|
||||
if os.Getenv("LOTUS_DISABLE_VM_BUF") == "iknowitsabadidea" {
|
||||
buflog.Warn("VM BLOCKSTORE BUFFERING IS DISABLED")
|
||||
buf = base
|
||||
} else {
|
||||
buf = NewMemory()
|
||||
}
|
||||
|
||||
bs := &BufferedBlockstore{
|
||||
read: base,
|
||||
write: buf,
|
||||
}
|
||||
return bs
|
||||
}
|
||||
|
||||
func NewTieredBstore(r Blockstore, w Blockstore) *BufferedBlockstore {
|
||||
return &BufferedBlockstore{
|
||||
read: r,
|
||||
write: w,
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
_ Blockstore = (*BufferedBlockstore)(nil)
|
||||
_ Viewer = (*BufferedBlockstore)(nil)
|
||||
)
|
||||
|
||||
func (bs *BufferedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
a, err := bs.read.AllKeysChan(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
b, err := bs.write.AllKeysChan(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := make(chan cid.Cid)
|
||||
go func() {
|
||||
defer close(out)
|
||||
for a != nil || b != nil {
|
||||
select {
|
||||
case val, ok := <-a:
|
||||
if !ok {
|
||||
a = nil
|
||||
} else {
|
||||
select {
|
||||
case out <- val:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
case val, ok := <-b:
|
||||
if !ok {
|
||||
b = nil
|
||||
} else {
|
||||
select {
|
||||
case out <- val:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) DeleteBlock(c cid.Cid) error {
|
||||
if err := bs.read.DeleteBlock(c); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return bs.write.DeleteBlock(c)
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) DeleteMany(cids []cid.Cid) error {
|
||||
if err := bs.read.DeleteMany(cids); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return bs.write.DeleteMany(cids)
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) View(c cid.Cid, callback func([]byte) error) error {
|
||||
// both stores are viewable.
|
||||
if err := bs.write.View(c, callback); err == ErrNotFound {
|
||||
// not found in write blockstore; fall through.
|
||||
} else {
|
||||
return err // propagate errors, or nil, i.e. found.
|
||||
}
|
||||
return bs.read.View(c, callback)
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) Get(c cid.Cid) (block.Block, error) {
|
||||
if out, err := bs.write.Get(c); err != nil {
|
||||
if err != ErrNotFound {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
return bs.read.Get(c)
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) GetSize(c cid.Cid) (int, error) {
|
||||
s, err := bs.read.GetSize(c)
|
||||
if err == ErrNotFound || s == 0 {
|
||||
return bs.write.GetSize(c)
|
||||
}
|
||||
|
||||
return s, err
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) Put(blk block.Block) error {
|
||||
has, err := bs.read.Has(blk.Cid()) // TODO: consider dropping this check
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if has {
|
||||
return nil
|
||||
}
|
||||
|
||||
return bs.write.Put(blk)
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) Has(c cid.Cid) (bool, error) {
|
||||
has, err := bs.write.Has(c)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if has {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return bs.read.Has(c)
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) HashOnRead(hor bool) {
|
||||
bs.read.HashOnRead(hor)
|
||||
bs.write.HashOnRead(hor)
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) PutMany(blks []block.Block) error {
|
||||
return bs.write.PutMany(blks)
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) Read() Blockstore {
|
||||
return bs.read
|
||||
}
|
25
blockstore/cached.go
Normal file
25
blockstore/cached.go
Normal file
@ -0,0 +1,25 @@
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
blockstore "github.com/ipfs/go-ipfs-blockstore"
|
||||
)
|
||||
|
||||
type CacheOpts = blockstore.CacheOpts
|
||||
|
||||
func DefaultCacheOpts() CacheOpts {
|
||||
return CacheOpts{
|
||||
HasBloomFilterSize: 0,
|
||||
HasBloomFilterHashes: 0,
|
||||
HasARCCacheSize: 512 << 10,
|
||||
}
|
||||
}
|
||||
|
||||
func CachedBlockstore(ctx context.Context, bs Blockstore, opts CacheOpts) (Blockstore, error) {
|
||||
cached, err := blockstore.CachedBlockstore(ctx, bs, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return WrapIDStore(cached), nil
|
||||
}
|
9
blockstore/doc.go
Normal file
9
blockstore/doc.go
Normal file
@ -0,0 +1,9 @@
|
||||
// Package blockstore and subpackages contain most of the blockstore
|
||||
// implementations used by Lotus.
|
||||
//
|
||||
// Blockstores not ultimately constructed out of the building blocks in this
|
||||
// package may not work properly.
|
||||
//
|
||||
// This package re-exports parts of the go-ipfs-blockstore package such that
|
||||
// no other package needs to import it directly, for ergonomics and traceability.
|
||||
package blockstore
|
@ -9,48 +9,61 @@ import (
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
blockstore "github.com/ipfs/go-ipfs-blockstore"
|
||||
logging "github.com/ipfs/go-log"
|
||||
)
|
||||
|
||||
var log = logging.Logger("blockstore")
|
||||
|
||||
type FallbackStore struct {
|
||||
blockstore.Blockstore
|
||||
|
||||
fallbackGetBlock func(context.Context, cid.Cid) (blocks.Block, error)
|
||||
lk sync.RWMutex
|
||||
// UnwrapFallbackStore takes a blockstore, and returns the underlying blockstore
|
||||
// if it was a FallbackStore. Otherwise, it just returns the supplied store
|
||||
// unmodified.
|
||||
func UnwrapFallbackStore(bs Blockstore) (Blockstore, bool) {
|
||||
if fbs, ok := bs.(*FallbackStore); ok {
|
||||
return fbs.Blockstore, true
|
||||
}
|
||||
return bs, false
|
||||
}
|
||||
|
||||
func (fbs *FallbackStore) SetFallback(fg func(context.Context, cid.Cid) (blocks.Block, error)) {
|
||||
// FallbackStore is a read-through store that queries another (potentially
|
||||
// remote) source if the block is not found locally. If the block is found
|
||||
// during the fallback, it stores it in the local store.
|
||||
type FallbackStore struct {
|
||||
Blockstore
|
||||
|
||||
lk sync.RWMutex
|
||||
// missFn is the function that will be invoked on a local miss to pull the
|
||||
// block from elsewhere.
|
||||
missFn func(context.Context, cid.Cid) (blocks.Block, error)
|
||||
}
|
||||
|
||||
var _ Blockstore = (*FallbackStore)(nil)
|
||||
|
||||
func (fbs *FallbackStore) SetFallback(missFn func(context.Context, cid.Cid) (blocks.Block, error)) {
|
||||
fbs.lk.Lock()
|
||||
defer fbs.lk.Unlock()
|
||||
|
||||
fbs.fallbackGetBlock = fg
|
||||
fbs.missFn = missFn
|
||||
}
|
||||
|
||||
func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
|
||||
log.Errorw("fallbackstore: Block not found locally, fetching from the network", "cid", c)
|
||||
log.Warnf("fallbackstore: block not found locally, fetching from the network; cid: %s", c)
|
||||
fbs.lk.RLock()
|
||||
defer fbs.lk.RUnlock()
|
||||
|
||||
if fbs.fallbackGetBlock == nil {
|
||||
if fbs.missFn == nil {
|
||||
// FallbackStore wasn't configured yet (chainstore/bitswap aren't up yet)
|
||||
// Wait for a bit and retry
|
||||
fbs.lk.RUnlock()
|
||||
time.Sleep(5 * time.Second)
|
||||
fbs.lk.RLock()
|
||||
|
||||
if fbs.fallbackGetBlock == nil {
|
||||
log.Errorw("fallbackstore: fallbackGetBlock not configured yet")
|
||||
return nil, blockstore.ErrNotFound
|
||||
if fbs.missFn == nil {
|
||||
log.Errorw("fallbackstore: missFn not configured yet")
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.TODO(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
b, err := fbs.fallbackGetBlock(ctx, c)
|
||||
b, err := fbs.missFn(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -69,7 +82,7 @@ func (fbs *FallbackStore) Get(c cid.Cid) (blocks.Block, error) {
|
||||
switch err {
|
||||
case nil:
|
||||
return b, nil
|
||||
case blockstore.ErrNotFound:
|
||||
case ErrNotFound:
|
||||
return fbs.getFallback(c)
|
||||
default:
|
||||
return b, err
|
||||
@ -81,7 +94,7 @@ func (fbs *FallbackStore) GetSize(c cid.Cid) (int, error) {
|
||||
switch err {
|
||||
case nil:
|
||||
return sz, nil
|
||||
case blockstore.ErrNotFound:
|
||||
case ErrNotFound:
|
||||
b, err := fbs.getFallback(c)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
@ -91,5 +104,3 @@ func (fbs *FallbackStore) GetSize(c cid.Cid) (int, error) {
|
||||
return sz, err
|
||||
}
|
||||
}
|
||||
|
||||
var _ blockstore.Blockstore = &FallbackStore{}
|
174
blockstore/idstore.go
Normal file
174
blockstore/idstore.go
Normal file
@ -0,0 +1,174 @@
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
mh "github.com/multiformats/go-multihash"
|
||||
)
|
||||
|
||||
var _ Blockstore = (*idstore)(nil)
|
||||
|
||||
type idstore struct {
|
||||
bs Blockstore
|
||||
}
|
||||
|
||||
func NewIDStore(bs Blockstore) Blockstore {
|
||||
return &idstore{bs: bs}
|
||||
}
|
||||
|
||||
func decodeCid(cid cid.Cid) (inline bool, data []byte, err error) {
|
||||
if cid.Prefix().MhType != mh.IDENTITY {
|
||||
return false, nil, nil
|
||||
}
|
||||
|
||||
dmh, err := mh.Decode(cid.Hash())
|
||||
if err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
|
||||
if dmh.Code == mh.IDENTITY {
|
||||
return true, dmh.Digest, nil
|
||||
}
|
||||
|
||||
return false, nil, err
|
||||
}
|
||||
|
||||
func (b *idstore) Has(cid cid.Cid) (bool, error) {
|
||||
inline, _, err := decodeCid(cid)
|
||||
if err != nil {
|
||||
return false, xerrors.Errorf("error decoding Cid: %w", err)
|
||||
}
|
||||
|
||||
if inline {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return b.bs.Has(cid)
|
||||
}
|
||||
|
||||
func (b *idstore) Get(cid cid.Cid) (blocks.Block, error) {
|
||||
inline, data, err := decodeCid(cid)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error decoding Cid: %w", err)
|
||||
}
|
||||
|
||||
if inline {
|
||||
return blocks.NewBlockWithCid(data, cid)
|
||||
}
|
||||
|
||||
return b.bs.Get(cid)
|
||||
}
|
||||
|
||||
func (b *idstore) GetSize(cid cid.Cid) (int, error) {
|
||||
inline, data, err := decodeCid(cid)
|
||||
if err != nil {
|
||||
return 0, xerrors.Errorf("error decoding Cid: %w", err)
|
||||
}
|
||||
|
||||
if inline {
|
||||
return len(data), err
|
||||
}
|
||||
|
||||
return b.bs.GetSize(cid)
|
||||
}
|
||||
|
||||
func (b *idstore) View(cid cid.Cid, cb func([]byte) error) error {
|
||||
inline, data, err := decodeCid(cid)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error decoding Cid: %w", err)
|
||||
}
|
||||
|
||||
if inline {
|
||||
return cb(data)
|
||||
}
|
||||
|
||||
return b.bs.View(cid, cb)
|
||||
}
|
||||
|
||||
func (b *idstore) Put(blk blocks.Block) error {
|
||||
inline, _, err := decodeCid(blk.Cid())
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error decoding Cid: %w", err)
|
||||
}
|
||||
|
||||
if inline {
|
||||
return nil
|
||||
}
|
||||
|
||||
return b.bs.Put(blk)
|
||||
}
|
||||
|
||||
func (b *idstore) PutMany(blks []blocks.Block) error {
|
||||
toPut := make([]blocks.Block, 0, len(blks))
|
||||
for _, blk := range blks {
|
||||
inline, _, err := decodeCid(blk.Cid())
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error decoding Cid: %w", err)
|
||||
}
|
||||
|
||||
if inline {
|
||||
continue
|
||||
}
|
||||
toPut = append(toPut, blk)
|
||||
}
|
||||
|
||||
if len(toPut) > 0 {
|
||||
return b.bs.PutMany(toPut)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *idstore) DeleteBlock(cid cid.Cid) error {
|
||||
inline, _, err := decodeCid(cid)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error decoding Cid: %w", err)
|
||||
}
|
||||
|
||||
if inline {
|
||||
return nil
|
||||
}
|
||||
|
||||
return b.bs.DeleteBlock(cid)
|
||||
}
|
||||
|
||||
func (b *idstore) DeleteMany(cids []cid.Cid) error {
|
||||
toDelete := make([]cid.Cid, 0, len(cids))
|
||||
for _, cid := range cids {
|
||||
inline, _, err := decodeCid(cid)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error decoding Cid: %w", err)
|
||||
}
|
||||
|
||||
if inline {
|
||||
continue
|
||||
}
|
||||
toDelete = append(toDelete, cid)
|
||||
}
|
||||
|
||||
if len(toDelete) > 0 {
|
||||
return b.bs.DeleteMany(toDelete)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *idstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
return b.bs.AllKeysChan(ctx)
|
||||
}
|
||||
|
||||
func (b *idstore) HashOnRead(enabled bool) {
|
||||
b.bs.HashOnRead(enabled)
|
||||
}
|
||||
|
||||
func (b *idstore) Close() error {
|
||||
if c, ok := b.bs.(io.Closer); ok {
|
||||
return c.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
@ -1,4 +1,4 @@
|
||||
package ipfsbstore
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@ -16,16 +16,16 @@ import (
|
||||
iface "github.com/ipfs/interface-go-ipfs-core"
|
||||
"github.com/ipfs/interface-go-ipfs-core/options"
|
||||
"github.com/ipfs/interface-go-ipfs-core/path"
|
||||
|
||||
"github.com/filecoin-project/lotus/lib/blockstore"
|
||||
)
|
||||
|
||||
type IpfsBstore struct {
|
||||
type IPFSBlockstore struct {
|
||||
ctx context.Context
|
||||
api, offlineAPI iface.CoreAPI
|
||||
}
|
||||
|
||||
func NewIpfsBstore(ctx context.Context, onlineMode bool) (*IpfsBstore, error) {
|
||||
var _ BasicBlockstore = (*IPFSBlockstore)(nil)
|
||||
|
||||
func NewLocalIPFSBlockstore(ctx context.Context, onlineMode bool) (Blockstore, error) {
|
||||
localApi, err := httpapi.NewLocalApi()
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("getting local ipfs api: %w", err)
|
||||
@ -34,6 +34,7 @@ func NewIpfsBstore(ctx context.Context, onlineMode bool) (*IpfsBstore, error) {
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("setting offline mode: %s", err)
|
||||
}
|
||||
|
||||
offlineAPI := api
|
||||
if onlineMode {
|
||||
offlineAPI, err = localApi.WithOptions(options.Api.Offline(true))
|
||||
@ -42,14 +43,16 @@ func NewIpfsBstore(ctx context.Context, onlineMode bool) (*IpfsBstore, error) {
|
||||
}
|
||||
}
|
||||
|
||||
return &IpfsBstore{
|
||||
bs := &IPFSBlockstore{
|
||||
ctx: ctx,
|
||||
api: api,
|
||||
offlineAPI: offlineAPI,
|
||||
}, nil
|
||||
}
|
||||
|
||||
return Adapt(bs), nil
|
||||
}
|
||||
|
||||
func NewRemoteIpfsBstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineMode bool) (*IpfsBstore, error) {
|
||||
func NewRemoteIPFSBlockstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineMode bool) (Blockstore, error) {
|
||||
httpApi, err := httpapi.NewApi(maddr)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("setting remote ipfs api: %w", err)
|
||||
@ -58,6 +61,7 @@ func NewRemoteIpfsBstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineM
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("applying offline mode: %s", err)
|
||||
}
|
||||
|
||||
offlineAPI := api
|
||||
if onlineMode {
|
||||
offlineAPI, err = httpApi.WithOptions(options.Api.Offline(true))
|
||||
@ -66,18 +70,20 @@ func NewRemoteIpfsBstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineM
|
||||
}
|
||||
}
|
||||
|
||||
return &IpfsBstore{
|
||||
bs := &IPFSBlockstore{
|
||||
ctx: ctx,
|
||||
api: api,
|
||||
offlineAPI: offlineAPI,
|
||||
}, nil
|
||||
}
|
||||
|
||||
return Adapt(bs), nil
|
||||
}
|
||||
|
||||
func (i *IpfsBstore) DeleteBlock(cid cid.Cid) error {
|
||||
func (i *IPFSBlockstore) DeleteBlock(cid cid.Cid) error {
|
||||
return xerrors.Errorf("not supported")
|
||||
}
|
||||
|
||||
func (i *IpfsBstore) Has(cid cid.Cid) (bool, error) {
|
||||
func (i *IPFSBlockstore) Has(cid cid.Cid) (bool, error) {
|
||||
_, err := i.offlineAPI.Block().Stat(i.ctx, path.IpldPath(cid))
|
||||
if err != nil {
|
||||
// The underlying client is running in Offline mode.
|
||||
@ -93,7 +99,7 @@ func (i *IpfsBstore) Has(cid cid.Cid) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (i *IpfsBstore) Get(cid cid.Cid) (blocks.Block, error) {
|
||||
func (i *IPFSBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
|
||||
rd, err := i.api.Block().Get(i.ctx, path.IpldPath(cid))
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("getting ipfs block: %w", err)
|
||||
@ -107,7 +113,7 @@ func (i *IpfsBstore) Get(cid cid.Cid) (blocks.Block, error) {
|
||||
return blocks.NewBlockWithCid(data, cid)
|
||||
}
|
||||
|
||||
func (i *IpfsBstore) GetSize(cid cid.Cid) (int, error) {
|
||||
func (i *IPFSBlockstore) GetSize(cid cid.Cid) (int, error) {
|
||||
st, err := i.api.Block().Stat(i.ctx, path.IpldPath(cid))
|
||||
if err != nil {
|
||||
return 0, xerrors.Errorf("getting ipfs block: %w", err)
|
||||
@ -116,7 +122,7 @@ func (i *IpfsBstore) GetSize(cid cid.Cid) (int, error) {
|
||||
return st.Size(), nil
|
||||
}
|
||||
|
||||
func (i *IpfsBstore) Put(block blocks.Block) error {
|
||||
func (i *IPFSBlockstore) Put(block blocks.Block) error {
|
||||
mhd, err := multihash.Decode(block.Cid().Hash())
|
||||
if err != nil {
|
||||
return err
|
||||
@ -128,7 +134,7 @@ func (i *IpfsBstore) Put(block blocks.Block) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func (i *IpfsBstore) PutMany(blocks []blocks.Block) error {
|
||||
func (i *IPFSBlockstore) PutMany(blocks []blocks.Block) error {
|
||||
// TODO: could be done in parallel
|
||||
|
||||
for _, block := range blocks {
|
||||
@ -140,12 +146,10 @@ func (i *IpfsBstore) PutMany(blocks []blocks.Block) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (i *IpfsBstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
func (i *IPFSBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
return nil, xerrors.Errorf("not supported")
|
||||
}
|
||||
|
||||
func (i *IpfsBstore) HashOnRead(enabled bool) {
|
||||
func (i *IPFSBlockstore) HashOnRead(enabled bool) {
|
||||
return // TODO: We could technically support this, but..
|
||||
}
|
||||
|
||||
var _ blockstore.Blockstore = &IpfsBstore{}
|
@ -7,20 +7,32 @@ import (
|
||||
"github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
// MemStore is a terminal blockstore that keeps blocks in memory.
|
||||
type MemStore map[cid.Cid]blocks.Block
|
||||
// NewMemory returns a temporary memory-backed blockstore.
|
||||
func NewMemory() MemBlockstore {
|
||||
return make(MemBlockstore)
|
||||
}
|
||||
|
||||
func (m MemStore) DeleteBlock(k cid.Cid) error {
|
||||
// MemBlockstore is a terminal blockstore that keeps blocks in memory.
|
||||
type MemBlockstore map[cid.Cid]blocks.Block
|
||||
|
||||
func (m MemBlockstore) DeleteBlock(k cid.Cid) error {
|
||||
delete(m, k)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m MemStore) Has(k cid.Cid) (bool, error) {
|
||||
func (m MemBlockstore) DeleteMany(ks []cid.Cid) error {
|
||||
for _, k := range ks {
|
||||
delete(m, k)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m MemBlockstore) Has(k cid.Cid) (bool, error) {
|
||||
_, ok := m[k]
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
func (m MemStore) View(k cid.Cid, callback func([]byte) error) error {
|
||||
func (m MemBlockstore) View(k cid.Cid, callback func([]byte) error) error {
|
||||
b, ok := m[k]
|
||||
if !ok {
|
||||
return ErrNotFound
|
||||
@ -28,7 +40,7 @@ func (m MemStore) View(k cid.Cid, callback func([]byte) error) error {
|
||||
return callback(b.RawData())
|
||||
}
|
||||
|
||||
func (m MemStore) Get(k cid.Cid) (blocks.Block, error) {
|
||||
func (m MemBlockstore) Get(k cid.Cid) (blocks.Block, error) {
|
||||
b, ok := m[k]
|
||||
if !ok {
|
||||
return nil, ErrNotFound
|
||||
@ -37,7 +49,7 @@ func (m MemStore) Get(k cid.Cid) (blocks.Block, error) {
|
||||
}
|
||||
|
||||
// GetSize returns the CIDs mapped BlockSize
|
||||
func (m MemStore) GetSize(k cid.Cid) (int, error) {
|
||||
func (m MemBlockstore) GetSize(k cid.Cid) (int, error) {
|
||||
b, ok := m[k]
|
||||
if !ok {
|
||||
return 0, ErrNotFound
|
||||
@ -46,7 +58,7 @@ func (m MemStore) GetSize(k cid.Cid) (int, error) {
|
||||
}
|
||||
|
||||
// Put puts a given block to the underlying datastore
|
||||
func (m MemStore) Put(b blocks.Block) error {
|
||||
func (m MemBlockstore) Put(b blocks.Block) error {
|
||||
// Convert to a basic block for safety, but try to reuse the existing
|
||||
// block if it's already a basic block.
|
||||
k := b.Cid()
|
||||
@ -64,7 +76,7 @@ func (m MemStore) Put(b blocks.Block) error {
|
||||
|
||||
// PutMany puts a slice of blocks at the same time using batching
|
||||
// capabilities of the underlying datastore whenever possible.
|
||||
func (m MemStore) PutMany(bs []blocks.Block) error {
|
||||
func (m MemBlockstore) PutMany(bs []blocks.Block) error {
|
||||
for _, b := range bs {
|
||||
_ = m.Put(b) // can't fail
|
||||
}
|
||||
@ -74,7 +86,7 @@ func (m MemStore) PutMany(bs []blocks.Block) error {
|
||||
// AllKeysChan returns a channel from which
|
||||
// the CIDs in the Blockstore can be read. It should respect
|
||||
// the given context, closing the channel if it becomes Done.
|
||||
func (m MemStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
func (m MemBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
ch := make(chan cid.Cid, len(m))
|
||||
for k := range m {
|
||||
ch <- k
|
||||
@ -85,6 +97,6 @@ func (m MemStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
|
||||
// HashOnRead specifies if every read block should be
|
||||
// rehashed to make sure it matches its CID.
|
||||
func (m MemStore) HashOnRead(enabled bool) {
|
||||
func (m MemBlockstore) HashOnRead(enabled bool) {
|
||||
// no-op
|
||||
}
|
154
blockstore/metrics.go
Normal file
154
blockstore/metrics.go
Normal file
@ -0,0 +1,154 @@
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"go.opencensus.io/stats"
|
||||
"go.opencensus.io/stats/view"
|
||||
"go.opencensus.io/tag"
|
||||
)
|
||||
|
||||
//
|
||||
// Currently unused, but kept in repo in case we introduce one of the candidate
|
||||
// cache implementations (Freecache, Ristretto), both of which report these
|
||||
// metrics.
|
||||
//
|
||||
|
||||
// CacheMetricsEmitInterval is the interval at which metrics are emitted onto
|
||||
// OpenCensus.
|
||||
var CacheMetricsEmitInterval = 5 * time.Second
|
||||
|
||||
var (
|
||||
CacheName, _ = tag.NewKey("cache_name")
|
||||
)
|
||||
|
||||
// CacheMeasures groups all metrics emitted by the blockstore caches.
|
||||
var CacheMeasures = struct {
|
||||
HitRatio *stats.Float64Measure
|
||||
Hits *stats.Int64Measure
|
||||
Misses *stats.Int64Measure
|
||||
Entries *stats.Int64Measure
|
||||
QueriesServed *stats.Int64Measure
|
||||
Adds *stats.Int64Measure
|
||||
Updates *stats.Int64Measure
|
||||
Evictions *stats.Int64Measure
|
||||
CostAdded *stats.Int64Measure
|
||||
CostEvicted *stats.Int64Measure
|
||||
SetsDropped *stats.Int64Measure
|
||||
SetsRejected *stats.Int64Measure
|
||||
QueriesDropped *stats.Int64Measure
|
||||
}{
|
||||
HitRatio: stats.Float64("blockstore/cache/hit_ratio", "Hit ratio of blockstore cache", stats.UnitDimensionless),
|
||||
Hits: stats.Int64("blockstore/cache/hits", "Total number of hits at blockstore cache", stats.UnitDimensionless),
|
||||
Misses: stats.Int64("blockstore/cache/misses", "Total number of misses at blockstore cache", stats.UnitDimensionless),
|
||||
Entries: stats.Int64("blockstore/cache/entry_count", "Total number of entries currently in the blockstore cache", stats.UnitDimensionless),
|
||||
QueriesServed: stats.Int64("blockstore/cache/queries_served", "Total number of queries served by the blockstore cache", stats.UnitDimensionless),
|
||||
Adds: stats.Int64("blockstore/cache/adds", "Total number of adds to blockstore cache", stats.UnitDimensionless),
|
||||
Updates: stats.Int64("blockstore/cache/updates", "Total number of updates in blockstore cache", stats.UnitDimensionless),
|
||||
Evictions: stats.Int64("blockstore/cache/evictions", "Total number of evictions from blockstore cache", stats.UnitDimensionless),
|
||||
CostAdded: stats.Int64("blockstore/cache/cost_added", "Total cost (byte size) of entries added into blockstore cache", stats.UnitBytes),
|
||||
CostEvicted: stats.Int64("blockstore/cache/cost_evicted", "Total cost (byte size) of entries evicted by blockstore cache", stats.UnitBytes),
|
||||
SetsDropped: stats.Int64("blockstore/cache/sets_dropped", "Total number of sets dropped by blockstore cache", stats.UnitDimensionless),
|
||||
SetsRejected: stats.Int64("blockstore/cache/sets_rejected", "Total number of sets rejected by blockstore cache", stats.UnitDimensionless),
|
||||
QueriesDropped: stats.Int64("blockstore/cache/queries_dropped", "Total number of queries dropped by blockstore cache", stats.UnitDimensionless),
|
||||
}
|
||||
|
||||
// CacheViews groups all cache-related default views.
|
||||
var CacheViews = struct {
|
||||
HitRatio *view.View
|
||||
Hits *view.View
|
||||
Misses *view.View
|
||||
Entries *view.View
|
||||
QueriesServed *view.View
|
||||
Adds *view.View
|
||||
Updates *view.View
|
||||
Evictions *view.View
|
||||
CostAdded *view.View
|
||||
CostEvicted *view.View
|
||||
SetsDropped *view.View
|
||||
SetsRejected *view.View
|
||||
QueriesDropped *view.View
|
||||
}{
|
||||
HitRatio: &view.View{
|
||||
Measure: CacheMeasures.HitRatio,
|
||||
Aggregation: view.LastValue(),
|
||||
TagKeys: []tag.Key{CacheName},
|
||||
},
|
||||
Hits: &view.View{
|
||||
Measure: CacheMeasures.Hits,
|
||||
Aggregation: view.LastValue(),
|
||||
TagKeys: []tag.Key{CacheName},
|
||||
},
|
||||
Misses: &view.View{
|
||||
Measure: CacheMeasures.Misses,
|
||||
Aggregation: view.LastValue(),
|
||||
TagKeys: []tag.Key{CacheName},
|
||||
},
|
||||
Entries: &view.View{
|
||||
Measure: CacheMeasures.Entries,
|
||||
Aggregation: view.LastValue(),
|
||||
TagKeys: []tag.Key{CacheName},
|
||||
},
|
||||
QueriesServed: &view.View{
|
||||
Measure: CacheMeasures.QueriesServed,
|
||||
Aggregation: view.LastValue(),
|
||||
TagKeys: []tag.Key{CacheName},
|
||||
},
|
||||
Adds: &view.View{
|
||||
Measure: CacheMeasures.Adds,
|
||||
Aggregation: view.LastValue(),
|
||||
TagKeys: []tag.Key{CacheName},
|
||||
},
|
||||
Updates: &view.View{
|
||||
Measure: CacheMeasures.Updates,
|
||||
Aggregation: view.LastValue(),
|
||||
TagKeys: []tag.Key{CacheName},
|
||||
},
|
||||
Evictions: &view.View{
|
||||
Measure: CacheMeasures.Evictions,
|
||||
Aggregation: view.LastValue(),
|
||||
TagKeys: []tag.Key{CacheName},
|
||||
},
|
||||
CostAdded: &view.View{
|
||||
Measure: CacheMeasures.CostAdded,
|
||||
Aggregation: view.LastValue(),
|
||||
TagKeys: []tag.Key{CacheName},
|
||||
},
|
||||
CostEvicted: &view.View{
|
||||
Measure: CacheMeasures.CostEvicted,
|
||||
Aggregation: view.LastValue(),
|
||||
TagKeys: []tag.Key{CacheName},
|
||||
},
|
||||
SetsDropped: &view.View{
|
||||
Measure: CacheMeasures.SetsDropped,
|
||||
Aggregation: view.LastValue(),
|
||||
TagKeys: []tag.Key{CacheName},
|
||||
},
|
||||
SetsRejected: &view.View{
|
||||
Measure: CacheMeasures.SetsRejected,
|
||||
Aggregation: view.LastValue(),
|
||||
TagKeys: []tag.Key{CacheName},
|
||||
},
|
||||
QueriesDropped: &view.View{
|
||||
Measure: CacheMeasures.QueriesDropped,
|
||||
Aggregation: view.LastValue(),
|
||||
TagKeys: []tag.Key{CacheName},
|
||||
},
|
||||
}
|
||||
|
||||
// DefaultViews exports all default views for this package.
|
||||
var DefaultViews = []*view.View{
|
||||
CacheViews.HitRatio,
|
||||
CacheViews.Hits,
|
||||
CacheViews.Misses,
|
||||
CacheViews.Entries,
|
||||
CacheViews.QueriesServed,
|
||||
CacheViews.Adds,
|
||||
CacheViews.Updates,
|
||||
CacheViews.Evictions,
|
||||
CacheViews.CostAdded,
|
||||
CacheViews.CostEvicted,
|
||||
CacheViews.SetsDropped,
|
||||
CacheViews.SetsRejected,
|
||||
CacheViews.QueriesDropped,
|
||||
}
|
38
blockstore/splitstore/markset.go
Normal file
38
blockstore/splitstore/markset.go
Normal file
@ -0,0 +1,38 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
cid "github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
// MarkSet is a utility to keep track of seen CID, and later query for them.
|
||||
//
|
||||
// * If the expected dataset is large, it can be backed by a datastore (e.g. bbolt).
|
||||
// * If a probabilistic result is acceptable, it can be backed by a bloom filter (default).
|
||||
type MarkSet interface {
|
||||
Mark(cid.Cid) error
|
||||
Has(cid.Cid) (bool, error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
// markBytes is deliberately a non-nil empty byte slice for serialization.
|
||||
var markBytes = []byte{}
|
||||
|
||||
type MarkSetEnv interface {
|
||||
Create(name string, sizeHint int64) (MarkSet, error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error) {
|
||||
switch mtype {
|
||||
case "", "bloom":
|
||||
return NewBloomMarkSetEnv()
|
||||
case "bolt":
|
||||
return NewBoltMarkSetEnv(filepath.Join(path, "markset.bolt"))
|
||||
default:
|
||||
return nil, xerrors.Errorf("unknown mark set type %s", mtype)
|
||||
}
|
||||
}
|
77
blockstore/splitstore/markset_bloom.go
Normal file
77
blockstore/splitstore/markset_bloom.go
Normal file
@ -0,0 +1,77 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
bbloom "github.com/ipfs/bbloom"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
const (
|
||||
BloomFilterMinSize = 10_000_000
|
||||
BloomFilterProbability = 0.01
|
||||
)
|
||||
|
||||
type BloomMarkSetEnv struct{}
|
||||
|
||||
var _ MarkSetEnv = (*BloomMarkSetEnv)(nil)
|
||||
|
||||
type BloomMarkSet struct {
|
||||
salt []byte
|
||||
bf *bbloom.Bloom
|
||||
}
|
||||
|
||||
var _ MarkSet = (*BloomMarkSet)(nil)
|
||||
|
||||
func NewBloomMarkSetEnv() (*BloomMarkSetEnv, error) {
|
||||
return &BloomMarkSetEnv{}, nil
|
||||
}
|
||||
|
||||
func (e *BloomMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
|
||||
size := int64(BloomFilterMinSize)
|
||||
for size < sizeHint {
|
||||
size += BloomFilterMinSize
|
||||
}
|
||||
|
||||
salt := make([]byte, 4)
|
||||
_, err := rand.Read(salt)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error reading salt: %w", err)
|
||||
}
|
||||
|
||||
bf, err := bbloom.New(float64(size), BloomFilterProbability)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error creating bloom filter: %w", err)
|
||||
}
|
||||
|
||||
return &BloomMarkSet{salt: salt, bf: bf}, nil
|
||||
}
|
||||
|
||||
func (e *BloomMarkSetEnv) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BloomMarkSet) saltedKey(cid cid.Cid) []byte {
|
||||
hash := cid.Hash()
|
||||
key := make([]byte, len(s.salt)+len(hash))
|
||||
n := copy(key, s.salt)
|
||||
copy(key[n:], hash)
|
||||
rehash := sha256.Sum256(key)
|
||||
return rehash[:]
|
||||
}
|
||||
|
||||
func (s *BloomMarkSet) Mark(cid cid.Cid) error {
|
||||
s.bf.Add(s.saltedKey(cid))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BloomMarkSet) Has(cid cid.Cid) (bool, error) {
|
||||
return s.bf.Has(s.saltedKey(cid)), nil
|
||||
}
|
||||
|
||||
func (s *BloomMarkSet) Close() error {
|
||||
return nil
|
||||
}
|
81
blockstore/splitstore/markset_bolt.go
Normal file
81
blockstore/splitstore/markset_bolt.go
Normal file
@ -0,0 +1,81 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
cid "github.com/ipfs/go-cid"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
type BoltMarkSetEnv struct {
|
||||
db *bolt.DB
|
||||
}
|
||||
|
||||
var _ MarkSetEnv = (*BoltMarkSetEnv)(nil)
|
||||
|
||||
type BoltMarkSet struct {
|
||||
db *bolt.DB
|
||||
bucketId []byte
|
||||
}
|
||||
|
||||
var _ MarkSet = (*BoltMarkSet)(nil)
|
||||
|
||||
func NewBoltMarkSetEnv(path string) (*BoltMarkSetEnv, error) {
|
||||
db, err := bolt.Open(path, 0644,
|
||||
&bolt.Options{
|
||||
Timeout: 1 * time.Second,
|
||||
NoSync: true,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &BoltMarkSetEnv{db: db}, nil
|
||||
}
|
||||
|
||||
func (e *BoltMarkSetEnv) Create(name string, hint int64) (MarkSet, error) {
|
||||
bucketId := []byte(name)
|
||||
err := e.db.Update(func(tx *bolt.Tx) error {
|
||||
_, err := tx.CreateBucketIfNotExists(bucketId)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error creating bolt db bucket %s: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &BoltMarkSet{db: e.db, bucketId: bucketId}, nil
|
||||
}
|
||||
|
||||
func (e *BoltMarkSetEnv) Close() error {
|
||||
return e.db.Close()
|
||||
}
|
||||
|
||||
func (s *BoltMarkSet) Mark(cid cid.Cid) error {
|
||||
return s.db.Update(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket(s.bucketId)
|
||||
return b.Put(cid.Hash(), markBytes)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *BoltMarkSet) Has(cid cid.Cid) (result bool, err error) {
|
||||
err = s.db.View(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket(s.bucketId)
|
||||
v := b.Get(cid.Hash())
|
||||
result = v != nil
|
||||
return nil
|
||||
})
|
||||
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (s *BoltMarkSet) Close() error {
|
||||
return s.db.Update(func(tx *bolt.Tx) error {
|
||||
return tx.DeleteBucket(s.bucketId)
|
||||
})
|
||||
}
|
138
blockstore/splitstore/markset_test.go
Normal file
138
blockstore/splitstore/markset_test.go
Normal file
@ -0,0 +1,138 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
|
||||
cid "github.com/ipfs/go-cid"
|
||||
"github.com/multiformats/go-multihash"
|
||||
)
|
||||
|
||||
func TestBoltMarkSet(t *testing.T) {
|
||||
testMarkSet(t, "bolt")
|
||||
}
|
||||
|
||||
func TestBloomMarkSet(t *testing.T) {
|
||||
testMarkSet(t, "bloom")
|
||||
}
|
||||
|
||||
func testMarkSet(t *testing.T, lsType string) {
|
||||
t.Helper()
|
||||
|
||||
path, err := ioutil.TempDir("", "sweep-test.*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
env, err := OpenMarkSetEnv(path, lsType)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer env.Close() //nolint:errcheck
|
||||
|
||||
hotSet, err := env.Create("hot", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
coldSet, err := env.Create("cold", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
makeCid := func(key string) cid.Cid {
|
||||
h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return cid.NewCidV1(cid.Raw, h)
|
||||
}
|
||||
|
||||
mustHave := func(s MarkSet, cid cid.Cid) {
|
||||
has, err := s.Has(cid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !has {
|
||||
t.Fatal("mark not found")
|
||||
}
|
||||
}
|
||||
|
||||
mustNotHave := func(s MarkSet, cid cid.Cid) {
|
||||
has, err := s.Has(cid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if has {
|
||||
t.Fatal("unexpected mark")
|
||||
}
|
||||
}
|
||||
|
||||
k1 := makeCid("a")
|
||||
k2 := makeCid("b")
|
||||
k3 := makeCid("c")
|
||||
k4 := makeCid("d")
|
||||
|
||||
hotSet.Mark(k1) //nolint
|
||||
hotSet.Mark(k2) //nolint
|
||||
coldSet.Mark(k3) //nolint
|
||||
|
||||
mustHave(hotSet, k1)
|
||||
mustHave(hotSet, k2)
|
||||
mustNotHave(hotSet, k3)
|
||||
mustNotHave(hotSet, k4)
|
||||
|
||||
mustNotHave(coldSet, k1)
|
||||
mustNotHave(coldSet, k2)
|
||||
mustHave(coldSet, k3)
|
||||
mustNotHave(coldSet, k4)
|
||||
|
||||
// close them and reopen to redo the dance
|
||||
|
||||
err = hotSet.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = coldSet.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
hotSet, err = env.Create("hot", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
coldSet, err = env.Create("cold", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
hotSet.Mark(k3) //nolint
|
||||
hotSet.Mark(k4) //nolint
|
||||
coldSet.Mark(k1) //nolint
|
||||
|
||||
mustNotHave(hotSet, k1)
|
||||
mustNotHave(hotSet, k2)
|
||||
mustHave(hotSet, k3)
|
||||
mustHave(hotSet, k4)
|
||||
|
||||
mustHave(coldSet, k1)
|
||||
mustNotHave(coldSet, k2)
|
||||
mustNotHave(coldSet, k3)
|
||||
mustNotHave(coldSet, k4)
|
||||
|
||||
err = hotSet.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = coldSet.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
1069
blockstore/splitstore/splitstore.go
Normal file
1069
blockstore/splitstore/splitstore.go
Normal file
File diff suppressed because it is too large
Load Diff
255
blockstore/splitstore/splitstore_test.go
Normal file
255
blockstore/splitstore/splitstore_test.go
Normal file
@ -0,0 +1,255 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/chain/types/mock"
|
||||
|
||||
cid "github.com/ipfs/go-cid"
|
||||
datastore "github.com/ipfs/go-datastore"
|
||||
dssync "github.com/ipfs/go-datastore/sync"
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
)
|
||||
|
||||
func init() {
|
||||
CompactionThreshold = 5
|
||||
CompactionCold = 1
|
||||
CompactionBoundary = 2
|
||||
logging.SetLogLevel("splitstore", "DEBUG")
|
||||
}
|
||||
|
||||
func testSplitStore(t *testing.T, cfg *Config) {
|
||||
chain := &mockChain{}
|
||||
// genesis
|
||||
genBlock := mock.MkBlock(nil, 0, 0)
|
||||
genTs := mock.TipSet(genBlock)
|
||||
chain.push(genTs)
|
||||
|
||||
// the myriads of stores
|
||||
ds := dssync.MutexWrap(datastore.NewMapDatastore())
|
||||
hot := blockstore.NewMemorySync()
|
||||
cold := blockstore.NewMemorySync()
|
||||
|
||||
// put the genesis block to cold store
|
||||
blk, err := genBlock.ToStorageBlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = cold.Put(blk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// open the splitstore
|
||||
ss, err := Open("", ds, hot, cold, cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ss.Close() //nolint
|
||||
|
||||
err = ss.Start(chain)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// make some tipsets, but not enough to cause compaction
|
||||
mkBlock := func(curTs *types.TipSet, i int) *types.TipSet {
|
||||
blk := mock.MkBlock(curTs, uint64(i), uint64(i))
|
||||
sblk, err := blk.ToStorageBlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = ss.Put(sblk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ts := mock.TipSet(blk)
|
||||
chain.push(ts)
|
||||
|
||||
return ts
|
||||
}
|
||||
|
||||
mkGarbageBlock := func(curTs *types.TipSet, i int) {
|
||||
blk := mock.MkBlock(curTs, uint64(i), uint64(i))
|
||||
sblk, err := blk.ToStorageBlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = ss.Put(sblk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
waitForCompaction := func() {
|
||||
for atomic.LoadInt32(&ss.compacting) == 1 {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
curTs := genTs
|
||||
for i := 1; i < 5; i++ {
|
||||
curTs = mkBlock(curTs, i)
|
||||
waitForCompaction()
|
||||
}
|
||||
|
||||
mkGarbageBlock(genTs, 1)
|
||||
|
||||
// count objects in the cold and hot stores
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
countBlocks := func(bs blockstore.Blockstore) int {
|
||||
count := 0
|
||||
ch, err := bs.AllKeysChan(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for range ch {
|
||||
count++
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
coldCnt := countBlocks(cold)
|
||||
hotCnt := countBlocks(hot)
|
||||
|
||||
if coldCnt != 1 {
|
||||
t.Errorf("expected %d blocks, but got %d", 1, coldCnt)
|
||||
}
|
||||
|
||||
if hotCnt != 5 {
|
||||
t.Errorf("expected %d blocks, but got %d", 5, hotCnt)
|
||||
}
|
||||
|
||||
// trigger a compaction
|
||||
for i := 5; i < 10; i++ {
|
||||
curTs = mkBlock(curTs, i)
|
||||
waitForCompaction()
|
||||
}
|
||||
|
||||
coldCnt = countBlocks(cold)
|
||||
hotCnt = countBlocks(hot)
|
||||
|
||||
if !cfg.EnableFullCompaction {
|
||||
if coldCnt != 5 {
|
||||
t.Errorf("expected %d cold blocks, but got %d", 5, coldCnt)
|
||||
}
|
||||
|
||||
if hotCnt != 5 {
|
||||
t.Errorf("expected %d hot blocks, but got %d", 5, hotCnt)
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.EnableFullCompaction && !cfg.EnableGC {
|
||||
if coldCnt != 3 {
|
||||
t.Errorf("expected %d cold blocks, but got %d", 3, coldCnt)
|
||||
}
|
||||
|
||||
if hotCnt != 7 {
|
||||
t.Errorf("expected %d hot blocks, but got %d", 7, hotCnt)
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.EnableFullCompaction && cfg.EnableGC {
|
||||
if coldCnt != 2 {
|
||||
t.Errorf("expected %d cold blocks, but got %d", 2, coldCnt)
|
||||
}
|
||||
|
||||
if hotCnt != 7 {
|
||||
t.Errorf("expected %d hot blocks, but got %d", 7, hotCnt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitStoreSimpleCompaction(t *testing.T) {
|
||||
testSplitStore(t, &Config{TrackingStoreType: "mem"})
|
||||
}
|
||||
|
||||
func TestSplitStoreFullCompactionWithoutGC(t *testing.T) {
|
||||
testSplitStore(t, &Config{
|
||||
TrackingStoreType: "mem",
|
||||
EnableFullCompaction: true,
|
||||
})
|
||||
}
|
||||
|
||||
func TestSplitStoreFullCompactionWithGC(t *testing.T) {
|
||||
testSplitStore(t, &Config{
|
||||
TrackingStoreType: "mem",
|
||||
EnableFullCompaction: true,
|
||||
EnableGC: true,
|
||||
})
|
||||
}
|
||||
|
||||
type mockChain struct {
|
||||
sync.Mutex
|
||||
tipsets []*types.TipSet
|
||||
listener func(revert []*types.TipSet, apply []*types.TipSet) error
|
||||
}
|
||||
|
||||
func (c *mockChain) push(ts *types.TipSet) {
|
||||
c.Lock()
|
||||
c.tipsets = append(c.tipsets, ts)
|
||||
c.Unlock()
|
||||
|
||||
if c.listener != nil {
|
||||
err := c.listener(nil, []*types.TipSet{ts})
|
||||
if err != nil {
|
||||
log.Errorf("mockchain: error dispatching listener: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *mockChain) GetTipsetByHeight(_ context.Context, epoch abi.ChainEpoch, _ *types.TipSet, _ bool) (*types.TipSet, error) {
|
||||
c.Lock()
|
||||
defer c.Unlock()
|
||||
|
||||
iEpoch := int(epoch)
|
||||
if iEpoch > len(c.tipsets) {
|
||||
return nil, fmt.Errorf("bad epoch %d", epoch)
|
||||
}
|
||||
|
||||
return c.tipsets[iEpoch-1], nil
|
||||
}
|
||||
|
||||
func (c *mockChain) GetHeaviestTipSet() *types.TipSet {
|
||||
c.Lock()
|
||||
defer c.Unlock()
|
||||
|
||||
return c.tipsets[len(c.tipsets)-1]
|
||||
}
|
||||
|
||||
func (c *mockChain) SubscribeHeadChanges(change func(revert []*types.TipSet, apply []*types.TipSet) error) {
|
||||
c.listener = change
|
||||
}
|
||||
|
||||
func (c *mockChain) WalkSnapshot(_ context.Context, ts *types.TipSet, epochs abi.ChainEpoch, _ bool, _ bool, f func(cid.Cid) error) error {
|
||||
c.Lock()
|
||||
defer c.Unlock()
|
||||
|
||||
start := int(ts.Height()) - 1
|
||||
end := start - int(epochs)
|
||||
if end < 0 {
|
||||
end = -1
|
||||
}
|
||||
for i := start; i > end; i-- {
|
||||
ts := c.tipsets[i]
|
||||
for _, cid := range ts.Cids() {
|
||||
err := f(cid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
109
blockstore/splitstore/tracking.go
Normal file
109
blockstore/splitstore/tracking.go
Normal file
@ -0,0 +1,109 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
// TrackingStore is a persistent store that tracks blocks that are added
|
||||
// to the hotstore, tracking the epoch at which they are written.
|
||||
type TrackingStore interface {
|
||||
Put(cid.Cid, abi.ChainEpoch) error
|
||||
PutBatch([]cid.Cid, abi.ChainEpoch) error
|
||||
Get(cid.Cid) (abi.ChainEpoch, error)
|
||||
Delete(cid.Cid) error
|
||||
DeleteBatch([]cid.Cid) error
|
||||
ForEach(func(cid.Cid, abi.ChainEpoch) error) error
|
||||
Sync() error
|
||||
Close() error
|
||||
}
|
||||
|
||||
// OpenTrackingStore opens a tracking store of the specified type in the
|
||||
// specified path.
|
||||
func OpenTrackingStore(path string, ttype string) (TrackingStore, error) {
|
||||
switch ttype {
|
||||
case "", "bolt":
|
||||
return OpenBoltTrackingStore(filepath.Join(path, "tracker.bolt"))
|
||||
case "mem":
|
||||
return NewMemTrackingStore(), nil
|
||||
default:
|
||||
return nil, xerrors.Errorf("unknown tracking store type %s", ttype)
|
||||
}
|
||||
}
|
||||
|
||||
// NewMemTrackingStore creates an in-memory tracking store.
|
||||
// This is only useful for test or situations where you don't want to open the
|
||||
// real tracking store (eg concurrent read only access on a node's datastore)
|
||||
func NewMemTrackingStore() *MemTrackingStore {
|
||||
return &MemTrackingStore{tab: make(map[cid.Cid]abi.ChainEpoch)}
|
||||
}
|
||||
|
||||
// MemTrackingStore is a simple in-memory tracking store
|
||||
type MemTrackingStore struct {
|
||||
sync.Mutex
|
||||
tab map[cid.Cid]abi.ChainEpoch
|
||||
}
|
||||
|
||||
var _ TrackingStore = (*MemTrackingStore)(nil)
|
||||
|
||||
func (s *MemTrackingStore) Put(cid cid.Cid, epoch abi.ChainEpoch) error {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
s.tab[cid] = epoch
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MemTrackingStore) PutBatch(cids []cid.Cid, epoch abi.ChainEpoch) error {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
for _, cid := range cids {
|
||||
s.tab[cid] = epoch
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MemTrackingStore) Get(cid cid.Cid) (abi.ChainEpoch, error) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
epoch, ok := s.tab[cid]
|
||||
if ok {
|
||||
return epoch, nil
|
||||
}
|
||||
return 0, xerrors.Errorf("missing tracking epoch for %s", cid)
|
||||
}
|
||||
|
||||
func (s *MemTrackingStore) Delete(cid cid.Cid) error {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
delete(s.tab, cid)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MemTrackingStore) DeleteBatch(cids []cid.Cid) error {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
for _, cid := range cids {
|
||||
delete(s.tab, cid)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MemTrackingStore) ForEach(f func(cid.Cid, abi.ChainEpoch) error) error {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
for cid, epoch := range s.tab {
|
||||
err := f(cid, epoch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MemTrackingStore) Sync() error { return nil }
|
||||
func (s *MemTrackingStore) Close() error { return nil }
|
120
blockstore/splitstore/tracking_bolt.go
Normal file
120
blockstore/splitstore/tracking_bolt.go
Normal file
@ -0,0 +1,120 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
cid "github.com/ipfs/go-cid"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
)
|
||||
|
||||
type BoltTrackingStore struct {
|
||||
db *bolt.DB
|
||||
bucketId []byte
|
||||
}
|
||||
|
||||
var _ TrackingStore = (*BoltTrackingStore)(nil)
|
||||
|
||||
func OpenBoltTrackingStore(path string) (*BoltTrackingStore, error) {
|
||||
opts := &bolt.Options{
|
||||
Timeout: 1 * time.Second,
|
||||
NoSync: true,
|
||||
}
|
||||
db, err := bolt.Open(path, 0644, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
bucketId := []byte("tracker")
|
||||
err = db.Update(func(tx *bolt.Tx) error {
|
||||
_, err := tx.CreateBucketIfNotExists(bucketId)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error creating bolt db bucket %s: %w", string(bucketId), err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
_ = db.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &BoltTrackingStore{db: db, bucketId: bucketId}, nil
|
||||
}
|
||||
|
||||
func (s *BoltTrackingStore) Put(cid cid.Cid, epoch abi.ChainEpoch) error {
|
||||
val := epochToBytes(epoch)
|
||||
return s.db.Batch(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket(s.bucketId)
|
||||
return b.Put(cid.Hash(), val)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *BoltTrackingStore) PutBatch(cids []cid.Cid, epoch abi.ChainEpoch) error {
|
||||
val := epochToBytes(epoch)
|
||||
return s.db.Batch(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket(s.bucketId)
|
||||
for _, cid := range cids {
|
||||
err := b.Put(cid.Hash(), val)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (s *BoltTrackingStore) Get(cid cid.Cid) (epoch abi.ChainEpoch, err error) {
|
||||
err = s.db.View(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket(s.bucketId)
|
||||
val := b.Get(cid.Hash())
|
||||
if val == nil {
|
||||
return xerrors.Errorf("missing tracking epoch for %s", cid)
|
||||
}
|
||||
epoch = bytesToEpoch(val)
|
||||
return nil
|
||||
})
|
||||
return epoch, err
|
||||
}
|
||||
|
||||
func (s *BoltTrackingStore) Delete(cid cid.Cid) error {
|
||||
return s.db.Batch(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket(s.bucketId)
|
||||
return b.Delete(cid.Hash())
|
||||
})
|
||||
}
|
||||
|
||||
func (s *BoltTrackingStore) DeleteBatch(cids []cid.Cid) error {
|
||||
return s.db.Batch(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket(s.bucketId)
|
||||
for _, cid := range cids {
|
||||
err := b.Delete(cid.Hash())
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error deleting %s", cid)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (s *BoltTrackingStore) ForEach(f func(cid.Cid, abi.ChainEpoch) error) error {
|
||||
return s.db.View(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket(s.bucketId)
|
||||
return b.ForEach(func(k, v []byte) error {
|
||||
cid := cid.NewCidV1(cid.Raw, k)
|
||||
epoch := bytesToEpoch(v)
|
||||
return f(cid, epoch)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func (s *BoltTrackingStore) Sync() error {
|
||||
return s.db.Sync()
|
||||
}
|
||||
|
||||
func (s *BoltTrackingStore) Close() error {
|
||||
return s.db.Close()
|
||||
}
|
130
blockstore/splitstore/tracking_test.go
Normal file
130
blockstore/splitstore/tracking_test.go
Normal file
@ -0,0 +1,130 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
|
||||
cid "github.com/ipfs/go-cid"
|
||||
"github.com/multiformats/go-multihash"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
)
|
||||
|
||||
func TestBoltTrackingStore(t *testing.T) {
|
||||
testTrackingStore(t, "bolt")
|
||||
}
|
||||
|
||||
func testTrackingStore(t *testing.T, tsType string) {
|
||||
t.Helper()
|
||||
|
||||
makeCid := func(key string) cid.Cid {
|
||||
h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return cid.NewCidV1(cid.Raw, h)
|
||||
}
|
||||
|
||||
mustHave := func(s TrackingStore, cid cid.Cid, epoch abi.ChainEpoch) {
|
||||
val, err := s.Get(cid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if val != epoch {
|
||||
t.Fatal("epoch mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
mustNotHave := func(s TrackingStore, cid cid.Cid) {
|
||||
_, err := s.Get(cid)
|
||||
if err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
path, err := ioutil.TempDir("", "snoop-test.*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s, err := OpenTrackingStore(path, tsType)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
k1 := makeCid("a")
|
||||
k2 := makeCid("b")
|
||||
k3 := makeCid("c")
|
||||
k4 := makeCid("d")
|
||||
|
||||
s.Put(k1, 1) //nolint
|
||||
s.Put(k2, 2) //nolint
|
||||
s.Put(k3, 3) //nolint
|
||||
s.Put(k4, 4) //nolint
|
||||
|
||||
mustHave(s, k1, 1)
|
||||
mustHave(s, k2, 2)
|
||||
mustHave(s, k3, 3)
|
||||
mustHave(s, k4, 4)
|
||||
|
||||
s.Delete(k1) // nolint
|
||||
s.Delete(k2) // nolint
|
||||
|
||||
mustNotHave(s, k1)
|
||||
mustNotHave(s, k2)
|
||||
mustHave(s, k3, 3)
|
||||
mustHave(s, k4, 4)
|
||||
|
||||
s.PutBatch([]cid.Cid{k1}, 1) //nolint
|
||||
s.PutBatch([]cid.Cid{k2}, 2) //nolint
|
||||
|
||||
mustHave(s, k1, 1)
|
||||
mustHave(s, k2, 2)
|
||||
mustHave(s, k3, 3)
|
||||
mustHave(s, k4, 4)
|
||||
|
||||
allKeys := map[string]struct{}{
|
||||
k1.String(): {},
|
||||
k2.String(): {},
|
||||
k3.String(): {},
|
||||
k4.String(): {},
|
||||
}
|
||||
|
||||
err = s.ForEach(func(k cid.Cid, _ abi.ChainEpoch) error {
|
||||
_, ok := allKeys[k.String()]
|
||||
if !ok {
|
||||
t.Fatal("unexpected key")
|
||||
}
|
||||
|
||||
delete(allKeys, k.String())
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(allKeys) != 0 {
|
||||
t.Fatal("not all keys were returned")
|
||||
}
|
||||
|
||||
// no close and reopen and ensure the keys still exist
|
||||
err = s.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s, err = OpenTrackingStore(path, tsType)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
mustHave(s, k1, 1)
|
||||
mustHave(s, k2, 2)
|
||||
mustHave(s, k3, 3)
|
||||
mustHave(s, k4, 4)
|
||||
|
||||
s.Close() //nolint:errcheck
|
||||
}
|
81
blockstore/sync.go
Normal file
81
blockstore/sync.go
Normal file
@ -0,0 +1,81 @@
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
// NewMemorySync returns a thread-safe in-memory blockstore.
|
||||
func NewMemorySync() *SyncBlockstore {
|
||||
return &SyncBlockstore{bs: make(MemBlockstore)}
|
||||
}
|
||||
|
||||
// SyncBlockstore is a terminal blockstore that is a synchronized version
|
||||
// of MemBlockstore.
|
||||
type SyncBlockstore struct {
|
||||
mu sync.RWMutex
|
||||
bs MemBlockstore // specifically use a memStore to save indirection overhead.
|
||||
}
|
||||
|
||||
func (m *SyncBlockstore) DeleteBlock(k cid.Cid) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.bs.DeleteBlock(k)
|
||||
}
|
||||
|
||||
func (m *SyncBlockstore) DeleteMany(ks []cid.Cid) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.bs.DeleteMany(ks)
|
||||
}
|
||||
|
||||
func (m *SyncBlockstore) Has(k cid.Cid) (bool, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.bs.Has(k)
|
||||
}
|
||||
|
||||
func (m *SyncBlockstore) View(k cid.Cid, callback func([]byte) error) error {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
return m.bs.View(k, callback)
|
||||
}
|
||||
|
||||
func (m *SyncBlockstore) Get(k cid.Cid) (blocks.Block, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.bs.Get(k)
|
||||
}
|
||||
|
||||
func (m *SyncBlockstore) GetSize(k cid.Cid) (int, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.bs.GetSize(k)
|
||||
}
|
||||
|
||||
func (m *SyncBlockstore) Put(b blocks.Block) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.bs.Put(b)
|
||||
}
|
||||
|
||||
func (m *SyncBlockstore) PutMany(bs []blocks.Block) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.bs.PutMany(bs)
|
||||
}
|
||||
|
||||
func (m *SyncBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
// this blockstore implementation doesn't do any async work.
|
||||
return m.bs.AllKeysChan(ctx)
|
||||
}
|
||||
|
||||
func (m *SyncBlockstore) HashOnRead(enabled bool) {
|
||||
// noop
|
||||
}
|
@ -1,4 +1,4 @@
|
||||
package timedbs
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
@ -10,37 +10,37 @@ import (
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/raulk/clock"
|
||||
"go.uber.org/multierr"
|
||||
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/lib/blockstore"
|
||||
)
|
||||
|
||||
// TimedCacheBS is a blockstore that keeps blocks for at least the specified
|
||||
// caching interval before discarding them. Garbage collection must be started
|
||||
// and stopped by calling Start/Stop.
|
||||
// TimedCacheBlockstore is a blockstore that keeps blocks for at least the
|
||||
// specified caching interval before discarding them. Garbage collection must
|
||||
// be started and stopped by calling Start/Stop.
|
||||
//
|
||||
// Under the covers, it's implemented with an active and an inactive blockstore
|
||||
// that are rotated every cache time interval. This means all blocks will be
|
||||
// stored at most 2x the cache interval.
|
||||
type TimedCacheBS struct {
|
||||
//
|
||||
// Create a new instance by calling the NewTimedCacheBlockstore constructor.
|
||||
type TimedCacheBlockstore struct {
|
||||
mu sync.RWMutex
|
||||
active, inactive blockstore.MemStore
|
||||
active, inactive MemBlockstore
|
||||
clock clock.Clock
|
||||
interval time.Duration
|
||||
closeCh chan struct{}
|
||||
doneRotatingCh chan struct{}
|
||||
}
|
||||
|
||||
func NewTimedCacheBS(cacheTime time.Duration) *TimedCacheBS {
|
||||
return &TimedCacheBS{
|
||||
active: blockstore.NewTemporary(),
|
||||
inactive: blockstore.NewTemporary(),
|
||||
interval: cacheTime,
|
||||
clock: build.Clock,
|
||||
func NewTimedCacheBlockstore(interval time.Duration) *TimedCacheBlockstore {
|
||||
b := &TimedCacheBlockstore{
|
||||
active: NewMemory(),
|
||||
inactive: NewMemory(),
|
||||
interval: interval,
|
||||
clock: clock.New(),
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (t *TimedCacheBS) Start(ctx context.Context) error {
|
||||
func (t *TimedCacheBlockstore) Start(_ context.Context) error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.closeCh != nil {
|
||||
@ -65,11 +65,11 @@ func (t *TimedCacheBS) Start(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *TimedCacheBS) Stop(ctx context.Context) error {
|
||||
func (t *TimedCacheBlockstore) Stop(_ context.Context) error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.closeCh == nil {
|
||||
return fmt.Errorf("not started started")
|
||||
return fmt.Errorf("not started")
|
||||
}
|
||||
select {
|
||||
case <-t.closeCh:
|
||||
@ -80,15 +80,15 @@ func (t *TimedCacheBS) Stop(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *TimedCacheBS) rotate() {
|
||||
newBs := blockstore.NewTemporary()
|
||||
func (t *TimedCacheBlockstore) rotate() {
|
||||
newBs := NewMemory()
|
||||
|
||||
t.mu.Lock()
|
||||
t.inactive, t.active = t.active, newBs
|
||||
t.mu.Unlock()
|
||||
}
|
||||
|
||||
func (t *TimedCacheBS) Put(b blocks.Block) error {
|
||||
func (t *TimedCacheBlockstore) Put(b blocks.Block) error {
|
||||
// Don't check the inactive set here. We want to keep this block for at
|
||||
// least one interval.
|
||||
t.mu.Lock()
|
||||
@ -96,33 +96,43 @@ func (t *TimedCacheBS) Put(b blocks.Block) error {
|
||||
return t.active.Put(b)
|
||||
}
|
||||
|
||||
func (t *TimedCacheBS) PutMany(bs []blocks.Block) error {
|
||||
func (t *TimedCacheBlockstore) PutMany(bs []blocks.Block) error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return t.active.PutMany(bs)
|
||||
}
|
||||
|
||||
func (t *TimedCacheBS) Get(k cid.Cid) (blocks.Block, error) {
|
||||
func (t *TimedCacheBlockstore) View(k cid.Cid, callback func([]byte) error) error {
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
err := t.active.View(k, callback)
|
||||
if err == ErrNotFound {
|
||||
err = t.inactive.View(k, callback)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *TimedCacheBlockstore) Get(k cid.Cid) (blocks.Block, error) {
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
b, err := t.active.Get(k)
|
||||
if err == blockstore.ErrNotFound {
|
||||
if err == ErrNotFound {
|
||||
b, err = t.inactive.Get(k)
|
||||
}
|
||||
return b, err
|
||||
}
|
||||
|
||||
func (t *TimedCacheBS) GetSize(k cid.Cid) (int, error) {
|
||||
func (t *TimedCacheBlockstore) GetSize(k cid.Cid) (int, error) {
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
size, err := t.active.GetSize(k)
|
||||
if err == blockstore.ErrNotFound {
|
||||
if err == ErrNotFound {
|
||||
size, err = t.inactive.GetSize(k)
|
||||
}
|
||||
return size, err
|
||||
}
|
||||
|
||||
func (t *TimedCacheBS) Has(k cid.Cid) (bool, error) {
|
||||
func (t *TimedCacheBlockstore) Has(k cid.Cid) (bool, error) {
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
if has, err := t.active.Has(k); err != nil {
|
||||
@ -133,17 +143,23 @@ func (t *TimedCacheBS) Has(k cid.Cid) (bool, error) {
|
||||
return t.inactive.Has(k)
|
||||
}
|
||||
|
||||
func (t *TimedCacheBS) HashOnRead(_ bool) {
|
||||
func (t *TimedCacheBlockstore) HashOnRead(_ bool) {
|
||||
// no-op
|
||||
}
|
||||
|
||||
func (t *TimedCacheBS) DeleteBlock(k cid.Cid) error {
|
||||
func (t *TimedCacheBlockstore) DeleteBlock(k cid.Cid) error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return multierr.Combine(t.active.DeleteBlock(k), t.inactive.DeleteBlock(k))
|
||||
}
|
||||
|
||||
func (t *TimedCacheBS) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
func (t *TimedCacheBlockstore) DeleteMany(ks []cid.Cid) error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return multierr.Combine(t.active.DeleteMany(ks), t.inactive.DeleteMany(ks))
|
||||
}
|
||||
|
||||
func (t *TimedCacheBlockstore) AllKeysChan(_ context.Context) (<-chan cid.Cid, error) {
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
|
@ -1,4 +1,4 @@
|
||||
package timedbs
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
@ -12,8 +12,8 @@ import (
|
||||
"github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
func TestTimedBSSimple(t *testing.T) {
|
||||
tc := NewTimedCacheBS(10 * time.Millisecond)
|
||||
func TestTimedCacheBlockstoreSimple(t *testing.T) {
|
||||
tc := NewTimedCacheBlockstore(10 * time.Millisecond)
|
||||
mClock := clock.NewMock()
|
||||
mClock.Set(time.Now())
|
||||
tc.clock = mClock
|
119
blockstore/union.go
Normal file
119
blockstore/union.go
Normal file
@ -0,0 +1,119 @@
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
type unionBlockstore []Blockstore
|
||||
|
||||
// Union returns an unioned blockstore.
|
||||
//
|
||||
// * Reads return from the first blockstore that has the value, querying in the
|
||||
// supplied order.
|
||||
// * Writes (puts and deltes) are broadcast to all stores.
|
||||
//
|
||||
func Union(stores ...Blockstore) Blockstore {
|
||||
return unionBlockstore(stores)
|
||||
}
|
||||
|
||||
func (m unionBlockstore) Has(cid cid.Cid) (has bool, err error) {
|
||||
for _, bs := range m {
|
||||
if has, err = bs.Has(cid); has || err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
return has, err
|
||||
}
|
||||
|
||||
func (m unionBlockstore) Get(cid cid.Cid) (blk blocks.Block, err error) {
|
||||
for _, bs := range m {
|
||||
if blk, err = bs.Get(cid); err == nil || err != ErrNotFound {
|
||||
break
|
||||
}
|
||||
}
|
||||
return blk, err
|
||||
}
|
||||
|
||||
func (m unionBlockstore) View(cid cid.Cid, callback func([]byte) error) (err error) {
|
||||
for _, bs := range m {
|
||||
if err = bs.View(cid, callback); err == nil || err != ErrNotFound {
|
||||
break
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (m unionBlockstore) GetSize(cid cid.Cid) (size int, err error) {
|
||||
for _, bs := range m {
|
||||
if size, err = bs.GetSize(cid); err == nil || err != ErrNotFound {
|
||||
break
|
||||
}
|
||||
}
|
||||
return size, err
|
||||
}
|
||||
|
||||
func (m unionBlockstore) Put(block blocks.Block) (err error) {
|
||||
for _, bs := range m {
|
||||
if err = bs.Put(block); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (m unionBlockstore) PutMany(blks []blocks.Block) (err error) {
|
||||
for _, bs := range m {
|
||||
if err = bs.PutMany(blks); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (m unionBlockstore) DeleteBlock(cid cid.Cid) (err error) {
|
||||
for _, bs := range m {
|
||||
if err = bs.DeleteBlock(cid); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (m unionBlockstore) DeleteMany(cids []cid.Cid) (err error) {
|
||||
for _, bs := range m {
|
||||
if err = bs.DeleteMany(cids); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (m unionBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
// this does not deduplicate; this interface needs to be revisited.
|
||||
outCh := make(chan cid.Cid)
|
||||
|
||||
go func() {
|
||||
defer close(outCh)
|
||||
|
||||
for _, bs := range m {
|
||||
ch, err := bs.AllKeysChan(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for cid := range ch {
|
||||
outCh <- cid
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return outCh, nil
|
||||
}
|
||||
|
||||
func (m unionBlockstore) HashOnRead(enabled bool) {
|
||||
for _, bs := range m {
|
||||
bs.HashOnRead(enabled)
|
||||
}
|
||||
}
|
102
blockstore/union_test.go
Normal file
102
blockstore/union_test.go
Normal file
@ -0,0 +1,102 @@
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var (
|
||||
b0 = blocks.NewBlock([]byte("abc"))
|
||||
b1 = blocks.NewBlock([]byte("foo"))
|
||||
b2 = blocks.NewBlock([]byte("bar"))
|
||||
)
|
||||
|
||||
func TestUnionBlockstore_Get(t *testing.T) {
|
||||
m1 := NewMemory()
|
||||
m2 := NewMemory()
|
||||
|
||||
_ = m1.Put(b1)
|
||||
_ = m2.Put(b2)
|
||||
|
||||
u := Union(m1, m2)
|
||||
|
||||
v1, err := u.Get(b1.Cid())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, b1.RawData(), v1.RawData())
|
||||
|
||||
v2, err := u.Get(b2.Cid())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, b2.RawData(), v2.RawData())
|
||||
}
|
||||
|
||||
func TestUnionBlockstore_Put_PutMany_Delete_AllKeysChan(t *testing.T) {
|
||||
m1 := NewMemory()
|
||||
m2 := NewMemory()
|
||||
|
||||
u := Union(m1, m2)
|
||||
|
||||
err := u.Put(b0)
|
||||
require.NoError(t, err)
|
||||
|
||||
var has bool
|
||||
|
||||
// write was broadcasted to all stores.
|
||||
has, _ = m1.Has(b0.Cid())
|
||||
require.True(t, has)
|
||||
|
||||
has, _ = m2.Has(b0.Cid())
|
||||
require.True(t, has)
|
||||
|
||||
has, _ = u.Has(b0.Cid())
|
||||
require.True(t, has)
|
||||
|
||||
// put many.
|
||||
err = u.PutMany([]blocks.Block{b1, b2})
|
||||
require.NoError(t, err)
|
||||
|
||||
// write was broadcasted to all stores.
|
||||
has, _ = m1.Has(b1.Cid())
|
||||
require.True(t, has)
|
||||
|
||||
has, _ = m1.Has(b2.Cid())
|
||||
require.True(t, has)
|
||||
|
||||
has, _ = m2.Has(b1.Cid())
|
||||
require.True(t, has)
|
||||
|
||||
has, _ = m2.Has(b2.Cid())
|
||||
require.True(t, has)
|
||||
|
||||
// also in the union store.
|
||||
has, _ = u.Has(b1.Cid())
|
||||
require.True(t, has)
|
||||
|
||||
has, _ = u.Has(b2.Cid())
|
||||
require.True(t, has)
|
||||
|
||||
// deleted from all stores.
|
||||
err = u.DeleteBlock(b1.Cid())
|
||||
require.NoError(t, err)
|
||||
|
||||
has, _ = u.Has(b1.Cid())
|
||||
require.False(t, has)
|
||||
|
||||
has, _ = m1.Has(b1.Cid())
|
||||
require.False(t, has)
|
||||
|
||||
has, _ = m2.Has(b1.Cid())
|
||||
require.False(t, has)
|
||||
|
||||
// check that AllKeysChan returns b0 and b2, twice (once per backing store)
|
||||
ch, err := u.AllKeysChan(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
var i int
|
||||
for range ch {
|
||||
i++
|
||||
}
|
||||
require.Equal(t, 4, i)
|
||||
}
|
@ -4,5 +4,6 @@ package build
|
||||
|
||||
import (
|
||||
_ "github.com/GeertJohan/go.rice/rice"
|
||||
_ "github.com/golang/mock/mockgen"
|
||||
_ "github.com/whyrusleeping/bencher"
|
||||
)
|
||||
|
@ -1,11 +1,5 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
var CurrentCommit string
|
||||
var BuildType int
|
||||
|
||||
@ -40,67 +34,3 @@ const BuildVersion = "1.5.0"
|
||||
func UserVersion() string {
|
||||
return BuildVersion + buildType() + CurrentCommit
|
||||
}
|
||||
|
||||
type Version uint32
|
||||
|
||||
func newVer(major, minor, patch uint8) Version {
|
||||
return Version(uint32(major)<<16 | uint32(minor)<<8 | uint32(patch))
|
||||
}
|
||||
|
||||
// Ints returns (major, minor, patch) versions
|
||||
func (ve Version) Ints() (uint32, uint32, uint32) {
|
||||
v := uint32(ve)
|
||||
return (v & majorOnlyMask) >> 16, (v & minorOnlyMask) >> 8, v & patchOnlyMask
|
||||
}
|
||||
|
||||
func (ve Version) String() string {
|
||||
vmj, vmi, vp := ve.Ints()
|
||||
return fmt.Sprintf("%d.%d.%d", vmj, vmi, vp)
|
||||
}
|
||||
|
||||
func (ve Version) EqMajorMinor(v2 Version) bool {
|
||||
return ve&minorMask == v2&minorMask
|
||||
}
|
||||
|
||||
type NodeType int
|
||||
|
||||
const (
|
||||
NodeUnknown NodeType = iota
|
||||
|
||||
NodeFull
|
||||
NodeMiner
|
||||
NodeWorker
|
||||
)
|
||||
|
||||
var RunningNodeType NodeType
|
||||
|
||||
func VersionForType(nodeType NodeType) (Version, error) {
|
||||
switch nodeType {
|
||||
case NodeFull:
|
||||
return FullAPIVersion, nil
|
||||
case NodeMiner:
|
||||
return MinerAPIVersion, nil
|
||||
case NodeWorker:
|
||||
return WorkerAPIVersion, nil
|
||||
default:
|
||||
return Version(0), xerrors.Errorf("unknown node type %d", nodeType)
|
||||
}
|
||||
}
|
||||
|
||||
// semver versions of the rpc api exposed
|
||||
var (
|
||||
FullAPIVersion = newVer(1, 1, 0)
|
||||
MinerAPIVersion = newVer(1, 0, 1)
|
||||
WorkerAPIVersion = newVer(1, 0, 0)
|
||||
)
|
||||
|
||||
//nolint:varcheck,deadcode
|
||||
const (
|
||||
majorMask = 0xff0000
|
||||
minorMask = 0xffff00
|
||||
patchMask = 0xffffff
|
||||
|
||||
majorOnlyMask = 0xff0000
|
||||
minorOnlyMask = 0x00ff00
|
||||
patchOnlyMask = 0x0000ff
|
||||
)
|
||||
|
@ -16,7 +16,7 @@ import (
|
||||
builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
|
||||
adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt"
|
||||
|
||||
bstore "github.com/filecoin-project/lotus/lib/blockstore"
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
)
|
||||
|
||||
func TestDiffAdtArray(t *testing.T) {
|
||||
@ -295,7 +295,7 @@ func (t *TestDiffArray) Remove(key uint64, val *typegen.Deferred) error {
|
||||
|
||||
func newContextStore() Store {
|
||||
ctx := context.Background()
|
||||
bs := bstore.NewTemporarySync()
|
||||
bs := bstore.NewMemorySync()
|
||||
store := cbornode.NewCborStore(bs)
|
||||
return WrapStore(ctx, store)
|
||||
}
|
||||
|
@ -5,8 +5,8 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/lib/blockstore"
|
||||
"github.com/ipfs/go-cid"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
@ -3,6 +3,7 @@ package state
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
@ -10,7 +11,7 @@ import (
|
||||
"github.com/filecoin-project/go-state-types/big"
|
||||
cbor "github.com/ipfs/go-ipld-cbor"
|
||||
|
||||
"github.com/filecoin-project/lotus/api/apibstore"
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/chain/actors/adt"
|
||||
init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/market"
|
||||
@ -23,7 +24,7 @@ type UserData interface{}
|
||||
|
||||
// ChainAPI abstracts out calls made by this class to external APIs
|
||||
type ChainAPI interface {
|
||||
apibstore.ChainIO
|
||||
api.ChainIO
|
||||
StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error)
|
||||
}
|
||||
|
||||
@ -36,7 +37,7 @@ type StatePredicates struct {
|
||||
func NewStatePredicates(api ChainAPI) *StatePredicates {
|
||||
return &StatePredicates{
|
||||
api: api,
|
||||
cst: cbor.NewCborStore(apibstore.NewAPIBlockstore(api)),
|
||||
cst: cbor.NewCborStore(blockstore.NewAPIBlockstore(api)),
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -23,9 +23,9 @@ import (
|
||||
adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt"
|
||||
tutils "github.com/filecoin-project/specs-actors/v2/support/testing"
|
||||
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/market"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
bstore "github.com/filecoin-project/lotus/lib/blockstore"
|
||||
)
|
||||
|
||||
var dummyCid cid.Cid
|
||||
@ -36,7 +36,7 @@ func init() {
|
||||
|
||||
func TestMarketPredicates(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
bs := bstore.NewTemporarySync()
|
||||
bs := bstore.NewMemorySync()
|
||||
store := adt2.WrapStore(ctx, cbornode.NewCborStore(bs))
|
||||
|
||||
oldDeal1 := &market2.DealState{
|
||||
@ -334,7 +334,7 @@ func TestMarketPredicates(t *testing.T) {
|
||||
|
||||
func TestMinerSectorChange(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
bs := bstore.NewTemporarySync()
|
||||
bs := bstore.NewMemorySync()
|
||||
store := adt2.WrapStore(ctx, cbornode.NewCborStore(bs))
|
||||
|
||||
nextID := uint64(0)
|
||||
|
@ -27,6 +27,7 @@ import (
|
||||
proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/actors/policy"
|
||||
"github.com/filecoin-project/lotus/chain/beacon"
|
||||
@ -40,7 +41,6 @@ import (
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
|
||||
"github.com/filecoin-project/lotus/genesis"
|
||||
"github.com/filecoin-project/lotus/journal"
|
||||
"github.com/filecoin-project/lotus/lib/blockstore"
|
||||
"github.com/filecoin-project/lotus/lib/sigs"
|
||||
"github.com/filecoin-project/lotus/node/repo"
|
||||
)
|
||||
@ -125,7 +125,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
|
||||
return nil, xerrors.Errorf("failed to get metadata datastore: %w", err)
|
||||
}
|
||||
|
||||
bs, err := lr.Blockstore(context.TODO(), repo.BlockstoreChain)
|
||||
bs, err := lr.Blockstore(context.TODO(), repo.UniversalBlockstore)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -8,8 +8,8 @@ import (
|
||||
"github.com/filecoin-project/specs-actors/actors/builtin"
|
||||
cbor "github.com/ipfs/go-ipld-cbor"
|
||||
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
bstore "github.com/filecoin-project/lotus/lib/blockstore"
|
||||
)
|
||||
|
||||
func SetupSystemActor(bs bstore.Blockstore) (*types.Actor, error) {
|
||||
|
@ -16,9 +16,9 @@ import (
|
||||
cbg "github.com/whyrusleeping/cbor-gen"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/genesis"
|
||||
bstore "github.com/filecoin-project/lotus/lib/blockstore"
|
||||
)
|
||||
|
||||
func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesis.Actor, rootVerifier genesis.Actor, remainder genesis.Actor) (int64, *types.Actor, map[address.Address]address.Address, error) {
|
||||
|
@ -9,9 +9,9 @@ import (
|
||||
reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward"
|
||||
cbor "github.com/ipfs/go-ipld-cbor"
|
||||
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
bstore "github.com/filecoin-project/lotus/lib/blockstore"
|
||||
)
|
||||
|
||||
func SetupRewardActor(bs bstore.Blockstore, qaPower big.Int) (*types.Actor, error) {
|
||||
|
@ -7,8 +7,8 @@ import (
|
||||
"github.com/filecoin-project/specs-actors/actors/builtin/cron"
|
||||
cbor "github.com/ipfs/go-ipld-cbor"
|
||||
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
bstore "github.com/filecoin-project/lotus/lib/blockstore"
|
||||
)
|
||||
|
||||
func SetupCronActor(bs bstore.Blockstore) (*types.Actor, error) {
|
||||
|
@ -9,8 +9,8 @@ import (
|
||||
power0 "github.com/filecoin-project/specs-actors/actors/builtin/power"
|
||||
cbor "github.com/ipfs/go-ipld-cbor"
|
||||
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
bstore "github.com/filecoin-project/lotus/lib/blockstore"
|
||||
)
|
||||
|
||||
func SetupStoragePowerActor(bs bstore.Blockstore) (*types.Actor, error) {
|
||||
|
@ -8,8 +8,8 @@ import (
|
||||
"github.com/filecoin-project/specs-actors/actors/util/adt"
|
||||
cbor "github.com/ipfs/go-ipld-cbor"
|
||||
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
bstore "github.com/filecoin-project/lotus/lib/blockstore"
|
||||
)
|
||||
|
||||
func SetupStorageMarketActor(bs bstore.Blockstore) (*types.Actor, error) {
|
||||
|
@ -10,8 +10,8 @@ import (
|
||||
verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
|
||||
"github.com/filecoin-project/specs-actors/actors/util/adt"
|
||||
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
bstore "github.com/filecoin-project/lotus/lib/blockstore"
|
||||
)
|
||||
|
||||
var RootVerifierID address.Address
|
||||
|
@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin"
|
||||
|
||||
"github.com/filecoin-project/lotus/journal"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
@ -26,13 +27,13 @@ import (
|
||||
verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
|
||||
adt0 "github.com/filecoin-project/specs-actors/actors/util/adt"
|
||||
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/state"
|
||||
"github.com/filecoin-project/lotus/chain/store"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/chain/vm"
|
||||
"github.com/filecoin-project/lotus/genesis"
|
||||
bstore "github.com/filecoin-project/lotus/lib/blockstore"
|
||||
"github.com/filecoin-project/lotus/lib/sigs"
|
||||
)
|
||||
|
||||
@ -233,13 +234,36 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
|
||||
|
||||
}
|
||||
|
||||
vregroot, err := address.NewIDAddress(80)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
switch template.VerifregRootKey.Type {
|
||||
case genesis.TAccount:
|
||||
var ainfo genesis.AccountMeta
|
||||
if err := json.Unmarshal(template.VerifregRootKey.Meta, &ainfo); err != nil {
|
||||
return nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
|
||||
}
|
||||
st, err := cst.Put(ctx, &account0.State{Address: ainfo.Owner})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if err = createMultisigAccount(ctx, bs, cst, state, vregroot, template.VerifregRootKey, keyIDs); err != nil {
|
||||
return nil, nil, xerrors.Errorf("failed to set up verified registry signer: %w", err)
|
||||
_, ok := keyIDs[ainfo.Owner]
|
||||
if ok {
|
||||
return nil, nil, fmt.Errorf("rootkey account has already been declared, cannot be assigned 80: %s", ainfo.Owner)
|
||||
}
|
||||
|
||||
err = state.SetActor(builtin.RootVerifierAddress, &types.Actor{
|
||||
Code: builtin0.AccountActorCodeID,
|
||||
Balance: template.VerifregRootKey.Balance,
|
||||
Head: st,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, xerrors.Errorf("setting verifreg rootkey account: %w", err)
|
||||
}
|
||||
case genesis.TMultisig:
|
||||
if err = createMultisigAccount(ctx, bs, cst, state, builtin.RootVerifierAddress, template.VerifregRootKey, keyIDs); err != nil {
|
||||
return nil, nil, xerrors.Errorf("failed to set up verified registry signer: %w", err)
|
||||
}
|
||||
default:
|
||||
return nil, nil, xerrors.Errorf("unknown account type for verifreg rootkey: %w", err)
|
||||
}
|
||||
|
||||
// Setup the first verifier as ID-address 81
|
||||
@ -300,8 +324,36 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
|
||||
|
||||
template.RemainderAccount.Balance = remainingFil
|
||||
|
||||
if err := createMultisigAccount(ctx, bs, cst, state, builtin.ReserveAddress, template.RemainderAccount, keyIDs); err != nil {
|
||||
return nil, nil, xerrors.Errorf("failed to set up remainder account: %w", err)
|
||||
switch template.RemainderAccount.Type {
|
||||
case genesis.TAccount:
|
||||
var ainfo genesis.AccountMeta
|
||||
if err := json.Unmarshal(template.RemainderAccount.Meta, &ainfo); err != nil {
|
||||
return nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
|
||||
}
|
||||
st, err := cst.Put(ctx, &account0.State{Address: ainfo.Owner})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
_, ok := keyIDs[ainfo.Owner]
|
||||
if ok {
|
||||
return nil, nil, fmt.Errorf("remainder account has already been declared, cannot be assigned 90: %s", ainfo.Owner)
|
||||
}
|
||||
|
||||
err = state.SetActor(builtin.ReserveAddress, &types.Actor{
|
||||
Code: builtin0.AccountActorCodeID,
|
||||
Balance: template.RemainderAccount.Balance,
|
||||
Head: st,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, xerrors.Errorf("setting remainder account: %w", err)
|
||||
}
|
||||
case genesis.TMultisig:
|
||||
if err = createMultisigAccount(ctx, bs, cst, state, builtin.ReserveAddress, template.RemainderAccount, keyIDs); err != nil {
|
||||
return nil, nil, xerrors.Errorf("failed to set up remainder: %w", err)
|
||||
}
|
||||
default:
|
||||
return nil, nil, xerrors.Errorf("unknown account type for remainder: %w", err)
|
||||
}
|
||||
|
||||
return state, keyIDs, nil
|
||||
@ -406,7 +458,7 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci
|
||||
StateBase: stateroot,
|
||||
Epoch: 0,
|
||||
Rand: &fakeRand{},
|
||||
Bstore: cs.Blockstore(),
|
||||
Bstore: cs.StateBlockstore(),
|
||||
Syscalls: mkFakedSigSyscalls(cs.VMSys()),
|
||||
CircSupplyCalc: nil,
|
||||
NtwkVersion: genesisNetworkVersion,
|
||||
|
@ -70,7 +70,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
|
||||
StateBase: sroot,
|
||||
Epoch: 0,
|
||||
Rand: &fakeRand{},
|
||||
Bstore: cs.Blockstore(),
|
||||
Bstore: cs.StateBlockstore(),
|
||||
Syscalls: mkFakedSigSyscalls(cs.VMSys()),
|
||||
CircSupplyCalc: csc,
|
||||
NtwkVersion: genesisNetworkVersion,
|
||||
|
@ -79,7 +79,7 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w api.WalletA
|
||||
}
|
||||
}
|
||||
|
||||
store := sm.ChainStore().Store(ctx)
|
||||
store := sm.ChainStore().ActorStore(ctx)
|
||||
blsmsgroot, err := toArray(store, blsMsgCids)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("building bls amt: %w", err)
|
||||
|
@ -59,7 +59,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
|
||||
StateBase: bstate,
|
||||
Epoch: bheight,
|
||||
Rand: store.NewChainRand(sm.cs, ts.Cids()),
|
||||
Bstore: sm.cs.Blockstore(),
|
||||
Bstore: sm.cs.StateBlockstore(),
|
||||
Syscalls: sm.cs.VMSys(),
|
||||
CircSupplyCalc: sm.GetVMCirculatingSupply,
|
||||
NtwkVersion: sm.GetNtwkVersion,
|
||||
@ -174,7 +174,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
|
||||
StateBase: state,
|
||||
Epoch: ts.Height() + 1,
|
||||
Rand: r,
|
||||
Bstore: sm.cs.Blockstore(),
|
||||
Bstore: sm.cs.StateBlockstore(),
|
||||
Syscalls: sm.cs.VMSys(),
|
||||
CircSupplyCalc: sm.GetVMCirculatingSupply,
|
||||
NtwkVersion: sm.GetNtwkVersion,
|
||||
|
@ -15,6 +15,7 @@ import (
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/big"
|
||||
"github.com/filecoin-project/go-state-types/network"
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/actors/adt"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin"
|
||||
@ -24,8 +25,6 @@ import (
|
||||
"github.com/filecoin-project/lotus/chain/store"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/chain/vm"
|
||||
bstore "github.com/filecoin-project/lotus/lib/blockstore"
|
||||
"github.com/filecoin-project/lotus/lib/bufbstore"
|
||||
builtin0 "github.com/filecoin-project/specs-actors/actors/builtin"
|
||||
miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
|
||||
multisig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig"
|
||||
@ -505,7 +504,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
|
||||
}
|
||||
case builtin0.StorageMinerActorCodeID:
|
||||
var st miner0.State
|
||||
if err := sm.ChainStore().Store(ctx).Get(ctx, act.Head, &st); err != nil {
|
||||
if err := sm.ChainStore().ActorStore(ctx).Get(ctx, act.Head, &st); err != nil {
|
||||
return xerrors.Errorf("failed to load miner state: %w", err)
|
||||
}
|
||||
|
||||
@ -549,7 +548,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
|
||||
return cid.Undef, xerrors.Errorf("failed to load power actor: %w", err)
|
||||
}
|
||||
|
||||
cst := cbor.NewCborStore(sm.ChainStore().Blockstore())
|
||||
cst := cbor.NewCborStore(sm.ChainStore().StateBlockstore())
|
||||
if err := cst.Get(ctx, powAct.Head, &ps); err != nil {
|
||||
return cid.Undef, xerrors.Errorf("failed to get power actor state: %w", err)
|
||||
}
|
||||
@ -583,7 +582,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
|
||||
}
|
||||
case builtin0.StorageMinerActorCodeID:
|
||||
var st miner0.State
|
||||
if err := sm.ChainStore().Store(ctx).Get(ctx, act.Head, &st); err != nil {
|
||||
if err := sm.ChainStore().ActorStore(ctx).Get(ctx, act.Head, &st); err != nil {
|
||||
return xerrors.Errorf("failed to load miner state: %w", err)
|
||||
}
|
||||
|
||||
@ -592,7 +591,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
|
||||
return xerrors.Errorf("failed to get miner info: %w", err)
|
||||
}
|
||||
|
||||
sectorsArr, err := adt0.AsArray(sm.ChainStore().Store(ctx), st.Sectors)
|
||||
sectorsArr, err := adt0.AsArray(sm.ChainStore().ActorStore(ctx), st.Sectors)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to load sectors array: %w", err)
|
||||
}
|
||||
@ -612,11 +611,11 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
|
||||
lbact, err := lbtree.GetActor(addr)
|
||||
if err == nil {
|
||||
var lbst miner0.State
|
||||
if err := sm.ChainStore().Store(ctx).Get(ctx, lbact.Head, &lbst); err != nil {
|
||||
if err := sm.ChainStore().ActorStore(ctx).Get(ctx, lbact.Head, &lbst); err != nil {
|
||||
return xerrors.Errorf("failed to load miner state: %w", err)
|
||||
}
|
||||
|
||||
lbsectors, err := adt0.AsArray(sm.ChainStore().Store(ctx), lbst.Sectors)
|
||||
lbsectors, err := adt0.AsArray(sm.ChainStore().ActorStore(ctx), lbst.Sectors)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to load lb sectors array: %w", err)
|
||||
}
|
||||
@ -712,7 +711,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
|
||||
}
|
||||
|
||||
func UpgradeIgnition(ctx context.Context, sm *StateManager, _ MigrationCache, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
|
||||
store := sm.cs.Store(ctx)
|
||||
store := sm.cs.ActorStore(ctx)
|
||||
|
||||
if build.UpgradeLiftoffHeight <= epoch {
|
||||
return cid.Undef, xerrors.Errorf("liftoff height must be beyond ignition height")
|
||||
@ -768,7 +767,7 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, _ MigrationCache, cb
|
||||
|
||||
func UpgradeRefuel(ctx context.Context, sm *StateManager, _ MigrationCache, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
|
||||
|
||||
store := sm.cs.Store(ctx)
|
||||
store := sm.cs.ActorStore(ctx)
|
||||
tree, err := sm.StateTree(root)
|
||||
if err != nil {
|
||||
return cid.Undef, xerrors.Errorf("getting state tree: %w", err)
|
||||
@ -793,7 +792,7 @@ func UpgradeRefuel(ctx context.Context, sm *StateManager, _ MigrationCache, cb E
|
||||
}
|
||||
|
||||
func UpgradeActorsV2(ctx context.Context, sm *StateManager, _ MigrationCache, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
|
||||
buf := bufbstore.NewTieredBstore(sm.cs.Blockstore(), bstore.NewTemporarySync())
|
||||
buf := blockstore.NewTieredBstore(sm.cs.StateBlockstore(), blockstore.NewMemorySync())
|
||||
store := store.ActorStore(ctx, buf)
|
||||
|
||||
info, err := store.Put(ctx, new(types.StateInfo0))
|
||||
@ -844,7 +843,7 @@ func UpgradeLiftoff(ctx context.Context, sm *StateManager, _ MigrationCache, cb
|
||||
return cid.Undef, xerrors.Errorf("getting state tree: %w", err)
|
||||
}
|
||||
|
||||
err = setNetworkName(ctx, sm.cs.Store(ctx), tree, "mainnet")
|
||||
err = setNetworkName(ctx, sm.cs.ActorStore(ctx), tree, "mainnet")
|
||||
if err != nil {
|
||||
return cid.Undef, xerrors.Errorf("setting network name: %w", err)
|
||||
}
|
||||
@ -853,7 +852,7 @@ func UpgradeLiftoff(ctx context.Context, sm *StateManager, _ MigrationCache, cb
|
||||
}
|
||||
|
||||
func UpgradeCalico(ctx context.Context, sm *StateManager, _ MigrationCache, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
|
||||
store := sm.cs.Store(ctx)
|
||||
store := sm.cs.ActorStore(ctx)
|
||||
var stateRoot types.StateRoot
|
||||
if err := store.Get(ctx, root, &stateRoot); err != nil {
|
||||
return cid.Undef, xerrors.Errorf("failed to decode state root: %w", err)
|
||||
@ -1010,7 +1009,7 @@ func upgradeActorsV3Common(
|
||||
root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet,
|
||||
config nv10.Config,
|
||||
) (cid.Cid, error) {
|
||||
buf := bufbstore.NewTieredBstore(sm.cs.Blockstore(), bstore.NewTemporarySync())
|
||||
buf := blockstore.NewTieredBstore(sm.cs.StateBlockstore(), blockstore.NewMemorySync())
|
||||
store := store.ActorStore(ctx, buf)
|
||||
|
||||
// Load the state root.
|
||||
@ -1240,7 +1239,7 @@ func resetGenesisMsigs0(ctx context.Context, sm *StateManager, store adt0.Store,
|
||||
return xerrors.Errorf("getting genesis tipset: %w", err)
|
||||
}
|
||||
|
||||
cst := cbor.NewCborStore(sm.cs.Blockstore())
|
||||
cst := cbor.NewCborStore(sm.cs.StateBlockstore())
|
||||
genesisTree, err := state.LoadStateTree(cst, gts.ParentState())
|
||||
if err != nil {
|
||||
return xerrors.Errorf("loading state tree: %w", err)
|
||||
|
@ -125,7 +125,7 @@ func TestForkHeightTriggers(t *testing.T) {
|
||||
Height: testForkHeight,
|
||||
Migration: func(ctx context.Context, sm *StateManager, cache MigrationCache, cb ExecCallback,
|
||||
root cid.Cid, height abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
|
||||
cst := ipldcbor.NewCborStore(sm.ChainStore().Blockstore())
|
||||
cst := ipldcbor.NewCborStore(sm.ChainStore().StateBlockstore())
|
||||
|
||||
st, err := sm.StateTree(root)
|
||||
if err != nil {
|
||||
|
@ -22,7 +22,7 @@ func (sm *StateManager) ParentStateTsk(tsk types.TipSetKey) (*state.StateTree, e
|
||||
}
|
||||
|
||||
func (sm *StateManager) ParentState(ts *types.TipSet) (*state.StateTree, error) {
|
||||
cst := cbor.NewCborStore(sm.cs.Blockstore())
|
||||
cst := cbor.NewCborStore(sm.cs.StateBlockstore())
|
||||
state, err := state.LoadStateTree(cst, sm.parentState(ts))
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("load state tree: %w", err)
|
||||
@ -32,7 +32,7 @@ func (sm *StateManager) ParentState(ts *types.TipSet) (*state.StateTree, error)
|
||||
}
|
||||
|
||||
func (sm *StateManager) StateTree(st cid.Cid) (*state.StateTree, error) {
|
||||
cst := cbor.NewCborStore(sm.cs.Blockstore())
|
||||
cst := cbor.NewCborStore(sm.cs.StateBlockstore())
|
||||
state, err := state.LoadStateTree(cst, st)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("load state tree: %w", err)
|
||||
|
@ -286,7 +286,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp
|
||||
StateBase: base,
|
||||
Epoch: epoch,
|
||||
Rand: r,
|
||||
Bstore: sm.cs.Blockstore(),
|
||||
Bstore: sm.cs.StateBlockstore(),
|
||||
Syscalls: sm.cs.VMSys(),
|
||||
CircSupplyCalc: sm.GetVMCirculatingSupply,
|
||||
NtwkVersion: sm.GetNtwkVersion,
|
||||
@ -430,7 +430,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp
|
||||
return cid.Cid{}, cid.Cid{}, err
|
||||
}
|
||||
|
||||
rectarr := blockadt.MakeEmptyArray(sm.cs.Store(ctx))
|
||||
rectarr := blockadt.MakeEmptyArray(sm.cs.ActorStore(ctx))
|
||||
for i, receipt := range receipts {
|
||||
if err := rectarr.Set(uint64(i), receipt); err != nil {
|
||||
return cid.Undef, cid.Undef, xerrors.Errorf("failed to build receipts amt: %w", err)
|
||||
@ -515,7 +515,7 @@ func (sm *StateManager) ResolveToKeyAddress(ctx context.Context, addr address.Ad
|
||||
ts = sm.cs.GetHeaviestTipSet()
|
||||
}
|
||||
|
||||
cst := cbor.NewCborStore(sm.cs.Blockstore())
|
||||
cst := cbor.NewCborStore(sm.cs.StateBlockstore())
|
||||
|
||||
// First try to resolve the actor in the parent state, so we don't have to compute anything.
|
||||
tree, err := state.LoadStateTree(cst, ts.ParentState())
|
||||
@ -556,7 +556,7 @@ func (sm *StateManager) GetBlsPublicKey(ctx context.Context, addr address.Addres
|
||||
}
|
||||
|
||||
func (sm *StateManager) LookupID(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) {
|
||||
cst := cbor.NewCborStore(sm.cs.Blockstore())
|
||||
cst := cbor.NewCborStore(sm.cs.StateBlockstore())
|
||||
state, err := state.LoadStateTree(cst, sm.parentState(ts))
|
||||
if err != nil {
|
||||
return address.Undef, xerrors.Errorf("load state tree: %w", err)
|
||||
@ -882,7 +882,7 @@ func (sm *StateManager) MarketBalance(ctx context.Context, addr address.Address,
|
||||
return api.MarketBalance{}, err
|
||||
}
|
||||
|
||||
mstate, err := market.Load(sm.cs.Store(ctx), act)
|
||||
mstate, err := market.Load(sm.cs.ActorStore(ctx), act)
|
||||
if err != nil {
|
||||
return api.MarketBalance{}, err
|
||||
}
|
||||
@ -966,7 +966,7 @@ func (sm *StateManager) setupGenesisVestingSchedule(ctx context.Context) error {
|
||||
return xerrors.Errorf("getting genesis tipset state: %w", err)
|
||||
}
|
||||
|
||||
cst := cbor.NewCborStore(sm.cs.Blockstore())
|
||||
cst := cbor.NewCborStore(sm.cs.StateBlockstore())
|
||||
sTree, err := state.LoadStateTree(cst, st)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("loading state tree: %w", err)
|
||||
@ -1325,7 +1325,7 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha
|
||||
unCirc = big.Add(unCirc, actor.Balance)
|
||||
|
||||
case a == market.Address:
|
||||
mst, err := market.Load(sm.cs.Store(ctx), actor)
|
||||
mst, err := market.Load(sm.cs.ActorStore(ctx), actor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -1342,7 +1342,7 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha
|
||||
circ = big.Add(circ, actor.Balance)
|
||||
|
||||
case builtin.IsStorageMinerActor(actor.Code):
|
||||
mst, err := miner.Load(sm.cs.Store(ctx), actor)
|
||||
mst, err := miner.Load(sm.cs.ActorStore(ctx), actor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -1359,7 +1359,7 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha
|
||||
}
|
||||
|
||||
case builtin.IsMultisigActor(actor.Code):
|
||||
mst, err := multisig.Load(sm.cs.Store(ctx), actor)
|
||||
mst, err := multisig.Load(sm.cs.ActorStore(ctx), actor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -1413,7 +1413,7 @@ func (sm *StateManager) GetPaychState(ctx context.Context, addr address.Address,
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
actState, err := paych.Load(sm.cs.Store(ctx), act)
|
||||
actState, err := paych.Load(sm.cs.ActorStore(ctx), act)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@ -1431,7 +1431,7 @@ func (sm *StateManager) GetMarketState(ctx context.Context, ts *types.TipSet) (m
|
||||
return nil, err
|
||||
}
|
||||
|
||||
actState, err := market.Load(sm.cs.Store(ctx), act)
|
||||
actState, err := market.Load(sm.cs.ActorStore(ctx), act)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -48,7 +48,7 @@ func GetNetworkName(ctx context.Context, sm *StateManager, st cid.Cid) (dtypes.N
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
ias, err := init_.Load(sm.cs.Store(ctx), act)
|
||||
ias, err := init_.Load(sm.cs.ActorStore(ctx), act)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@ -65,7 +65,7 @@ func GetMinerWorkerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr
|
||||
if err != nil {
|
||||
return address.Undef, xerrors.Errorf("(get sset) failed to load miner actor: %w", err)
|
||||
}
|
||||
mas, err := miner.Load(sm.cs.Store(ctx), act)
|
||||
mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
|
||||
if err != nil {
|
||||
return address.Undef, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err)
|
||||
}
|
||||
@ -75,7 +75,7 @@ func GetMinerWorkerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr
|
||||
return address.Undef, xerrors.Errorf("failed to load actor info: %w", err)
|
||||
}
|
||||
|
||||
return vm.ResolveToKeyAddr(state, sm.cs.Store(ctx), info.Worker)
|
||||
return vm.ResolveToKeyAddr(state, sm.cs.ActorStore(ctx), info.Worker)
|
||||
}
|
||||
|
||||
func GetPower(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (power.Claim, power.Claim, bool, error) {
|
||||
@ -88,7 +88,7 @@ func GetPowerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr addres
|
||||
return power.Claim{}, power.Claim{}, false, xerrors.Errorf("(get sset) failed to load power actor state: %w", err)
|
||||
}
|
||||
|
||||
pas, err := power.Load(sm.cs.Store(ctx), act)
|
||||
pas, err := power.Load(sm.cs.ActorStore(ctx), act)
|
||||
if err != nil {
|
||||
return power.Claim{}, power.Claim{}, false, err
|
||||
}
|
||||
@ -123,7 +123,7 @@ func PreCommitInfo(ctx context.Context, sm *StateManager, maddr address.Address,
|
||||
return nil, xerrors.Errorf("(get sset) failed to load miner actor: %w", err)
|
||||
}
|
||||
|
||||
mas, err := miner.Load(sm.cs.Store(ctx), act)
|
||||
mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err)
|
||||
}
|
||||
@ -137,7 +137,7 @@ func MinerSectorInfo(ctx context.Context, sm *StateManager, maddr address.Addres
|
||||
return nil, xerrors.Errorf("(get sset) failed to load miner actor: %w", err)
|
||||
}
|
||||
|
||||
mas, err := miner.Load(sm.cs.Store(ctx), act)
|
||||
mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err)
|
||||
}
|
||||
@ -151,7 +151,7 @@ func GetSectorsForWinningPoSt(ctx context.Context, nv network.Version, pv ffiwra
|
||||
return nil, xerrors.Errorf("failed to load miner actor: %w", err)
|
||||
}
|
||||
|
||||
mas, err := miner.Load(sm.cs.Store(ctx), act)
|
||||
mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("failed to load miner actor state: %w", err)
|
||||
}
|
||||
@ -249,7 +249,7 @@ func GetMinerSlashed(ctx context.Context, sm *StateManager, ts *types.TipSet, ma
|
||||
return false, xerrors.Errorf("failed to load power actor: %w", err)
|
||||
}
|
||||
|
||||
spas, err := power.Load(sm.cs.Store(ctx), act)
|
||||
spas, err := power.Load(sm.cs.ActorStore(ctx), act)
|
||||
if err != nil {
|
||||
return false, xerrors.Errorf("failed to load power actor state: %w", err)
|
||||
}
|
||||
@ -272,7 +272,7 @@ func GetStorageDeal(ctx context.Context, sm *StateManager, dealID abi.DealID, ts
|
||||
return nil, xerrors.Errorf("failed to load market actor: %w", err)
|
||||
}
|
||||
|
||||
state, err := market.Load(sm.cs.Store(ctx), act)
|
||||
state, err := market.Load(sm.cs.ActorStore(ctx), act)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("failed to load market actor state: %w", err)
|
||||
}
|
||||
@ -320,7 +320,7 @@ func ListMinerActors(ctx context.Context, sm *StateManager, ts *types.TipSet) ([
|
||||
return nil, xerrors.Errorf("failed to load power actor: %w", err)
|
||||
}
|
||||
|
||||
powState, err := power.Load(sm.cs.Store(ctx), act)
|
||||
powState, err := power.Load(sm.cs.ActorStore(ctx), act)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("failed to load power actor state: %w", err)
|
||||
}
|
||||
@ -353,7 +353,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
|
||||
StateBase: base,
|
||||
Epoch: height,
|
||||
Rand: r,
|
||||
Bstore: sm.cs.Blockstore(),
|
||||
Bstore: sm.cs.StateBlockstore(),
|
||||
Syscalls: sm.cs.VMSys(),
|
||||
CircSupplyCalc: sm.GetVMCirculatingSupply,
|
||||
NtwkVersion: sm.GetNtwkVersion,
|
||||
@ -474,7 +474,7 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule
|
||||
return nil, xerrors.Errorf("failed to load miner actor: %w", err)
|
||||
}
|
||||
|
||||
mas, err := miner.Load(sm.cs.Store(ctx), act)
|
||||
mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("failed to load miner actor state: %w", err)
|
||||
}
|
||||
@ -623,7 +623,7 @@ func minerHasMinPower(ctx context.Context, sm *StateManager, addr address.Addres
|
||||
return false, xerrors.Errorf("loading power actor state: %w", err)
|
||||
}
|
||||
|
||||
ps, err := power.Load(sm.cs.Store(ctx), pact)
|
||||
ps, err := power.Load(sm.cs.ActorStore(ctx), pact)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@ -654,7 +654,7 @@ func MinerEligibleToMine(ctx context.Context, sm *StateManager, addr address.Add
|
||||
return false, xerrors.Errorf("loading power actor state: %w", err)
|
||||
}
|
||||
|
||||
pstate, err := power.Load(sm.cs.Store(ctx), pact)
|
||||
pstate, err := power.Load(sm.cs.ActorStore(ctx), pact)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@ -664,7 +664,7 @@ func MinerEligibleToMine(ctx context.Context, sm *StateManager, addr address.Add
|
||||
return false, xerrors.Errorf("loading miner actor state: %w", err)
|
||||
}
|
||||
|
||||
mstate, err := miner.Load(sm.cs.Store(ctx), mact)
|
||||
mstate, err := miner.Load(sm.cs.ActorStore(ctx), mact)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@ -696,7 +696,7 @@ func MinerEligibleToMine(ctx context.Context, sm *StateManager, addr address.Add
|
||||
}
|
||||
|
||||
func CheckTotalFIL(ctx context.Context, sm *StateManager, ts *types.TipSet) (abi.TokenAmount, error) {
|
||||
str, err := state.LoadStateTree(sm.ChainStore().Store(ctx), ts.ParentState())
|
||||
str, err := state.LoadStateTree(sm.ChainStore().ActorStore(ctx), ts.ParentState())
|
||||
if err != nil {
|
||||
return abi.TokenAmount{}, err
|
||||
}
|
||||
|
@ -6,10 +6,10 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/chain/gen"
|
||||
"github.com/filecoin-project/lotus/chain/store"
|
||||
"github.com/filecoin-project/lotus/chain/types/mock"
|
||||
"github.com/filecoin-project/lotus/lib/blockstore"
|
||||
datastore "github.com/ipfs/go-datastore"
|
||||
syncds "github.com/ipfs/go-datastore/sync"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@ -30,7 +30,7 @@ func TestIndexSeeks(t *testing.T) {
|
||||
|
||||
ctx := context.TODO()
|
||||
|
||||
nbs := blockstore.NewTemporarySync()
|
||||
nbs := blockstore.NewMemorySync()
|
||||
cs := store.NewChainStore(nbs, nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil, nil)
|
||||
defer cs.Close() //nolint:errcheck
|
||||
|
||||
|
@ -23,12 +23,12 @@ import (
|
||||
blockadt "github.com/filecoin-project/specs-actors/actors/util/adt"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/actors/adt"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin"
|
||||
"github.com/filecoin-project/lotus/chain/vm"
|
||||
"github.com/filecoin-project/lotus/journal"
|
||||
bstore "github.com/filecoin-project/lotus/lib/blockstore"
|
||||
"github.com/filecoin-project/lotus/metrics"
|
||||
|
||||
"go.opencensus.io/stats"
|
||||
@ -81,7 +81,7 @@ func init() {
|
||||
}
|
||||
|
||||
// ReorgNotifee represents a callback that gets called upon reorgs.
|
||||
type ReorgNotifee func(rev, app []*types.TipSet) error
|
||||
type ReorgNotifee = func(rev, app []*types.TipSet) error
|
||||
|
||||
// Journal event types.
|
||||
const (
|
||||
@ -107,11 +107,11 @@ type HeadChangeEvt struct {
|
||||
// 1. a tipset cache
|
||||
// 2. a block => messages references cache.
|
||||
type ChainStore struct {
|
||||
bs bstore.Blockstore
|
||||
localbs bstore.Blockstore
|
||||
ds dstore.Batching
|
||||
chainBlockstore bstore.Blockstore
|
||||
stateBlockstore bstore.Blockstore
|
||||
metadataDs dstore.Batching
|
||||
|
||||
localviewer bstore.Viewer
|
||||
chainLocalBlockstore bstore.Blockstore
|
||||
|
||||
heaviestLk sync.Mutex
|
||||
heaviest *types.TipSet
|
||||
@ -139,30 +139,29 @@ type ChainStore struct {
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// localbs is guaranteed to fail Get* if requested block isn't stored locally
|
||||
func NewChainStore(bs bstore.Blockstore, localbs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder, j journal.Journal) *ChainStore {
|
||||
mmCache, _ := lru.NewARC(DefaultMsgMetaCacheSize)
|
||||
tsCache, _ := lru.NewARC(DefaultTipSetCacheSize)
|
||||
func NewChainStore(chainBs bstore.Blockstore, stateBs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder, j journal.Journal) *ChainStore {
|
||||
c, _ := lru.NewARC(DefaultMsgMetaCacheSize)
|
||||
tsc, _ := lru.NewARC(DefaultTipSetCacheSize)
|
||||
if j == nil {
|
||||
j = journal.NilJournal()
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
// unwraps the fallback store in case one is configured.
|
||||
// some methods _need_ to operate on a local blockstore only.
|
||||
localbs, _ := bstore.UnwrapFallbackStore(chainBs)
|
||||
cs := &ChainStore{
|
||||
bs: bs,
|
||||
localbs: localbs,
|
||||
ds: ds,
|
||||
bestTips: pubsub.New(64),
|
||||
tipsets: make(map[abi.ChainEpoch][]cid.Cid),
|
||||
mmCache: mmCache,
|
||||
tsCache: tsCache,
|
||||
vmcalls: vmcalls,
|
||||
cancelFn: cancel,
|
||||
journal: j,
|
||||
}
|
||||
|
||||
if v, ok := localbs.(bstore.Viewer); ok {
|
||||
cs.localviewer = v
|
||||
chainBlockstore: chainBs,
|
||||
stateBlockstore: stateBs,
|
||||
chainLocalBlockstore: localbs,
|
||||
metadataDs: ds,
|
||||
bestTips: pubsub.New(64),
|
||||
tipsets: make(map[abi.ChainEpoch][]cid.Cid),
|
||||
mmCache: c,
|
||||
tsCache: tsc,
|
||||
vmcalls: vmcalls,
|
||||
cancelFn: cancel,
|
||||
journal: j,
|
||||
}
|
||||
|
||||
cs.evtTypes = [1]journal.EventType{
|
||||
@ -216,7 +215,7 @@ func (cs *ChainStore) Close() error {
|
||||
}
|
||||
|
||||
func (cs *ChainStore) Load() error {
|
||||
head, err := cs.ds.Get(chainHeadKey)
|
||||
head, err := cs.metadataDs.Get(chainHeadKey)
|
||||
if err == dstore.ErrNotFound {
|
||||
log.Warn("no previous chain state found")
|
||||
return nil
|
||||
@ -246,7 +245,7 @@ func (cs *ChainStore) writeHead(ts *types.TipSet) error {
|
||||
return xerrors.Errorf("failed to marshal tipset: %w", err)
|
||||
}
|
||||
|
||||
if err := cs.ds.Put(chainHeadKey, data); err != nil {
|
||||
if err := cs.metadataDs.Put(chainHeadKey, data); err != nil {
|
||||
return xerrors.Errorf("failed to write chain head to datastore: %w", err)
|
||||
}
|
||||
|
||||
@ -306,13 +305,13 @@ func (cs *ChainStore) SubscribeHeadChanges(f ReorgNotifee) {
|
||||
func (cs *ChainStore) IsBlockValidated(ctx context.Context, blkid cid.Cid) (bool, error) {
|
||||
key := blockValidationCacheKeyPrefix.Instance(blkid.String())
|
||||
|
||||
return cs.ds.Has(key)
|
||||
return cs.metadataDs.Has(key)
|
||||
}
|
||||
|
||||
func (cs *ChainStore) MarkBlockAsValidated(ctx context.Context, blkid cid.Cid) error {
|
||||
key := blockValidationCacheKeyPrefix.Instance(blkid.String())
|
||||
|
||||
if err := cs.ds.Put(key, []byte{0}); err != nil {
|
||||
if err := cs.metadataDs.Put(key, []byte{0}); err != nil {
|
||||
return xerrors.Errorf("cache block validation: %w", err)
|
||||
}
|
||||
|
||||
@ -322,7 +321,7 @@ func (cs *ChainStore) MarkBlockAsValidated(ctx context.Context, blkid cid.Cid) e
|
||||
func (cs *ChainStore) UnmarkBlockAsValidated(ctx context.Context, blkid cid.Cid) error {
|
||||
key := blockValidationCacheKeyPrefix.Instance(blkid.String())
|
||||
|
||||
if err := cs.ds.Delete(key); err != nil {
|
||||
if err := cs.metadataDs.Delete(key); err != nil {
|
||||
return xerrors.Errorf("removing from valid block cache: %w", err)
|
||||
}
|
||||
|
||||
@ -339,7 +338,7 @@ func (cs *ChainStore) SetGenesis(b *types.BlockHeader) error {
|
||||
return err
|
||||
}
|
||||
|
||||
return cs.ds.Put(dstore.NewKey("0"), b.Cid().Bytes())
|
||||
return cs.metadataDs.Put(dstore.NewKey("0"), b.Cid().Bytes())
|
||||
}
|
||||
|
||||
func (cs *ChainStore) PutTipSet(ctx context.Context, ts *types.TipSet) error {
|
||||
@ -594,7 +593,7 @@ func (cs *ChainStore) takeHeaviestTipSet(ctx context.Context, ts *types.TipSet)
|
||||
// FlushValidationCache removes all results of block validation from the
|
||||
// chain metadata store. Usually the first step after a new chain import.
|
||||
func (cs *ChainStore) FlushValidationCache() error {
|
||||
return FlushValidationCache(cs.ds)
|
||||
return FlushValidationCache(cs.metadataDs)
|
||||
}
|
||||
|
||||
func FlushValidationCache(ds datastore.Batching) error {
|
||||
@ -653,7 +652,7 @@ func (cs *ChainStore) SetHead(ts *types.TipSet) error {
|
||||
// Contains returns whether our BlockStore has all blocks in the supplied TipSet.
|
||||
func (cs *ChainStore) Contains(ts *types.TipSet) (bool, error) {
|
||||
for _, c := range ts.Cids() {
|
||||
has, err := cs.bs.Has(c)
|
||||
has, err := cs.chainBlockstore.Has(c)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@ -668,16 +667,8 @@ func (cs *ChainStore) Contains(ts *types.TipSet) (bool, error) {
|
||||
// GetBlock fetches a BlockHeader with the supplied CID. It returns
|
||||
// blockstore.ErrNotFound if the block was not found in the BlockStore.
|
||||
func (cs *ChainStore) GetBlock(c cid.Cid) (*types.BlockHeader, error) {
|
||||
if cs.localviewer == nil {
|
||||
sb, err := cs.localbs.Get(c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return types.DecodeBlock(sb.RawData())
|
||||
}
|
||||
|
||||
var blk *types.BlockHeader
|
||||
err := cs.localviewer.View(c, func(b []byte) (err error) {
|
||||
err := cs.chainLocalBlockstore.View(c, func(b []byte) (err error) {
|
||||
blk, err = types.DecodeBlock(b)
|
||||
return err
|
||||
})
|
||||
@ -851,7 +842,7 @@ func (cs *ChainStore) PersistBlockHeaders(b ...*types.BlockHeader) error {
|
||||
end = len(b)
|
||||
}
|
||||
|
||||
err = multierr.Append(err, cs.bs.PutMany(sbs[start:end]))
|
||||
err = multierr.Append(err, cs.chainLocalBlockstore.PutMany(sbs[start:end]))
|
||||
}
|
||||
|
||||
return err
|
||||
@ -875,7 +866,7 @@ func PutMessage(bs bstore.Blockstore, m storable) (cid.Cid, error) {
|
||||
}
|
||||
|
||||
func (cs *ChainStore) PutMessage(m storable) (cid.Cid, error) {
|
||||
return PutMessage(cs.bs, m)
|
||||
return PutMessage(cs.chainBlockstore, m)
|
||||
}
|
||||
|
||||
func (cs *ChainStore) expandTipset(b *types.BlockHeader) (*types.TipSet, error) {
|
||||
@ -936,7 +927,7 @@ func (cs *ChainStore) AddBlock(ctx context.Context, b *types.BlockHeader) error
|
||||
}
|
||||
|
||||
func (cs *ChainStore) GetGenesis() (*types.BlockHeader, error) {
|
||||
data, err := cs.ds.Get(dstore.NewKey("0"))
|
||||
data, err := cs.metadataDs.Get(dstore.NewKey("0"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -962,17 +953,8 @@ func (cs *ChainStore) GetCMessage(c cid.Cid) (types.ChainMsg, error) {
|
||||
}
|
||||
|
||||
func (cs *ChainStore) GetMessage(c cid.Cid) (*types.Message, error) {
|
||||
if cs.localviewer == nil {
|
||||
sb, err := cs.localbs.Get(c)
|
||||
if err != nil {
|
||||
log.Errorf("get message get failed: %s: %s", c, err)
|
||||
return nil, err
|
||||
}
|
||||
return types.DecodeMessage(sb.RawData())
|
||||
}
|
||||
|
||||
var msg *types.Message
|
||||
err := cs.localviewer.View(c, func(b []byte) (err error) {
|
||||
err := cs.chainLocalBlockstore.View(c, func(b []byte) (err error) {
|
||||
msg, err = types.DecodeMessage(b)
|
||||
return err
|
||||
})
|
||||
@ -980,17 +962,8 @@ func (cs *ChainStore) GetMessage(c cid.Cid) (*types.Message, error) {
|
||||
}
|
||||
|
||||
func (cs *ChainStore) GetSignedMessage(c cid.Cid) (*types.SignedMessage, error) {
|
||||
if cs.localviewer == nil {
|
||||
sb, err := cs.localbs.Get(c)
|
||||
if err != nil {
|
||||
log.Errorf("get message get failed: %s: %s", c, err)
|
||||
return nil, err
|
||||
}
|
||||
return types.DecodeSignedMessage(sb.RawData())
|
||||
}
|
||||
|
||||
var msg *types.SignedMessage
|
||||
err := cs.localviewer.View(c, func(b []byte) (err error) {
|
||||
err := cs.chainLocalBlockstore.View(c, func(b []byte) (err error) {
|
||||
msg, err = types.DecodeSignedMessage(b)
|
||||
return err
|
||||
})
|
||||
@ -1000,7 +973,7 @@ func (cs *ChainStore) GetSignedMessage(c cid.Cid) (*types.SignedMessage, error)
|
||||
func (cs *ChainStore) readAMTCids(root cid.Cid) ([]cid.Cid, error) {
|
||||
ctx := context.TODO()
|
||||
// block headers use adt0, for now.
|
||||
a, err := blockadt.AsArray(cs.Store(ctx), root)
|
||||
a, err := blockadt.AsArray(cs.ActorStore(ctx), root)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("amt load: %w", err)
|
||||
}
|
||||
@ -1124,7 +1097,7 @@ func (cs *ChainStore) ReadMsgMetaCids(mmc cid.Cid) ([]cid.Cid, []cid.Cid, error)
|
||||
return mmcids.bls, mmcids.secpk, nil
|
||||
}
|
||||
|
||||
cst := cbor.NewCborStore(cs.localbs)
|
||||
cst := cbor.NewCborStore(cs.chainLocalBlockstore)
|
||||
var msgmeta types.MsgMeta
|
||||
if err := cst.Get(context.TODO(), mmc, &msgmeta); err != nil {
|
||||
return nil, nil, xerrors.Errorf("failed to load msgmeta (%s): %w", mmc, err)
|
||||
@ -1194,7 +1167,7 @@ func (cs *ChainStore) MessagesForBlock(b *types.BlockHeader) ([]*types.Message,
|
||||
func (cs *ChainStore) GetParentReceipt(b *types.BlockHeader, i int) (*types.MessageReceipt, error) {
|
||||
ctx := context.TODO()
|
||||
// block headers use adt0, for now.
|
||||
a, err := blockadt.AsArray(cs.Store(ctx), b.ParentMessageReceipts)
|
||||
a, err := blockadt.AsArray(cs.ActorStore(ctx), b.ParentMessageReceipts)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("amt load: %w", err)
|
||||
}
|
||||
@ -1237,16 +1210,26 @@ func (cs *ChainStore) LoadSignedMessagesFromCids(cids []cid.Cid) ([]*types.Signe
|
||||
return msgs, nil
|
||||
}
|
||||
|
||||
func (cs *ChainStore) Blockstore() bstore.Blockstore {
|
||||
return cs.bs
|
||||
// ChainBlockstore returns the chain blockstore. Currently the chain and state
|
||||
// // stores are both backed by the same physical store, albeit with different
|
||||
// // caching policies, but in the future they will segregate.
|
||||
func (cs *ChainStore) ChainBlockstore() bstore.Blockstore {
|
||||
return cs.chainBlockstore
|
||||
}
|
||||
|
||||
// StateBlockstore returns the state blockstore. Currently the chain and state
|
||||
// stores are both backed by the same physical store, albeit with different
|
||||
// caching policies, but in the future they will segregate.
|
||||
func (cs *ChainStore) StateBlockstore() bstore.Blockstore {
|
||||
return cs.stateBlockstore
|
||||
}
|
||||
|
||||
func ActorStore(ctx context.Context, bs bstore.Blockstore) adt.Store {
|
||||
return adt.WrapStore(ctx, cbor.NewCborStore(bs))
|
||||
}
|
||||
|
||||
func (cs *ChainStore) Store(ctx context.Context) adt.Store {
|
||||
return ActorStore(ctx, cs.bs)
|
||||
func (cs *ChainStore) ActorStore(ctx context.Context) adt.Store {
|
||||
return ActorStore(ctx, cs.stateBlockstore)
|
||||
}
|
||||
|
||||
func (cs *ChainStore) VMSys() vm.SyscallBuilder {
|
||||
@ -1444,8 +1427,9 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo
|
||||
return xerrors.Errorf("failed to write car header: %s", err)
|
||||
}
|
||||
|
||||
return cs.WalkSnapshot(ctx, ts, inclRecentRoots, skipOldMsgs, func(c cid.Cid) error {
|
||||
blk, err := cs.bs.Get(c)
|
||||
unionBs := bstore.Union(cs.stateBlockstore, cs.chainBlockstore)
|
||||
return cs.WalkSnapshot(ctx, ts, inclRecentRoots, skipOldMsgs, true, func(c cid.Cid) error {
|
||||
blk, err := unionBs.Get(c)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("writing object to car, bs.Get: %w", err)
|
||||
}
|
||||
@ -1458,7 +1442,7 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo
|
||||
})
|
||||
}
|
||||
|
||||
func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, skipOldMsgs bool, cb func(cid.Cid) error) error {
|
||||
func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, skipOldMsgs, skipMsgReceipts bool, cb func(cid.Cid) error) error {
|
||||
if ts == nil {
|
||||
ts = cs.GetHeaviestTipSet()
|
||||
}
|
||||
@ -1478,7 +1462,7 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
|
||||
return err
|
||||
}
|
||||
|
||||
data, err := cs.bs.Get(blk)
|
||||
data, err := cs.chainBlockstore.Get(blk)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("getting block: %w", err)
|
||||
}
|
||||
@ -1498,7 +1482,7 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
|
||||
var cids []cid.Cid
|
||||
if !skipOldMsgs || b.Height > ts.Height()-inclRecentRoots {
|
||||
if walked.Visit(b.Messages) {
|
||||
mcids, err := recurseLinks(cs.bs, walked, b.Messages, []cid.Cid{b.Messages})
|
||||
mcids, err := recurseLinks(cs.chainBlockstore, walked, b.Messages, []cid.Cid{b.Messages})
|
||||
if err != nil {
|
||||
return xerrors.Errorf("recursing messages failed: %w", err)
|
||||
}
|
||||
@ -1519,13 +1503,17 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
|
||||
|
||||
if b.Height == 0 || b.Height > ts.Height()-inclRecentRoots {
|
||||
if walked.Visit(b.ParentStateRoot) {
|
||||
cids, err := recurseLinks(cs.bs, walked, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot})
|
||||
cids, err := recurseLinks(cs.stateBlockstore, walked, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot})
|
||||
if err != nil {
|
||||
return xerrors.Errorf("recursing genesis state failed: %w", err)
|
||||
}
|
||||
|
||||
out = append(out, cids...)
|
||||
}
|
||||
|
||||
if !skipMsgReceipts && walked.Visit(b.ParentMessageReceipts) {
|
||||
out = append(out, b.ParentMessageReceipts)
|
||||
}
|
||||
}
|
||||
|
||||
for _, c := range out {
|
||||
@ -1561,7 +1549,12 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
|
||||
}
|
||||
|
||||
func (cs *ChainStore) Import(r io.Reader) (*types.TipSet, error) {
|
||||
header, err := car.LoadCar(cs.Blockstore(), r)
|
||||
// TODO: writing only to the state blockstore is incorrect.
|
||||
// At this time, both the state and chain blockstores are backed by the
|
||||
// universal store. When we physically segregate the stores, we will need
|
||||
// to route state objects to the state blockstore, and chain objects to
|
||||
// the chain blockstore.
|
||||
header, err := car.LoadCar(cs.StateBlockstore(), r)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("loadcar failed: %w", err)
|
||||
}
|
||||
|
@ -11,12 +11,12 @@ import (
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/crypto"
|
||||
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/chain/actors/policy"
|
||||
"github.com/filecoin-project/lotus/chain/gen"
|
||||
"github.com/filecoin-project/lotus/chain/stmgr"
|
||||
"github.com/filecoin-project/lotus/chain/store"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/lib/blockstore"
|
||||
"github.com/filecoin-project/lotus/node/repo"
|
||||
)
|
||||
|
||||
@ -52,7 +52,7 @@ func BenchmarkGetRandomness(b *testing.B) {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
bs, err := lr.Blockstore(context.TODO(), repo.BlockstoreChain)
|
||||
bs, err := lr.Blockstore(context.TODO(), repo.UniversalBlockstore)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@ -104,7 +104,7 @@ func TestChainExportImport(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
nbs := blockstore.NewTemporary()
|
||||
nbs := blockstore.NewMemory()
|
||||
cs := store.NewChainStore(nbs, nbs, datastore.NewMapDatastore(), nil, nil)
|
||||
defer cs.Close() //nolint:errcheck
|
||||
|
||||
@ -139,7 +139,7 @@ func TestChainExportImportFull(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
nbs := blockstore.NewTemporary()
|
||||
nbs := blockstore.NewMemory()
|
||||
cs := store.NewChainStore(nbs, nbs, datastore.NewMapDatastore(), nil, nil)
|
||||
defer cs.Close() //nolint:errcheck
|
||||
|
||||
|
@ -28,7 +28,7 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn
|
||||
|
||||
tpow := big2.Zero()
|
||||
{
|
||||
cst := cbor.NewCborStore(cs.Blockstore())
|
||||
cst := cbor.NewCborStore(cs.StateBlockstore())
|
||||
state, err := state.LoadStateTree(cst, ts.ParentState())
|
||||
if err != nil {
|
||||
return types.NewInt(0), xerrors.Errorf("load state tree: %w", err)
|
||||
@ -39,7 +39,7 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn
|
||||
return types.NewInt(0), xerrors.Errorf("get power actor: %w", err)
|
||||
}
|
||||
|
||||
powState, err := power.Load(cs.Store(ctx), act)
|
||||
powState, err := power.Load(cs.ActorStore(ctx), act)
|
||||
if err != nil {
|
||||
return types.NewInt(0), xerrors.Errorf("failed to load power actor state: %w", err)
|
||||
}
|
||||
|
@ -7,13 +7,13 @@ import (
|
||||
"time"
|
||||
|
||||
address "github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain"
|
||||
"github.com/filecoin-project/lotus/chain/messagepool"
|
||||
"github.com/filecoin-project/lotus/chain/stmgr"
|
||||
"github.com/filecoin-project/lotus/chain/store"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/lib/blockstore"
|
||||
"github.com/filecoin-project/lotus/lib/sigs"
|
||||
"github.com/filecoin-project/lotus/metrics"
|
||||
"github.com/filecoin-project/lotus/node/impl/client"
|
||||
@ -101,7 +101,7 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha
|
||||
[]tag.Mutator{tag.Insert(metrics.MinerID, blk.Header.Miner.String())},
|
||||
metrics.BlockDelay.M(delay),
|
||||
)
|
||||
log.Warnf("Received block with large delay %d from miner %s", delay, blk.Header.Miner)
|
||||
log.Warnw("received block with large delay from miner", "block", blk.Cid(), "delay", delay, "miner", blk.Header.Miner)
|
||||
}
|
||||
|
||||
if s.InformNewBlock(msg.ReceivedFrom, &types.FullBlock{
|
||||
@ -392,7 +392,7 @@ func (bv *BlockValidator) isChainNearSynced() bool {
|
||||
func (bv *BlockValidator) validateMsgMeta(ctx context.Context, msg *types.BlockMsg) error {
|
||||
// TODO there has to be a simpler way to do this without the blockstore dance
|
||||
// block headers use adt0
|
||||
store := blockadt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewTemporary()))
|
||||
store := blockadt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewMemory()))
|
||||
bmArr := blockadt.MakeEmptyArray(store)
|
||||
smArr := blockadt.MakeEmptyArray(store)
|
||||
|
||||
|
@ -44,6 +44,7 @@ import (
|
||||
proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/power"
|
||||
"github.com/filecoin-project/lotus/chain/beacon"
|
||||
@ -54,7 +55,6 @@ import (
|
||||
"github.com/filecoin-project/lotus/chain/store"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/chain/vm"
|
||||
bstore "github.com/filecoin-project/lotus/lib/blockstore"
|
||||
"github.com/filecoin-project/lotus/lib/sigs"
|
||||
"github.com/filecoin-project/lotus/metrics"
|
||||
)
|
||||
@ -321,7 +321,7 @@ func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error {
|
||||
|
||||
// We use a temporary bstore here to avoid writing intermediate pieces
|
||||
// into the blockstore.
|
||||
blockstore := bstore.NewTemporary()
|
||||
blockstore := bstore.NewMemory()
|
||||
cst := cbor.NewCborStore(blockstore)
|
||||
|
||||
var bcids, scids []cid.Cid
|
||||
@ -354,7 +354,7 @@ func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error {
|
||||
}
|
||||
|
||||
// Finally, flush.
|
||||
return vm.Copy(context.TODO(), blockstore, syncer.store.Blockstore(), smroot)
|
||||
return vm.Copy(context.TODO(), blockstore, syncer.store.ChainBlockstore(), smroot)
|
||||
}
|
||||
|
||||
func (syncer *Syncer) LocalPeer() peer.ID {
|
||||
@ -640,7 +640,7 @@ func (syncer *Syncer) minerIsValid(ctx context.Context, maddr address.Address, b
|
||||
return xerrors.Errorf("failed to load power actor: %w", err)
|
||||
}
|
||||
|
||||
powState, err := power.Load(syncer.store.Store(ctx), act)
|
||||
powState, err := power.Load(syncer.store.ActorStore(ctx), act)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to load power actor state: %w", err)
|
||||
}
|
||||
@ -1055,7 +1055,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock
|
||||
return err
|
||||
}
|
||||
|
||||
st, err := state.LoadStateTree(syncer.store.Store(ctx), stateroot)
|
||||
st, err := state.LoadStateTree(syncer.store.ActorStore(ctx), stateroot)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to load base state tree: %w", err)
|
||||
}
|
||||
@ -1102,7 +1102,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock
|
||||
}
|
||||
|
||||
// Validate message arrays in a temporary blockstore.
|
||||
tmpbs := bstore.NewTemporary()
|
||||
tmpbs := bstore.NewMemory()
|
||||
tmpstore := blockadt.WrapStore(ctx, cbor.NewCborStore(tmpbs))
|
||||
|
||||
bmArr := blockadt.MakeEmptyArray(tmpstore)
|
||||
@ -1172,7 +1172,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock
|
||||
}
|
||||
|
||||
// Finally, flush.
|
||||
return vm.Copy(ctx, tmpbs, syncer.store.Blockstore(), mrcid)
|
||||
return vm.Copy(ctx, tmpbs, syncer.store.ChainBlockstore(), mrcid)
|
||||
}
|
||||
|
||||
func (syncer *Syncer) verifyBlsAggregate(ctx context.Context, sig *crypto.Signature, msgs []cid.Cid, pubks [][]byte) error {
|
||||
@ -1553,7 +1553,7 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS
|
||||
|
||||
for bsi := 0; bsi < len(bstout); bsi++ {
|
||||
// temp storage so we don't persist data we dont want to
|
||||
bs := bstore.NewTemporary()
|
||||
bs := bstore.NewMemory()
|
||||
blks := cbor.NewCborStore(bs)
|
||||
|
||||
this := headers[i-bsi]
|
||||
@ -1574,7 +1574,7 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS
|
||||
return err
|
||||
}
|
||||
|
||||
if err := copyBlockstore(ctx, bs, syncer.store.Blockstore()); err != nil {
|
||||
if err := copyBlockstore(ctx, bs, syncer.store.ChainBlockstore()); err != nil {
|
||||
return xerrors.Errorf("message processing failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
@ -28,6 +28,7 @@ import (
|
||||
"github.com/filecoin-project/go-state-types/exitcode"
|
||||
"github.com/filecoin-project/go-state-types/network"
|
||||
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/actors/adt"
|
||||
"github.com/filecoin-project/lotus/chain/actors/aerrors"
|
||||
@ -36,9 +37,6 @@ import (
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/reward"
|
||||
"github.com/filecoin-project/lotus/chain/state"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/lib/blockstore"
|
||||
bstore "github.com/filecoin-project/lotus/lib/blockstore"
|
||||
"github.com/filecoin-project/lotus/lib/bufbstore"
|
||||
)
|
||||
|
||||
const MaxCallDepth = 4096
|
||||
@ -208,7 +206,7 @@ type VM struct {
|
||||
cstate *state.StateTree
|
||||
base cid.Cid
|
||||
cst *cbor.BasicIpldStore
|
||||
buf *bufbstore.BufferedBS
|
||||
buf *blockstore.BufferedBlockstore
|
||||
blockHeight abi.ChainEpoch
|
||||
areg *ActorRegistry
|
||||
rand Rand
|
||||
@ -224,7 +222,7 @@ type VMOpts struct {
|
||||
StateBase cid.Cid
|
||||
Epoch abi.ChainEpoch
|
||||
Rand Rand
|
||||
Bstore bstore.Blockstore
|
||||
Bstore blockstore.Blockstore
|
||||
Syscalls SyscallBuilder
|
||||
CircSupplyCalc CircSupplyCalculator
|
||||
NtwkVersion NtwkVersionGetter // TODO: stebalien: In what cases do we actually need this? It seems like even when creating new networks we want to use the 'global'/build-default version getter
|
||||
@ -233,7 +231,7 @@ type VMOpts struct {
|
||||
}
|
||||
|
||||
func NewVM(ctx context.Context, opts *VMOpts) (*VM, error) {
|
||||
buf := bufbstore.NewBufferedBstore(opts.Bstore)
|
||||
buf := blockstore.NewBuffered(opts.Bstore)
|
||||
cst := cbor.NewCborStore(buf)
|
||||
state, err := state.LoadStateTree(cst, opts.StateBase)
|
||||
if err != nil {
|
||||
|
@ -9,6 +9,7 @@ import (
|
||||
"github.com/filecoin-project/go-jsonrpc/auth"
|
||||
|
||||
"github.com/filecoin-project/lotus/api/apistruct"
|
||||
cliutil "github.com/filecoin-project/lotus/cli/util"
|
||||
"github.com/filecoin-project/lotus/node/repo"
|
||||
)
|
||||
|
||||
@ -127,7 +128,7 @@ var authApiInfoToken = &cli.Command{
|
||||
|
||||
// TODO: Log in audit log when it is implemented
|
||||
|
||||
fmt.Printf("%s=%s:%s\n", envForRepo(t), string(token), ainfo.Addr)
|
||||
fmt.Printf("%s=%s:%s\n", cliutil.EnvForRepo(t), string(token), ainfo.Addr)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
@ -51,7 +51,10 @@ func BackupCmd(repoFlag string, rt repo.RepoType, getApi BackupApiFn) *cli.Comma
|
||||
return xerrors.Errorf("getting metadata datastore: %w", err)
|
||||
}
|
||||
|
||||
bds := backupds.Wrap(mds)
|
||||
bds, err := backupds.Wrap(mds, backupds.NoLogdir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fpath, err := homedir.Expand(cctx.Args().First())
|
||||
if err != nil {
|
||||
|
@ -642,7 +642,10 @@ var chainListCmd = &cli.Command{
|
||||
gasUsed += r.GasUsed
|
||||
}
|
||||
|
||||
fmt.Printf("\ttipset: \t%d msgs, %d / %d (%0.2f%%)\n", len(msgs), gasUsed, limitSum, 100*float64(gasUsed)/float64(limitSum))
|
||||
gasEfficiency := 100 * float64(gasUsed) / float64(limitSum)
|
||||
gasCapacity := 100 * float64(limitSum) / float64(build.BlockGasLimit)
|
||||
|
||||
fmt.Printf("\ttipset: \t%d msgs, %d (%0.2f%%) / %d (%0.2f%%)\n", len(msgs), gasUsed, gasEfficiency, limitSum, gasCapacity)
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
|
@ -1195,6 +1195,11 @@ var clientListAsksCmd = &cli.Command{
|
||||
&cli.BoolFlag{
|
||||
Name: "by-ping",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "output-format",
|
||||
Value: "text",
|
||||
Usage: "Either 'text' or 'csv'",
|
||||
},
|
||||
},
|
||||
Action: func(cctx *cli.Context) error {
|
||||
api, closer, err := GetFullNodeAPI(cctx)
|
||||
@ -1214,11 +1219,16 @@ var clientListAsksCmd = &cli.Command{
|
||||
return asks[i].Ping < asks[j].Ping
|
||||
})
|
||||
}
|
||||
pfmt := "%s: min:%s max:%s price:%s/GiB/Epoch verifiedPrice:%s/GiB/Epoch ping:%s\n"
|
||||
if cctx.String("output-format") == "csv" {
|
||||
fmt.Printf("Miner,Min,Max,Price,VerifiedPrice,Ping\n")
|
||||
pfmt = "%s,%s,%s,%s,%s,%s\n"
|
||||
}
|
||||
|
||||
for _, a := range asks {
|
||||
ask := a.Ask
|
||||
|
||||
fmt.Printf("%s: min:%s max:%s price:%s/GiB/Epoch verifiedPrice:%s/GiB/Epoch ping:%s\n", ask.Miner,
|
||||
fmt.Printf(pfmt, ask.Miner,
|
||||
types.SizeStr(types.NewInt(uint64(ask.MinPieceSize))),
|
||||
types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize))),
|
||||
types.FIL(ask.Price),
|
||||
@ -1237,7 +1247,13 @@ type QueriedAsk struct {
|
||||
}
|
||||
|
||||
func GetAsks(ctx context.Context, api lapi.FullNode) ([]QueriedAsk, error) {
|
||||
color.Blue(".. getting miner list")
|
||||
isTTY := true
|
||||
if fileInfo, _ := os.Stdout.Stat(); (fileInfo.Mode() & os.ModeCharDevice) == 0 {
|
||||
isTTY = false
|
||||
}
|
||||
if isTTY {
|
||||
color.Blue(".. getting miner list")
|
||||
}
|
||||
miners, err := api.StateListMiners(ctx, types.EmptyTSK)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("getting miner list: %w", err)
|
||||
@ -1282,14 +1298,18 @@ loop:
|
||||
for {
|
||||
select {
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
fmt.Printf("\r* Found %d miners with power", atomic.LoadInt64(&found))
|
||||
if isTTY {
|
||||
fmt.Printf("\r* Found %d miners with power", atomic.LoadInt64(&found))
|
||||
}
|
||||
case <-done:
|
||||
break loop
|
||||
}
|
||||
}
|
||||
fmt.Printf("\r* Found %d miners with power\n", atomic.LoadInt64(&found))
|
||||
if isTTY {
|
||||
fmt.Printf("\r* Found %d miners with power\n", atomic.LoadInt64(&found))
|
||||
|
||||
color.Blue(".. querying asks")
|
||||
color.Blue(".. querying asks")
|
||||
}
|
||||
|
||||
var asks []QueriedAsk
|
||||
var queried, got int64
|
||||
@ -1349,12 +1369,16 @@ loop2:
|
||||
for {
|
||||
select {
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
fmt.Printf("\r* Queried %d asks, got %d responses", atomic.LoadInt64(&queried), atomic.LoadInt64(&got))
|
||||
if isTTY {
|
||||
fmt.Printf("\r* Queried %d asks, got %d responses", atomic.LoadInt64(&queried), atomic.LoadInt64(&got))
|
||||
}
|
||||
case <-done:
|
||||
break loop2
|
||||
}
|
||||
}
|
||||
fmt.Printf("\r* Queried %d asks, got %d responses\n", atomic.LoadInt64(&queried), atomic.LoadInt64(&got))
|
||||
if isTTY {
|
||||
fmt.Printf("\r* Queried %d asks, got %d responses\n", atomic.LoadInt64(&queried), atomic.LoadInt64(&got))
|
||||
}
|
||||
|
||||
sort.Slice(asks, func(i, j int) bool {
|
||||
return asks[i].Ask.Price.LessThan(asks[j].Ask.Price)
|
||||
|
269
cli/cmd.go
269
cli/cmd.go
@ -1,34 +1,17 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
"github.com/mitchellh/go-homedir"
|
||||
"github.com/urfave/cli/v2"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-jsonrpc"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/api/client"
|
||||
cliutil "github.com/filecoin-project/lotus/cli/util"
|
||||
"github.com/filecoin-project/lotus/node/repo"
|
||||
)
|
||||
|
||||
var log = logging.Logger("cli")
|
||||
|
||||
const (
|
||||
metadataTraceContext = "traceContext"
|
||||
)
|
||||
|
||||
// custom CLI error
|
||||
|
||||
type ErrCmdFailed struct {
|
||||
@ -46,253 +29,31 @@ func NewCliError(s string) error {
|
||||
// ApiConnector returns API instance
|
||||
type ApiConnector func() api.FullNode
|
||||
|
||||
// The flag passed on the command line with the listen address of the API
|
||||
// server (only used by the tests)
|
||||
func flagForAPI(t repo.RepoType) string {
|
||||
switch t {
|
||||
case repo.FullNode:
|
||||
return "api-url"
|
||||
case repo.StorageMiner:
|
||||
return "miner-api-url"
|
||||
case repo.Worker:
|
||||
return "worker-api-url"
|
||||
default:
|
||||
panic(fmt.Sprintf("Unknown repo type: %v", t))
|
||||
}
|
||||
}
|
||||
|
||||
func flagForRepo(t repo.RepoType) string {
|
||||
switch t {
|
||||
case repo.FullNode:
|
||||
return "repo"
|
||||
case repo.StorageMiner:
|
||||
return "miner-repo"
|
||||
case repo.Worker:
|
||||
return "worker-repo"
|
||||
default:
|
||||
panic(fmt.Sprintf("Unknown repo type: %v", t))
|
||||
}
|
||||
}
|
||||
|
||||
func envForRepo(t repo.RepoType) string {
|
||||
switch t {
|
||||
case repo.FullNode:
|
||||
return "FULLNODE_API_INFO"
|
||||
case repo.StorageMiner:
|
||||
return "MINER_API_INFO"
|
||||
case repo.Worker:
|
||||
return "WORKER_API_INFO"
|
||||
default:
|
||||
panic(fmt.Sprintf("Unknown repo type: %v", t))
|
||||
}
|
||||
}
|
||||
|
||||
// TODO remove after deprecation period
|
||||
func envForRepoDeprecation(t repo.RepoType) string {
|
||||
switch t {
|
||||
case repo.FullNode:
|
||||
return "FULLNODE_API_INFO"
|
||||
case repo.StorageMiner:
|
||||
return "STORAGE_API_INFO"
|
||||
case repo.Worker:
|
||||
return "WORKER_API_INFO"
|
||||
default:
|
||||
panic(fmt.Sprintf("Unknown repo type: %v", t))
|
||||
}
|
||||
}
|
||||
|
||||
func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (cliutil.APIInfo, error) {
|
||||
// Check if there was a flag passed with the listen address of the API
|
||||
// server (only used by the tests)
|
||||
apiFlag := flagForAPI(t)
|
||||
if ctx.IsSet(apiFlag) {
|
||||
strma := ctx.String(apiFlag)
|
||||
strma = strings.TrimSpace(strma)
|
||||
|
||||
return cliutil.APIInfo{Addr: strma}, nil
|
||||
func GetFullNodeServices(ctx *cli.Context) (ServicesAPI, error) {
|
||||
if tn, ok := ctx.App.Metadata["test-services"]; ok {
|
||||
return tn.(ServicesAPI), nil
|
||||
}
|
||||
|
||||
envKey := envForRepo(t)
|
||||
env, ok := os.LookupEnv(envKey)
|
||||
if !ok {
|
||||
// TODO remove after deprecation period
|
||||
envKey = envForRepoDeprecation(t)
|
||||
env, ok = os.LookupEnv(envKey)
|
||||
if ok {
|
||||
log.Warnf("Use deprecation env(%s) value, please use env(%s) instead.", envKey, envForRepo(t))
|
||||
}
|
||||
}
|
||||
if ok {
|
||||
return cliutil.ParseApiInfo(env), nil
|
||||
}
|
||||
|
||||
repoFlag := flagForRepo(t)
|
||||
|
||||
p, err := homedir.Expand(ctx.String(repoFlag))
|
||||
api, c, err := GetFullNodeAPI(ctx)
|
||||
if err != nil {
|
||||
return cliutil.APIInfo{}, xerrors.Errorf("could not expand home dir (%s): %w", repoFlag, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
r, err := repo.NewFS(p)
|
||||
if err != nil {
|
||||
return cliutil.APIInfo{}, xerrors.Errorf("could not open repo at path: %s; %w", p, err)
|
||||
}
|
||||
|
||||
ma, err := r.APIEndpoint()
|
||||
if err != nil {
|
||||
return cliutil.APIInfo{}, xerrors.Errorf("could not get api endpoint: %w", err)
|
||||
}
|
||||
|
||||
token, err := r.APIToken()
|
||||
if err != nil {
|
||||
log.Warnf("Couldn't load CLI token, capabilities may be limited: %v", err)
|
||||
}
|
||||
|
||||
return cliutil.APIInfo{
|
||||
Addr: ma.String(),
|
||||
Token: token,
|
||||
}, nil
|
||||
return &ServicesImpl{api: api, closer: c}, nil
|
||||
}
|
||||
|
||||
func GetRawAPI(ctx *cli.Context, t repo.RepoType) (string, http.Header, error) {
|
||||
ainfo, err := GetAPIInfo(ctx, t)
|
||||
if err != nil {
|
||||
return "", nil, xerrors.Errorf("could not get API info: %w", err)
|
||||
}
|
||||
var GetAPIInfo = cliutil.GetAPIInfo
|
||||
var GetRawAPI = cliutil.GetRawAPI
|
||||
var GetAPI = cliutil.GetAPI
|
||||
|
||||
addr, err := ainfo.DialArgs()
|
||||
if err != nil {
|
||||
return "", nil, xerrors.Errorf("could not get DialArgs: %w", err)
|
||||
}
|
||||
var DaemonContext = cliutil.DaemonContext
|
||||
var ReqContext = cliutil.ReqContext
|
||||
|
||||
return addr, ainfo.AuthHeader(), nil
|
||||
}
|
||||
var GetFullNodeAPI = cliutil.GetFullNodeAPI
|
||||
var GetGatewayAPI = cliutil.GetGatewayAPI
|
||||
|
||||
func GetAPI(ctx *cli.Context) (api.Common, jsonrpc.ClientCloser, error) {
|
||||
ti, ok := ctx.App.Metadata["repoType"]
|
||||
if !ok {
|
||||
log.Errorf("unknown repo type, are you sure you want to use GetAPI?")
|
||||
ti = repo.FullNode
|
||||
}
|
||||
t, ok := ti.(repo.RepoType)
|
||||
if !ok {
|
||||
log.Errorf("repoType type does not match the type of repo.RepoType")
|
||||
}
|
||||
|
||||
if tn, ok := ctx.App.Metadata["testnode-storage"]; ok {
|
||||
return tn.(api.StorageMiner), func() {}, nil
|
||||
}
|
||||
if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
|
||||
return tn.(api.FullNode), func() {}, nil
|
||||
}
|
||||
|
||||
addr, headers, err := GetRawAPI(ctx, t)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return client.NewCommonRPC(ctx.Context, addr, headers)
|
||||
}
|
||||
|
||||
func GetFullNodeAPI(ctx *cli.Context) (api.FullNode, jsonrpc.ClientCloser, error) {
|
||||
if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
|
||||
return tn.(api.FullNode), func() {}, nil
|
||||
}
|
||||
|
||||
addr, headers, err := GetRawAPI(ctx, repo.FullNode)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return client.NewFullNodeRPC(ctx.Context, addr, headers)
|
||||
}
|
||||
|
||||
type GetStorageMinerOptions struct {
|
||||
PreferHttp bool
|
||||
}
|
||||
|
||||
type GetStorageMinerOption func(*GetStorageMinerOptions)
|
||||
|
||||
func StorageMinerUseHttp(opts *GetStorageMinerOptions) {
|
||||
opts.PreferHttp = true
|
||||
}
|
||||
|
||||
func GetStorageMinerAPI(ctx *cli.Context, opts ...GetStorageMinerOption) (api.StorageMiner, jsonrpc.ClientCloser, error) {
|
||||
var options GetStorageMinerOptions
|
||||
for _, opt := range opts {
|
||||
opt(&options)
|
||||
}
|
||||
|
||||
if tn, ok := ctx.App.Metadata["testnode-storage"]; ok {
|
||||
return tn.(api.StorageMiner), func() {}, nil
|
||||
}
|
||||
|
||||
addr, headers, err := GetRawAPI(ctx, repo.StorageMiner)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if options.PreferHttp {
|
||||
u, err := url.Parse(addr)
|
||||
if err != nil {
|
||||
return nil, nil, xerrors.Errorf("parsing miner api URL: %w", err)
|
||||
}
|
||||
|
||||
switch u.Scheme {
|
||||
case "ws":
|
||||
u.Scheme = "http"
|
||||
case "wss":
|
||||
u.Scheme = "https"
|
||||
}
|
||||
|
||||
addr = u.String()
|
||||
}
|
||||
|
||||
return client.NewStorageMinerRPC(ctx.Context, addr, headers)
|
||||
}
|
||||
|
||||
func GetWorkerAPI(ctx *cli.Context) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
|
||||
addr, headers, err := GetRawAPI(ctx, repo.Worker)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return client.NewWorkerRPC(ctx.Context, addr, headers)
|
||||
}
|
||||
|
||||
func GetGatewayAPI(ctx *cli.Context) (api.GatewayAPI, jsonrpc.ClientCloser, error) {
|
||||
addr, headers, err := GetRawAPI(ctx, repo.FullNode)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return client.NewGatewayRPC(ctx.Context, addr, headers)
|
||||
}
|
||||
|
||||
func DaemonContext(cctx *cli.Context) context.Context {
|
||||
if mtCtx, ok := cctx.App.Metadata[metadataTraceContext]; ok {
|
||||
return mtCtx.(context.Context)
|
||||
}
|
||||
|
||||
return context.Background()
|
||||
}
|
||||
|
||||
// ReqContext returns context for cli execution. Calling it for the first time
|
||||
// installs SIGTERM handler that will close returned context.
|
||||
// Not safe for concurrent execution.
|
||||
func ReqContext(cctx *cli.Context) context.Context {
|
||||
tCtx := DaemonContext(cctx)
|
||||
|
||||
ctx, done := context.WithCancel(tCtx)
|
||||
sigChan := make(chan os.Signal, 2)
|
||||
go func() {
|
||||
<-sigChan
|
||||
done()
|
||||
}()
|
||||
signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP)
|
||||
|
||||
return ctx
|
||||
}
|
||||
var GetStorageMinerAPI = cliutil.GetStorageMinerAPI
|
||||
var GetWorkerAPI = cliutil.GetWorkerAPI
|
||||
|
||||
var CommonCommands = []*cli.Command{
|
||||
netCmd,
|
||||
|
@ -29,7 +29,7 @@ import (
|
||||
init2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init"
|
||||
msig2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig"
|
||||
|
||||
"github.com/filecoin-project/lotus/api/apibstore"
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/actors/adt"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/multisig"
|
||||
@ -202,7 +202,7 @@ var msigInspectCmd = &cli.Command{
|
||||
defer closer()
|
||||
ctx := ReqContext(cctx)
|
||||
|
||||
store := adt.WrapStore(ctx, cbor.NewCborStore(apibstore.NewAPIBlockstore(api)))
|
||||
store := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewAPIBlockstore(api)))
|
||||
|
||||
maddr, err := address.NewFromString(cctx.Args().First())
|
||||
if err != nil {
|
||||
@ -1275,7 +1275,7 @@ var msigLockApproveCmd = &cli.Command{
|
||||
params, actErr := actors.SerializeParams(&msig2.LockBalanceParams{
|
||||
StartEpoch: abi.ChainEpoch(start),
|
||||
UnlockDuration: abi.ChainEpoch(duration),
|
||||
Amount: abi.NewTokenAmount(amount.Int64()),
|
||||
Amount: big.Int(amount),
|
||||
})
|
||||
|
||||
if actErr != nil {
|
||||
@ -1367,7 +1367,7 @@ var msigLockCancelCmd = &cli.Command{
|
||||
params, actErr := actors.SerializeParams(&msig2.LockBalanceParams{
|
||||
StartEpoch: abi.ChainEpoch(start),
|
||||
UnlockDuration: abi.ChainEpoch(duration),
|
||||
Amount: abi.NewTokenAmount(amount.Int64()),
|
||||
Amount: big.Int(amount),
|
||||
})
|
||||
|
||||
if actErr != nil {
|
||||
|
50
cli/net.go
50
cli/net.go
@ -48,6 +48,11 @@ var NetPeers = &cli.Command{
|
||||
Aliases: []string{"a"},
|
||||
Usage: "Print agent name",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "extended",
|
||||
Aliases: []string{"x"},
|
||||
Usage: "Print extended peer information in json",
|
||||
},
|
||||
},
|
||||
Action: func(cctx *cli.Context) error {
|
||||
api, closer, err := GetAPI(cctx)
|
||||
@ -65,18 +70,42 @@ var NetPeers = &cli.Command{
|
||||
return strings.Compare(string(peers[i].ID), string(peers[j].ID)) > 0
|
||||
})
|
||||
|
||||
for _, peer := range peers {
|
||||
var agent string
|
||||
if cctx.Bool("agent") {
|
||||
agent, err = api.NetAgentVersion(ctx, peer.ID)
|
||||
if cctx.Bool("extended") {
|
||||
// deduplicate
|
||||
seen := make(map[peer.ID]struct{})
|
||||
|
||||
for _, peer := range peers {
|
||||
_, dup := seen[peer.ID]
|
||||
if dup {
|
||||
continue
|
||||
}
|
||||
seen[peer.ID] = struct{}{}
|
||||
|
||||
info, err := api.NetPeerInfo(ctx, peer.ID)
|
||||
if err != nil {
|
||||
log.Warnf("getting agent version: %s", err)
|
||||
log.Warnf("error getting extended peer info: %s", err)
|
||||
} else {
|
||||
agent = ", " + agent
|
||||
bytes, err := json.Marshal(&info)
|
||||
if err != nil {
|
||||
log.Warnf("error marshalling extended peer info: %s", err)
|
||||
} else {
|
||||
fmt.Println(string(bytes))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("%s, %s%s\n", peer.ID, peer.Addrs, agent)
|
||||
} else {
|
||||
for _, peer := range peers {
|
||||
var agent string
|
||||
if cctx.Bool("agent") {
|
||||
agent, err = api.NetAgentVersion(ctx, peer.ID)
|
||||
if err != nil {
|
||||
log.Warnf("getting agent version: %s", err)
|
||||
} else {
|
||||
agent = ", " + agent
|
||||
}
|
||||
}
|
||||
fmt.Printf("%s, %s%s\n", peer.ID, peer.Addrs, agent)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
@ -88,8 +117,9 @@ var netScores = &cli.Command{
|
||||
Usage: "Print peers' pubsub scores",
|
||||
Flags: []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
Name: "extended",
|
||||
Usage: "print extended peer scores in json",
|
||||
Name: "extended",
|
||||
Aliases: []string{"x"},
|
||||
Usage: "print extended peer scores in json",
|
||||
},
|
||||
},
|
||||
Action: func(cctx *cli.Context) error {
|
||||
|
@ -20,8 +20,8 @@ import (
|
||||
cbor "github.com/ipfs/go-ipld-cbor"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/filecoin-project/lotus/api/apibstore"
|
||||
"github.com/filecoin-project/lotus/api/test"
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/events"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
@ -400,7 +400,7 @@ func getPaychState(ctx context.Context, t *testing.T, node test.TestNode, chAddr
|
||||
act, err := node.StateGetActor(ctx, chAddr, types.EmptyTSK)
|
||||
require.NoError(t, err)
|
||||
|
||||
store := cbor.NewCborStore(apibstore.NewAPIBlockstore(node))
|
||||
store := cbor.NewCborStore(blockstore.NewAPIBlockstore(node))
|
||||
chState, err := paych.Load(adt.WrapStore(ctx, store), act)
|
||||
require.NoError(t, err)
|
||||
|
||||
|
144
cli/send.go
144
cli/send.go
@ -1,22 +1,17 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
cbg "github.com/whyrusleeping/cbor-gen"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin"
|
||||
"github.com/filecoin-project/lotus/chain/stmgr"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
)
|
||||
|
||||
@ -72,15 +67,16 @@ var sendCmd = &cli.Command{
|
||||
return ShowHelp(cctx, fmt.Errorf("'send' expects two arguments, target and amount"))
|
||||
}
|
||||
|
||||
api, closer, err := GetFullNodeAPI(cctx)
|
||||
srv, err := GetFullNodeServices(cctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
defer srv.Close() //nolint:errcheck
|
||||
|
||||
ctx := ReqContext(cctx)
|
||||
var params SendParams
|
||||
|
||||
toAddr, err := address.NewFromString(cctx.Args().Get(0))
|
||||
params.To, err = address.NewFromString(cctx.Args().Get(0))
|
||||
if err != nil {
|
||||
return ShowHelp(cctx, fmt.Errorf("failed to parse target address: %w", err))
|
||||
}
|
||||
@ -89,123 +85,75 @@ var sendCmd = &cli.Command{
|
||||
if err != nil {
|
||||
return ShowHelp(cctx, fmt.Errorf("failed to parse amount: %w", err))
|
||||
}
|
||||
params.Val = abi.TokenAmount(val)
|
||||
|
||||
var fromAddr address.Address
|
||||
if from := cctx.String("from"); from == "" {
|
||||
defaddr, err := api.WalletDefaultAddress(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fromAddr = defaddr
|
||||
} else {
|
||||
if from := cctx.String("from"); from != "" {
|
||||
addr, err := address.NewFromString(from)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fromAddr = addr
|
||||
params.From = addr
|
||||
}
|
||||
|
||||
gp, err := types.BigFromString(cctx.String("gas-premium"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
gfc, err := types.BigFromString(cctx.String("gas-feecap"))
|
||||
if err != nil {
|
||||
return err
|
||||
if cctx.IsSet("gas-premium") {
|
||||
gp, err := types.BigFromString(cctx.String("gas-premium"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
params.GasPremium = &gp
|
||||
}
|
||||
|
||||
method := abi.MethodNum(cctx.Uint64("method"))
|
||||
if cctx.IsSet("gas-feecap") {
|
||||
gfc, err := types.BigFromString(cctx.String("gas-feecap"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
params.GasFeeCap = &gfc
|
||||
}
|
||||
|
||||
if cctx.IsSet("gas-limit") {
|
||||
limit := cctx.Int64("gas-limit")
|
||||
params.GasLimit = &limit
|
||||
}
|
||||
|
||||
params.Method = abi.MethodNum(cctx.Uint64("method"))
|
||||
|
||||
var params []byte
|
||||
if cctx.IsSet("params-json") {
|
||||
decparams, err := decodeTypedParams(ctx, api, toAddr, method, cctx.String("params-json"))
|
||||
decparams, err := srv.DecodeTypedParamsFromJSON(ctx, params.To, params.Method, cctx.String("params-json"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decode json params: %w", err)
|
||||
}
|
||||
params = decparams
|
||||
params.Params = decparams
|
||||
}
|
||||
if cctx.IsSet("params-hex") {
|
||||
if params != nil {
|
||||
if params.Params != nil {
|
||||
return fmt.Errorf("can only specify one of 'params-json' and 'params-hex'")
|
||||
}
|
||||
decparams, err := hex.DecodeString(cctx.String("params-hex"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decode hex params: %w", err)
|
||||
}
|
||||
params = decparams
|
||||
params.Params = decparams
|
||||
}
|
||||
|
||||
msg := &types.Message{
|
||||
From: fromAddr,
|
||||
To: toAddr,
|
||||
Value: types.BigInt(val),
|
||||
GasPremium: gp,
|
||||
GasFeeCap: gfc,
|
||||
GasLimit: cctx.Int64("gas-limit"),
|
||||
Method: method,
|
||||
Params: params,
|
||||
}
|
||||
|
||||
if !cctx.Bool("force") {
|
||||
// Funds insufficient check
|
||||
fromBalance, err := api.WalletBalance(ctx, msg.From)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
totalCost := types.BigAdd(types.BigMul(msg.GasFeeCap, types.NewInt(uint64(msg.GasLimit))), msg.Value)
|
||||
|
||||
if fromBalance.LessThan(totalCost) {
|
||||
fmt.Printf("WARNING: From balance %s less than total cost %s\n", types.FIL(fromBalance), types.FIL(totalCost))
|
||||
return fmt.Errorf("--force must be specified for this action to have an effect; you have been warned")
|
||||
}
|
||||
}
|
||||
params.Force = cctx.Bool("force")
|
||||
|
||||
if cctx.IsSet("nonce") {
|
||||
msg.Nonce = cctx.Uint64("nonce")
|
||||
sm, err := api.WalletSignMessage(ctx, fromAddr, msg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = api.MpoolPush(ctx, sm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(sm.Cid())
|
||||
} else {
|
||||
sm, err := api.MpoolPushMessage(ctx, msg, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(sm.Cid())
|
||||
n := cctx.Uint64("nonce")
|
||||
params.Nonce = &n
|
||||
}
|
||||
|
||||
msgCid, err := srv.Send(ctx, params)
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrSendBalanceTooLow) {
|
||||
return fmt.Errorf("--force must be specified for this action to have an effect; you have been warned: %w", err)
|
||||
}
|
||||
return xerrors.Errorf("executing send: %w", err)
|
||||
}
|
||||
|
||||
fmt.Fprintf(cctx.App.Writer, "%s\n", msgCid)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
func decodeTypedParams(ctx context.Context, fapi api.FullNode, to address.Address, method abi.MethodNum, paramstr string) ([]byte, error) {
|
||||
act, err := fapi.StateGetActor(ctx, to, types.EmptyTSK)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
methodMeta, found := stmgr.MethodsMap[act.Code][method]
|
||||
if !found {
|
||||
return nil, fmt.Errorf("method %d not found on actor %s", method, act.Code)
|
||||
}
|
||||
|
||||
p := reflect.New(methodMeta.Params.Elem()).Interface().(cbg.CBORMarshaler)
|
||||
|
||||
if err := json.Unmarshal([]byte(paramstr), p); err != nil {
|
||||
return nil, fmt.Errorf("unmarshaling input into params type: %w", err)
|
||||
}
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
if err := p.MarshalCBOR(buf); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
128
cli/send_test.go
Normal file
128
cli/send_test.go
Normal file
@ -0,0 +1,128 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
types "github.com/filecoin-project/lotus/chain/types"
|
||||
gomock "github.com/golang/mock/gomock"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
ucli "github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
var arbtCid = (&types.Message{
|
||||
From: mustAddr(address.NewIDAddress(2)),
|
||||
To: mustAddr(address.NewIDAddress(1)),
|
||||
Value: types.NewInt(1000),
|
||||
}).Cid()
|
||||
|
||||
func mustAddr(a address.Address, err error) address.Address {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func newMockApp(t *testing.T, cmd *ucli.Command) (*ucli.App, *MockServicesAPI, *bytes.Buffer, func()) {
|
||||
app := ucli.NewApp()
|
||||
app.Commands = ucli.Commands{cmd}
|
||||
app.Setup()
|
||||
|
||||
mockCtrl := gomock.NewController(t)
|
||||
mockSrvcs := NewMockServicesAPI(mockCtrl)
|
||||
app.Metadata["test-services"] = mockSrvcs
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
app.Writer = buf
|
||||
|
||||
return app, mockSrvcs, buf, mockCtrl.Finish
|
||||
}
|
||||
|
||||
func TestSendCLI(t *testing.T) {
|
||||
oneFil := abi.TokenAmount(types.MustParseFIL("1"))
|
||||
|
||||
t.Run("simple", func(t *testing.T) {
|
||||
app, mockSrvcs, buf, done := newMockApp(t, sendCmd)
|
||||
defer done()
|
||||
|
||||
gomock.InOrder(
|
||||
mockSrvcs.EXPECT().Send(gomock.Any(), SendParams{
|
||||
To: mustAddr(address.NewIDAddress(1)),
|
||||
Val: oneFil,
|
||||
}).Return(arbtCid, nil),
|
||||
mockSrvcs.EXPECT().Close(),
|
||||
)
|
||||
err := app.Run([]string{"lotus", "send", "t01", "1"})
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, arbtCid.String()+"\n", buf.String())
|
||||
})
|
||||
t.Run("ErrSendBalanceTooLow", func(t *testing.T) {
|
||||
app, mockSrvcs, _, done := newMockApp(t, sendCmd)
|
||||
defer done()
|
||||
|
||||
gomock.InOrder(
|
||||
mockSrvcs.EXPECT().Send(gomock.Any(), SendParams{
|
||||
To: mustAddr(address.NewIDAddress(1)),
|
||||
Val: oneFil,
|
||||
}).Return(cid.Undef, ErrSendBalanceTooLow),
|
||||
mockSrvcs.EXPECT().Close(),
|
||||
)
|
||||
err := app.Run([]string{"lotus", "send", "t01", "1"})
|
||||
assert.ErrorIs(t, err, ErrSendBalanceTooLow)
|
||||
})
|
||||
t.Run("generic-err-is-forwarded", func(t *testing.T) {
|
||||
app, mockSrvcs, _, done := newMockApp(t, sendCmd)
|
||||
defer done()
|
||||
|
||||
errMark := errors.New("something")
|
||||
gomock.InOrder(
|
||||
mockSrvcs.EXPECT().Send(gomock.Any(), SendParams{
|
||||
To: mustAddr(address.NewIDAddress(1)),
|
||||
Val: oneFil,
|
||||
}).Return(cid.Undef, errMark),
|
||||
mockSrvcs.EXPECT().Close(),
|
||||
)
|
||||
err := app.Run([]string{"lotus", "send", "t01", "1"})
|
||||
assert.ErrorIs(t, err, errMark)
|
||||
})
|
||||
|
||||
t.Run("from-specific", func(t *testing.T) {
|
||||
app, mockSrvcs, buf, done := newMockApp(t, sendCmd)
|
||||
defer done()
|
||||
|
||||
gomock.InOrder(
|
||||
mockSrvcs.EXPECT().Send(gomock.Any(), SendParams{
|
||||
To: mustAddr(address.NewIDAddress(1)),
|
||||
From: mustAddr(address.NewIDAddress(2)),
|
||||
Val: oneFil,
|
||||
}).Return(arbtCid, nil),
|
||||
mockSrvcs.EXPECT().Close(),
|
||||
)
|
||||
err := app.Run([]string{"lotus", "send", "--from=t02", "t01", "1"})
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, arbtCid.String()+"\n", buf.String())
|
||||
})
|
||||
|
||||
t.Run("nonce-specific", func(t *testing.T) {
|
||||
app, mockSrvcs, buf, done := newMockApp(t, sendCmd)
|
||||
defer done()
|
||||
zero := uint64(0)
|
||||
|
||||
gomock.InOrder(
|
||||
mockSrvcs.EXPECT().Send(gomock.Any(), SendParams{
|
||||
To: mustAddr(address.NewIDAddress(1)),
|
||||
Nonce: &zero,
|
||||
Val: oneFil,
|
||||
}).Return(arbtCid, nil),
|
||||
mockSrvcs.EXPECT().Close(),
|
||||
)
|
||||
err := app.Run([]string{"lotus", "send", "--nonce=0", "t01", "1"})
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, arbtCid.String()+"\n", buf.String())
|
||||
})
|
||||
|
||||
}
|
166
cli/services.go
Normal file
166
cli/services.go
Normal file
@ -0,0 +1,166 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-jsonrpc"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/chain/stmgr"
|
||||
types "github.com/filecoin-project/lotus/chain/types"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
cbg "github.com/whyrusleeping/cbor-gen"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
//go:generate go run github.com/golang/mock/mockgen -destination=servicesmock_test.go -package=cli -self_package github.com/filecoin-project/lotus/cli . ServicesAPI
|
||||
|
||||
type ServicesAPI interface {
|
||||
// Sends executes a send given SendParams
|
||||
Send(ctx context.Context, params SendParams) (cid.Cid, error)
|
||||
// DecodeTypedParamsFromJSON takes in information needed to identify a method and converts JSON
|
||||
// parameters to bytes of their CBOR encoding
|
||||
DecodeTypedParamsFromJSON(ctx context.Context, to address.Address, method abi.MethodNum, paramstr string) ([]byte, error)
|
||||
|
||||
// Close ends the session of services and disconnects from RPC, using Services after Close is called
|
||||
// most likely will result in an error
|
||||
// Should not be called concurrently
|
||||
Close() error
|
||||
}
|
||||
|
||||
type ServicesImpl struct {
|
||||
api api.FullNode
|
||||
closer jsonrpc.ClientCloser
|
||||
}
|
||||
|
||||
func (s *ServicesImpl) Close() error {
|
||||
if s.closer == nil {
|
||||
return xerrors.Errorf("Services already closed")
|
||||
}
|
||||
s.closer()
|
||||
s.closer = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *ServicesImpl) DecodeTypedParamsFromJSON(ctx context.Context, to address.Address, method abi.MethodNum, paramstr string) ([]byte, error) {
|
||||
act, err := s.api.StateGetActor(ctx, to, types.EmptyTSK)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
methodMeta, found := stmgr.MethodsMap[act.Code][method]
|
||||
if !found {
|
||||
return nil, fmt.Errorf("method %d not found on actor %s", method, act.Code)
|
||||
}
|
||||
|
||||
p := reflect.New(methodMeta.Params.Elem()).Interface().(cbg.CBORMarshaler)
|
||||
|
||||
if err := json.Unmarshal([]byte(paramstr), p); err != nil {
|
||||
return nil, fmt.Errorf("unmarshaling input into params type: %w", err)
|
||||
}
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
if err := p.MarshalCBOR(buf); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
type SendParams struct {
|
||||
To address.Address
|
||||
From address.Address
|
||||
Val abi.TokenAmount
|
||||
|
||||
GasPremium *abi.TokenAmount
|
||||
GasFeeCap *abi.TokenAmount
|
||||
GasLimit *int64
|
||||
|
||||
Nonce *uint64
|
||||
Method abi.MethodNum
|
||||
Params []byte
|
||||
|
||||
Force bool
|
||||
}
|
||||
|
||||
// This is specialised Send for Send command
|
||||
// There might be room for generic Send that other commands can use to send their messages
|
||||
// We will see
|
||||
|
||||
var ErrSendBalanceTooLow = errors.New("balance too low")
|
||||
|
||||
func (s *ServicesImpl) Send(ctx context.Context, params SendParams) (cid.Cid, error) {
|
||||
if params.From == address.Undef {
|
||||
defaddr, err := s.api.WalletDefaultAddress(ctx)
|
||||
if err != nil {
|
||||
return cid.Undef, err
|
||||
}
|
||||
params.From = defaddr
|
||||
}
|
||||
|
||||
msg := &types.Message{
|
||||
From: params.From,
|
||||
To: params.To,
|
||||
Value: params.Val,
|
||||
|
||||
Method: params.Method,
|
||||
Params: params.Params,
|
||||
}
|
||||
|
||||
if params.GasPremium != nil {
|
||||
msg.GasPremium = *params.GasPremium
|
||||
} else {
|
||||
msg.GasPremium = types.NewInt(0)
|
||||
}
|
||||
if params.GasFeeCap != nil {
|
||||
msg.GasFeeCap = *params.GasFeeCap
|
||||
} else {
|
||||
msg.GasFeeCap = types.NewInt(0)
|
||||
}
|
||||
if params.GasLimit != nil {
|
||||
msg.GasLimit = *params.GasLimit
|
||||
} else {
|
||||
msg.GasLimit = 0
|
||||
}
|
||||
|
||||
if !params.Force {
|
||||
// Funds insufficient check
|
||||
fromBalance, err := s.api.WalletBalance(ctx, msg.From)
|
||||
if err != nil {
|
||||
return cid.Undef, err
|
||||
}
|
||||
totalCost := types.BigAdd(types.BigMul(msg.GasFeeCap, types.NewInt(uint64(msg.GasLimit))), msg.Value)
|
||||
|
||||
if fromBalance.LessThan(totalCost) {
|
||||
return cid.Undef, xerrors.Errorf("From balance %s less than total cost %s: %w", types.FIL(fromBalance), types.FIL(totalCost), ErrSendBalanceTooLow)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if params.Nonce != nil {
|
||||
msg.Nonce = *params.Nonce
|
||||
sm, err := s.api.WalletSignMessage(ctx, params.From, msg)
|
||||
if err != nil {
|
||||
return cid.Undef, err
|
||||
}
|
||||
|
||||
_, err = s.api.MpoolPush(ctx, sm)
|
||||
if err != nil {
|
||||
return cid.Undef, err
|
||||
}
|
||||
|
||||
return sm.Cid(), nil
|
||||
}
|
||||
|
||||
sm, err := s.api.MpoolPushMessage(ctx, msg, nil)
|
||||
if err != nil {
|
||||
return cid.Undef, err
|
||||
}
|
||||
|
||||
return sm.Cid(), nil
|
||||
}
|
266
cli/services_send_test.go
Normal file
266
cli/services_send_test.go
Normal file
@ -0,0 +1,266 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-state-types/big"
|
||||
"github.com/filecoin-project/go-state-types/crypto"
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/api/mocks"
|
||||
types "github.com/filecoin-project/lotus/chain/types"
|
||||
gomock "github.com/golang/mock/gomock"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
type markerKeyType struct{}
|
||||
|
||||
var markerKey = markerKeyType{}
|
||||
|
||||
type contextMatcher struct {
|
||||
marker *int
|
||||
}
|
||||
|
||||
// Matches returns whether x is a match.
|
||||
func (cm contextMatcher) Matches(x interface{}) bool {
|
||||
ctx, ok := x.(context.Context)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
maybeMarker, ok := ctx.Value(markerKey).(*int)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
return cm.marker == maybeMarker
|
||||
}
|
||||
|
||||
func (cm contextMatcher) String() string {
|
||||
return fmt.Sprintf("Context with Value(%v/%T, %p)", markerKey, markerKey, cm.marker)
|
||||
}
|
||||
|
||||
func ContextWithMarker(ctx context.Context) (context.Context, gomock.Matcher) {
|
||||
marker := new(int)
|
||||
outCtx := context.WithValue(ctx, markerKey, marker)
|
||||
return outCtx, contextMatcher{marker: marker}
|
||||
|
||||
}
|
||||
|
||||
func setupMockSrvcs(t *testing.T) (*ServicesImpl, *mocks.MockFullNode) {
|
||||
mockCtrl := gomock.NewController(t)
|
||||
|
||||
mockApi := mocks.NewMockFullNode(mockCtrl)
|
||||
|
||||
srvcs := &ServicesImpl{
|
||||
api: mockApi,
|
||||
closer: mockCtrl.Finish,
|
||||
}
|
||||
return srvcs, mockApi
|
||||
}
|
||||
|
||||
func fakeSign(msg *types.Message) *types.SignedMessage {
|
||||
return &types.SignedMessage{
|
||||
Message: *msg,
|
||||
Signature: crypto.Signature{Type: crypto.SigTypeSecp256k1, Data: make([]byte, 32)},
|
||||
}
|
||||
}
|
||||
|
||||
func makeMessageSigner() (*cid.Cid, interface{}) {
|
||||
smCid := cid.Undef
|
||||
return &smCid,
|
||||
func(_ context.Context, msg *types.Message, _ *api.MessageSendSpec) (*types.SignedMessage, error) {
|
||||
sm := fakeSign(msg)
|
||||
smCid = sm.Cid()
|
||||
return sm, nil
|
||||
}
|
||||
}
|
||||
|
||||
type MessageMatcher SendParams
|
||||
|
||||
var _ gomock.Matcher = MessageMatcher{}
|
||||
|
||||
// Matches returns whether x is a match.
|
||||
func (mm MessageMatcher) Matches(x interface{}) bool {
|
||||
m, ok := x.(*types.Message)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
if mm.From != address.Undef && mm.From != m.From {
|
||||
return false
|
||||
}
|
||||
if mm.To != address.Undef && mm.To != m.To {
|
||||
return false
|
||||
}
|
||||
|
||||
if types.BigCmp(mm.Val, m.Value) != 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
if mm.Nonce != nil && *mm.Nonce != m.Nonce {
|
||||
return false
|
||||
}
|
||||
|
||||
if mm.GasPremium != nil && big.Cmp(*mm.GasPremium, m.GasPremium) != 0 {
|
||||
return false
|
||||
}
|
||||
if mm.GasPremium == nil && m.GasPremium.Sign() != 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
if mm.GasFeeCap != nil && big.Cmp(*mm.GasFeeCap, m.GasFeeCap) != 0 {
|
||||
return false
|
||||
}
|
||||
if mm.GasFeeCap == nil && m.GasFeeCap.Sign() != 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
if mm.GasLimit != nil && *mm.GasLimit != m.GasLimit {
|
||||
return false
|
||||
}
|
||||
|
||||
if mm.GasLimit == nil && m.GasLimit != 0 {
|
||||
return false
|
||||
}
|
||||
// handle rest of options
|
||||
return true
|
||||
}
|
||||
|
||||
// String describes what the matcher matches.
|
||||
func (mm MessageMatcher) String() string {
|
||||
return fmt.Sprintf("%#v", SendParams(mm))
|
||||
}
|
||||
|
||||
func TestSendService(t *testing.T) {
|
||||
addrGen := address.NewForTestGetter()
|
||||
a1 := addrGen()
|
||||
a2 := addrGen()
|
||||
|
||||
const balance = 10000
|
||||
|
||||
params := SendParams{
|
||||
From: a1,
|
||||
To: a2,
|
||||
Val: types.NewInt(balance - 100),
|
||||
}
|
||||
|
||||
ctx, ctxM := ContextWithMarker(context.Background())
|
||||
|
||||
t.Run("happy", func(t *testing.T) {
|
||||
params := params
|
||||
srvcs, mockApi := setupMockSrvcs(t)
|
||||
defer srvcs.Close() //nolint:errcheck
|
||||
msgCid, sign := makeMessageSigner()
|
||||
gomock.InOrder(
|
||||
mockApi.EXPECT().WalletBalance(ctxM, params.From).Return(types.NewInt(balance), nil),
|
||||
mockApi.EXPECT().MpoolPushMessage(ctxM, MessageMatcher(params), nil).DoAndReturn(sign),
|
||||
)
|
||||
|
||||
c, err := srvcs.Send(ctx, params)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, *msgCid, c)
|
||||
})
|
||||
|
||||
t.Run("balance-too-low", func(t *testing.T) {
|
||||
params := params
|
||||
srvcs, mockApi := setupMockSrvcs(t)
|
||||
defer srvcs.Close() //nolint:errcheck
|
||||
gomock.InOrder(
|
||||
mockApi.EXPECT().WalletBalance(ctxM, a1).Return(types.NewInt(balance-200), nil),
|
||||
// no MpoolPushMessage
|
||||
)
|
||||
|
||||
c, err := srvcs.Send(ctx, params)
|
||||
assert.Equal(t, c, cid.Undef)
|
||||
assert.ErrorIs(t, err, ErrSendBalanceTooLow)
|
||||
})
|
||||
|
||||
t.Run("force", func(t *testing.T) {
|
||||
params := params
|
||||
params.Force = true
|
||||
srvcs, mockApi := setupMockSrvcs(t)
|
||||
defer srvcs.Close() //nolint:errcheck
|
||||
msgCid, sign := makeMessageSigner()
|
||||
gomock.InOrder(
|
||||
mockApi.EXPECT().WalletBalance(ctxM, a1).Return(types.NewInt(balance-200), nil).AnyTimes(),
|
||||
mockApi.EXPECT().MpoolPushMessage(ctxM, MessageMatcher(params), nil).DoAndReturn(sign),
|
||||
)
|
||||
|
||||
c, err := srvcs.Send(ctx, params)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, *msgCid, c)
|
||||
})
|
||||
|
||||
t.Run("default-from", func(t *testing.T) {
|
||||
params := params
|
||||
params.From = address.Undef
|
||||
mm := MessageMatcher(params)
|
||||
mm.From = a1
|
||||
|
||||
srvcs, mockApi := setupMockSrvcs(t)
|
||||
defer srvcs.Close() //nolint:errcheck
|
||||
msgCid, sign := makeMessageSigner()
|
||||
gomock.InOrder(
|
||||
mockApi.EXPECT().WalletDefaultAddress(ctxM).Return(a1, nil),
|
||||
mockApi.EXPECT().WalletBalance(ctxM, a1).Return(types.NewInt(balance), nil),
|
||||
mockApi.EXPECT().MpoolPushMessage(ctxM, mm, nil).DoAndReturn(sign),
|
||||
)
|
||||
|
||||
c, err := srvcs.Send(ctx, params)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, *msgCid, c)
|
||||
})
|
||||
|
||||
t.Run("set-nonce", func(t *testing.T) {
|
||||
params := params
|
||||
n := uint64(5)
|
||||
params.Nonce = &n
|
||||
mm := MessageMatcher(params)
|
||||
|
||||
srvcs, mockApi := setupMockSrvcs(t)
|
||||
defer srvcs.Close() //nolint:errcheck
|
||||
_, _ = mm, mockApi
|
||||
|
||||
var sm *types.SignedMessage
|
||||
gomock.InOrder(
|
||||
mockApi.EXPECT().WalletBalance(ctxM, a1).Return(types.NewInt(balance), nil),
|
||||
mockApi.EXPECT().WalletSignMessage(ctxM, a1, mm).DoAndReturn(
|
||||
func(_ context.Context, _ address.Address, msg *types.Message) (*types.SignedMessage, error) {
|
||||
sm = fakeSign(msg)
|
||||
|
||||
// now we expect MpoolPush with that SignedMessage
|
||||
mockApi.EXPECT().MpoolPush(ctxM, sm).Return(sm.Cid(), nil)
|
||||
return sm, nil
|
||||
}),
|
||||
)
|
||||
|
||||
c, err := srvcs.Send(ctx, params)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, sm.Cid(), c)
|
||||
})
|
||||
|
||||
t.Run("gas-params", func(t *testing.T) {
|
||||
params := params
|
||||
limit := int64(1)
|
||||
params.GasLimit = &limit
|
||||
gfc := big.NewInt(100)
|
||||
params.GasFeeCap = &gfc
|
||||
gp := big.NewInt(10)
|
||||
params.GasPremium = &gp
|
||||
|
||||
srvcs, mockApi := setupMockSrvcs(t)
|
||||
defer srvcs.Close() //nolint:errcheck
|
||||
msgCid, sign := makeMessageSigner()
|
||||
gomock.InOrder(
|
||||
mockApi.EXPECT().WalletBalance(ctxM, params.From).Return(types.NewInt(balance), nil),
|
||||
mockApi.EXPECT().MpoolPushMessage(ctxM, MessageMatcher(params), nil).DoAndReturn(sign),
|
||||
)
|
||||
|
||||
c, err := srvcs.Send(ctx, params)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, *msgCid, c)
|
||||
})
|
||||
}
|
81
cli/servicesmock_test.go
Normal file
81
cli/servicesmock_test.go
Normal file
@ -0,0 +1,81 @@
|
||||
// Code generated by MockGen. DO NOT EDIT.
|
||||
// Source: github.com/filecoin-project/lotus/cli (interfaces: ServicesAPI)
|
||||
|
||||
// Package cli is a generated GoMock package.
|
||||
package cli
|
||||
|
||||
import (
|
||||
context "context"
|
||||
go_address "github.com/filecoin-project/go-address"
|
||||
abi "github.com/filecoin-project/go-state-types/abi"
|
||||
gomock "github.com/golang/mock/gomock"
|
||||
go_cid "github.com/ipfs/go-cid"
|
||||
reflect "reflect"
|
||||
)
|
||||
|
||||
// MockServicesAPI is a mock of ServicesAPI interface
|
||||
type MockServicesAPI struct {
|
||||
ctrl *gomock.Controller
|
||||
recorder *MockServicesAPIMockRecorder
|
||||
}
|
||||
|
||||
// MockServicesAPIMockRecorder is the mock recorder for MockServicesAPI
|
||||
type MockServicesAPIMockRecorder struct {
|
||||
mock *MockServicesAPI
|
||||
}
|
||||
|
||||
// NewMockServicesAPI creates a new mock instance
|
||||
func NewMockServicesAPI(ctrl *gomock.Controller) *MockServicesAPI {
|
||||
mock := &MockServicesAPI{ctrl: ctrl}
|
||||
mock.recorder = &MockServicesAPIMockRecorder{mock}
|
||||
return mock
|
||||
}
|
||||
|
||||
// EXPECT returns an object that allows the caller to indicate expected use
|
||||
func (m *MockServicesAPI) EXPECT() *MockServicesAPIMockRecorder {
|
||||
return m.recorder
|
||||
}
|
||||
|
||||
// Close mocks base method
|
||||
func (m *MockServicesAPI) Close() error {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "Close")
|
||||
ret0, _ := ret[0].(error)
|
||||
return ret0
|
||||
}
|
||||
|
||||
// Close indicates an expected call of Close
|
||||
func (mr *MockServicesAPIMockRecorder) Close() *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockServicesAPI)(nil).Close))
|
||||
}
|
||||
|
||||
// DecodeTypedParamsFromJSON mocks base method
|
||||
func (m *MockServicesAPI) DecodeTypedParamsFromJSON(arg0 context.Context, arg1 go_address.Address, arg2 abi.MethodNum, arg3 string) ([]byte, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "DecodeTypedParamsFromJSON", arg0, arg1, arg2, arg3)
|
||||
ret0, _ := ret[0].([]byte)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// DecodeTypedParamsFromJSON indicates an expected call of DecodeTypedParamsFromJSON
|
||||
func (mr *MockServicesAPIMockRecorder) DecodeTypedParamsFromJSON(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DecodeTypedParamsFromJSON", reflect.TypeOf((*MockServicesAPI)(nil).DecodeTypedParamsFromJSON), arg0, arg1, arg2, arg3)
|
||||
}
|
||||
|
||||
// Send mocks base method
|
||||
func (m *MockServicesAPI) Send(arg0 context.Context, arg1 SendParams) (go_cid.Cid, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "Send", arg0, arg1)
|
||||
ret0, _ := ret[0].(go_cid.Cid)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// Send indicates an expected call of Send
|
||||
func (mr *MockServicesAPIMockRecorder) Send(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockServicesAPI)(nil).Send), arg0, arg1)
|
||||
}
|
@ -34,7 +34,7 @@ import (
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
lapi "github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/api/apibstore"
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/state"
|
||||
"github.com/filecoin-project/lotus/chain/stmgr"
|
||||
@ -1010,7 +1010,7 @@ var stateComputeStateCmd = &cli.Command{
|
||||
}
|
||||
|
||||
if cctx.Bool("html") {
|
||||
st, err := state.LoadStateTree(cbor.NewCborStore(apibstore.NewAPIBlockstore(api)), stout.Root)
|
||||
st, err := state.LoadStateTree(cbor.NewCborStore(blockstore.NewAPIBlockstore(api)), stout.Root)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("loading state tree: %w", err)
|
||||
}
|
||||
|
274
cli/util/api.go
Normal file
274
cli/util/api.go
Normal file
@ -0,0 +1,274 @@
|
||||
package cliutil
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/mitchellh/go-homedir"
|
||||
"github.com/urfave/cli/v2"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-jsonrpc"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/api/client"
|
||||
"github.com/filecoin-project/lotus/node/repo"
|
||||
)
|
||||
|
||||
const (
|
||||
metadataTraceContext = "traceContext"
|
||||
)
|
||||
|
||||
// The flag passed on the command line with the listen address of the API
|
||||
// server (only used by the tests)
|
||||
func flagForAPI(t repo.RepoType) string {
|
||||
switch t {
|
||||
case repo.FullNode:
|
||||
return "api-url"
|
||||
case repo.StorageMiner:
|
||||
return "miner-api-url"
|
||||
case repo.Worker:
|
||||
return "worker-api-url"
|
||||
default:
|
||||
panic(fmt.Sprintf("Unknown repo type: %v", t))
|
||||
}
|
||||
}
|
||||
|
||||
func flagForRepo(t repo.RepoType) string {
|
||||
switch t {
|
||||
case repo.FullNode:
|
||||
return "repo"
|
||||
case repo.StorageMiner:
|
||||
return "miner-repo"
|
||||
case repo.Worker:
|
||||
return "worker-repo"
|
||||
default:
|
||||
panic(fmt.Sprintf("Unknown repo type: %v", t))
|
||||
}
|
||||
}
|
||||
|
||||
func EnvForRepo(t repo.RepoType) string {
|
||||
switch t {
|
||||
case repo.FullNode:
|
||||
return "FULLNODE_API_INFO"
|
||||
case repo.StorageMiner:
|
||||
return "MINER_API_INFO"
|
||||
case repo.Worker:
|
||||
return "WORKER_API_INFO"
|
||||
default:
|
||||
panic(fmt.Sprintf("Unknown repo type: %v", t))
|
||||
}
|
||||
}
|
||||
|
||||
// TODO remove after deprecation period
|
||||
func envForRepoDeprecation(t repo.RepoType) string {
|
||||
switch t {
|
||||
case repo.FullNode:
|
||||
return "FULLNODE_API_INFO"
|
||||
case repo.StorageMiner:
|
||||
return "STORAGE_API_INFO"
|
||||
case repo.Worker:
|
||||
return "WORKER_API_INFO"
|
||||
default:
|
||||
panic(fmt.Sprintf("Unknown repo type: %v", t))
|
||||
}
|
||||
}
|
||||
|
||||
func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) {
|
||||
// Check if there was a flag passed with the listen address of the API
|
||||
// server (only used by the tests)
|
||||
apiFlag := flagForAPI(t)
|
||||
if ctx.IsSet(apiFlag) {
|
||||
strma := ctx.String(apiFlag)
|
||||
strma = strings.TrimSpace(strma)
|
||||
|
||||
return APIInfo{Addr: strma}, nil
|
||||
}
|
||||
|
||||
envKey := EnvForRepo(t)
|
||||
env, ok := os.LookupEnv(envKey)
|
||||
if !ok {
|
||||
// TODO remove after deprecation period
|
||||
envKey = envForRepoDeprecation(t)
|
||||
env, ok = os.LookupEnv(envKey)
|
||||
if ok {
|
||||
log.Warnf("Use deprecation env(%s) value, please use env(%s) instead.", envKey, EnvForRepo(t))
|
||||
}
|
||||
}
|
||||
if ok {
|
||||
return ParseApiInfo(env), nil
|
||||
}
|
||||
|
||||
repoFlag := flagForRepo(t)
|
||||
|
||||
p, err := homedir.Expand(ctx.String(repoFlag))
|
||||
if err != nil {
|
||||
return APIInfo{}, xerrors.Errorf("could not expand home dir (%s): %w", repoFlag, err)
|
||||
}
|
||||
|
||||
r, err := repo.NewFS(p)
|
||||
if err != nil {
|
||||
return APIInfo{}, xerrors.Errorf("could not open repo at path: %s; %w", p, err)
|
||||
}
|
||||
|
||||
ma, err := r.APIEndpoint()
|
||||
if err != nil {
|
||||
return APIInfo{}, xerrors.Errorf("could not get api endpoint: %w", err)
|
||||
}
|
||||
|
||||
token, err := r.APIToken()
|
||||
if err != nil {
|
||||
log.Warnf("Couldn't load CLI token, capabilities may be limited: %v", err)
|
||||
}
|
||||
|
||||
return APIInfo{
|
||||
Addr: ma.String(),
|
||||
Token: token,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func GetRawAPI(ctx *cli.Context, t repo.RepoType) (string, http.Header, error) {
|
||||
ainfo, err := GetAPIInfo(ctx, t)
|
||||
if err != nil {
|
||||
return "", nil, xerrors.Errorf("could not get API info: %w", err)
|
||||
}
|
||||
|
||||
addr, err := ainfo.DialArgs()
|
||||
if err != nil {
|
||||
return "", nil, xerrors.Errorf("could not get DialArgs: %w", err)
|
||||
}
|
||||
|
||||
return addr, ainfo.AuthHeader(), nil
|
||||
}
|
||||
|
||||
func GetAPI(ctx *cli.Context) (api.Common, jsonrpc.ClientCloser, error) {
|
||||
ti, ok := ctx.App.Metadata["repoType"]
|
||||
if !ok {
|
||||
log.Errorf("unknown repo type, are you sure you want to use GetAPI?")
|
||||
ti = repo.FullNode
|
||||
}
|
||||
t, ok := ti.(repo.RepoType)
|
||||
if !ok {
|
||||
log.Errorf("repoType type does not match the type of repo.RepoType")
|
||||
}
|
||||
|
||||
if tn, ok := ctx.App.Metadata["testnode-storage"]; ok {
|
||||
return tn.(api.StorageMiner), func() {}, nil
|
||||
}
|
||||
if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
|
||||
return tn.(api.FullNode), func() {}, nil
|
||||
}
|
||||
|
||||
addr, headers, err := GetRawAPI(ctx, t)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return client.NewCommonRPC(ctx.Context, addr, headers)
|
||||
}
|
||||
|
||||
func GetFullNodeAPI(ctx *cli.Context) (api.FullNode, jsonrpc.ClientCloser, error) {
|
||||
if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
|
||||
return tn.(api.FullNode), func() {}, nil
|
||||
}
|
||||
|
||||
addr, headers, err := GetRawAPI(ctx, repo.FullNode)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return client.NewFullNodeRPC(ctx.Context, addr, headers)
|
||||
}
|
||||
|
||||
type GetStorageMinerOptions struct {
|
||||
PreferHttp bool
|
||||
}
|
||||
|
||||
type GetStorageMinerOption func(*GetStorageMinerOptions)
|
||||
|
||||
func StorageMinerUseHttp(opts *GetStorageMinerOptions) {
|
||||
opts.PreferHttp = true
|
||||
}
|
||||
|
||||
func GetStorageMinerAPI(ctx *cli.Context, opts ...GetStorageMinerOption) (api.StorageMiner, jsonrpc.ClientCloser, error) {
|
||||
var options GetStorageMinerOptions
|
||||
for _, opt := range opts {
|
||||
opt(&options)
|
||||
}
|
||||
|
||||
if tn, ok := ctx.App.Metadata["testnode-storage"]; ok {
|
||||
return tn.(api.StorageMiner), func() {}, nil
|
||||
}
|
||||
|
||||
addr, headers, err := GetRawAPI(ctx, repo.StorageMiner)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if options.PreferHttp {
|
||||
u, err := url.Parse(addr)
|
||||
if err != nil {
|
||||
return nil, nil, xerrors.Errorf("parsing miner api URL: %w", err)
|
||||
}
|
||||
|
||||
switch u.Scheme {
|
||||
case "ws":
|
||||
u.Scheme = "http"
|
||||
case "wss":
|
||||
u.Scheme = "https"
|
||||
}
|
||||
|
||||
addr = u.String()
|
||||
}
|
||||
|
||||
return client.NewStorageMinerRPC(ctx.Context, addr, headers)
|
||||
}
|
||||
|
||||
func GetWorkerAPI(ctx *cli.Context) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
|
||||
addr, headers, err := GetRawAPI(ctx, repo.Worker)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return client.NewWorkerRPC(ctx.Context, addr, headers)
|
||||
}
|
||||
|
||||
func GetGatewayAPI(ctx *cli.Context) (api.GatewayAPI, jsonrpc.ClientCloser, error) {
|
||||
addr, headers, err := GetRawAPI(ctx, repo.FullNode)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return client.NewGatewayRPC(ctx.Context, addr, headers)
|
||||
}
|
||||
|
||||
func DaemonContext(cctx *cli.Context) context.Context {
|
||||
if mtCtx, ok := cctx.App.Metadata[metadataTraceContext]; ok {
|
||||
return mtCtx.(context.Context)
|
||||
}
|
||||
|
||||
return context.Background()
|
||||
}
|
||||
|
||||
// ReqContext returns context for cli execution. Calling it for the first time
|
||||
// installs SIGTERM handler that will close returned context.
|
||||
// Not safe for concurrent execution.
|
||||
func ReqContext(cctx *cli.Context) context.Context {
|
||||
tCtx := DaemonContext(cctx)
|
||||
|
||||
ctx, done := context.WithCancel(tCtx)
|
||||
sigChan := make(chan os.Signal, 2)
|
||||
go func() {
|
||||
<-sigChan
|
||||
done()
|
||||
}()
|
||||
signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP)
|
||||
|
||||
return ctx
|
||||
}
|
@ -20,18 +20,17 @@ import (
|
||||
"github.com/cockroachdb/pebble"
|
||||
"github.com/cockroachdb/pebble/bloom"
|
||||
"github.com/ipfs/go-cid"
|
||||
metricsi "github.com/ipfs/go-metrics-interface"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
badgerbs "github.com/filecoin-project/lotus/blockstore/badger"
|
||||
"github.com/filecoin-project/lotus/chain/stmgr"
|
||||
"github.com/filecoin-project/lotus/chain/store"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/chain/vm"
|
||||
lcli "github.com/filecoin-project/lotus/cli"
|
||||
"github.com/filecoin-project/lotus/lib/blockstore"
|
||||
badgerbs "github.com/filecoin-project/lotus/lib/blockstore/badger"
|
||||
_ "github.com/filecoin-project/lotus/lib/sigs/bls"
|
||||
_ "github.com/filecoin-project/lotus/lib/sigs/secp"
|
||||
"github.com/filecoin-project/lotus/node/repo"
|
||||
@ -204,7 +203,7 @@ var importBenchCmd = &cli.Command{
|
||||
case cctx.Bool("use-native-badger"):
|
||||
log.Info("using native badger")
|
||||
var opts badgerbs.Options
|
||||
if opts, err = repo.BadgerBlockstoreOptions(repo.BlockstoreChain, tdir, false); err != nil {
|
||||
if opts, err = repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, tdir, false); err != nil {
|
||||
return err
|
||||
}
|
||||
opts.SyncWrites = false
|
||||
@ -229,21 +228,13 @@ var importBenchCmd = &cli.Command{
|
||||
if ds != nil {
|
||||
ds = measure.New("dsbench", ds)
|
||||
defer ds.Close() //nolint:errcheck
|
||||
bs = blockstore.NewBlockstore(ds)
|
||||
bs = blockstore.FromDatastore(ds)
|
||||
}
|
||||
|
||||
if c, ok := bs.(io.Closer); ok {
|
||||
defer c.Close() //nolint:errcheck
|
||||
}
|
||||
|
||||
ctx := metricsi.CtxScope(context.Background(), "lotus")
|
||||
cacheOpts := blockstore.DefaultCacheOpts()
|
||||
cacheOpts.HasBloomFilterSize = 0
|
||||
bs, err = blockstore.CachedBlockstore(ctx, bs, cacheOpts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var verifier ffiwrapper.Verifier = ffiwrapper.ProofVerifier
|
||||
if cctx.IsSet("syscall-cache") {
|
||||
scds, err := badger.NewDatastore(cctx.String("syscall-cache"), &badger.DefaultOptions)
|
||||
@ -267,6 +258,15 @@ var importBenchCmd = &cli.Command{
|
||||
|
||||
stm := stmgr.NewStateManager(cs)
|
||||
|
||||
var carFile *os.File
|
||||
// open the CAR file if one is provided.
|
||||
if path := cctx.String("car"); path != "" {
|
||||
var err error
|
||||
if carFile, err = os.Open(path); err != nil {
|
||||
return xerrors.Errorf("failed to open provided CAR file: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
startTime := time.Now()
|
||||
|
||||
// register a gauge that reports how long since the measurable
|
||||
@ -308,18 +308,7 @@ var importBenchCmd = &cli.Command{
|
||||
writeProfile("allocs")
|
||||
}()
|
||||
|
||||
var carFile *os.File
|
||||
|
||||
// open the CAR file if one is provided.
|
||||
if path := cctx.String("car"); path != "" {
|
||||
var err error
|
||||
if carFile, err = os.Open(path); err != nil {
|
||||
return xerrors.Errorf("failed to open provided CAR file: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
var head *types.TipSet
|
||||
|
||||
// --- IMPORT ---
|
||||
if !cctx.Bool("no-import") {
|
||||
if cctx.Bool("global-profile") {
|
||||
|
@ -15,7 +15,7 @@ import (
|
||||
"github.com/filecoin-project/go-state-types/big"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/api/apibstore"
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/power"
|
||||
"github.com/filecoin-project/lotus/chain/events/state"
|
||||
@ -202,7 +202,7 @@ func (p *Processor) processMiners(ctx context.Context, minerTips map[types.TipSe
|
||||
log.Debugw("Processed Miners", "duration", time.Since(start).String())
|
||||
}()
|
||||
|
||||
stor := store.ActorStore(ctx, apibstore.NewAPIBlockstore(p.node))
|
||||
stor := store.ActorStore(ctx, blockstore.NewAPIBlockstore(p.node))
|
||||
|
||||
var out []minerActorInfo
|
||||
// TODO add parallel calls if this becomes slow
|
||||
@ -649,7 +649,7 @@ func (p *Processor) getMinerStateAt(ctx context.Context, maddr address.Address,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return miner.Load(store.ActorStore(ctx, apibstore.NewAPIBlockstore(p.node)), prevActor)
|
||||
return miner.Load(store.ActorStore(ctx, blockstore.NewAPIBlockstore(p.node)), prevActor)
|
||||
}
|
||||
|
||||
func (p *Processor) getMinerPreCommitChanges(ctx context.Context, m minerActorInfo) (*miner.PreCommitChanges, error) {
|
||||
|
@ -34,7 +34,7 @@ var (
|
||||
// gatewayDepsAPI defines the API methods that the GatewayAPI depends on
|
||||
// (to make it easy to mock for tests)
|
||||
type gatewayDepsAPI interface {
|
||||
Version(context.Context) (api.Version, error)
|
||||
Version(context.Context) (api.APIVersion, error)
|
||||
ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error)
|
||||
ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error)
|
||||
ChainGetNode(ctx context.Context, p string) (*api.IpldObject, error)
|
||||
@ -130,7 +130,7 @@ func (a *GatewayAPI) checkTimestamp(at time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *GatewayAPI) Version(ctx context.Context) (api.Version, error) {
|
||||
func (a *GatewayAPI) Version(ctx context.Context) (api.APIVersion, error) {
|
||||
return a.api.Version(ctx)
|
||||
}
|
||||
|
||||
|
@ -77,7 +77,7 @@ var runCmd = &cli.Command{
|
||||
|
||||
// Register all metric views
|
||||
if err := view.Register(
|
||||
metrics.DefaultViews...,
|
||||
metrics.ChainNodeViews...,
|
||||
); err != nil {
|
||||
log.Fatalf("Cannot register the view: %v", err)
|
||||
}
|
||||
|
@ -31,6 +31,7 @@ import (
|
||||
"github.com/filecoin-project/lotus/api/apistruct"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
lcli "github.com/filecoin-project/lotus/cli"
|
||||
cliutil "github.com/filecoin-project/lotus/cli/util"
|
||||
sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
|
||||
@ -49,7 +50,7 @@ const FlagWorkerRepo = "worker-repo"
|
||||
const FlagWorkerRepoDeprecation = "workerrepo"
|
||||
|
||||
func main() {
|
||||
build.RunningNodeType = build.NodeWorker
|
||||
api.RunningNodeType = api.NodeWorker
|
||||
|
||||
lotuslog.SetupLogLevels()
|
||||
|
||||
@ -183,7 +184,7 @@ var runCmd = &cli.Command{
|
||||
var closer func()
|
||||
var err error
|
||||
for {
|
||||
nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx, lcli.StorageMinerUseHttp)
|
||||
nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx, cliutil.StorageMinerUseHttp)
|
||||
if err == nil {
|
||||
_, err = nodeApi.Version(ctx)
|
||||
if err == nil {
|
||||
@ -210,8 +211,8 @@ var runCmd = &cli.Command{
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if v.APIVersion != build.MinerAPIVersion {
|
||||
return xerrors.Errorf("lotus-miner API version doesn't match: expected: %s", api.Version{APIVersion: build.MinerAPIVersion})
|
||||
if v.APIVersion != api.MinerAPIVersion {
|
||||
return xerrors.Errorf("lotus-miner API version doesn't match: expected: %s", api.APIVersion{APIVersion: api.MinerAPIVersion})
|
||||
}
|
||||
log.Infof("Remote version %s", v)
|
||||
|
||||
|
@ -8,7 +8,7 @@ import (
|
||||
"github.com/mitchellh/go-homedir"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
|
||||
@ -23,8 +23,8 @@ type worker struct {
|
||||
disabled int64
|
||||
}
|
||||
|
||||
func (w *worker) Version(context.Context) (build.Version, error) {
|
||||
return build.WorkerAPIVersion, nil
|
||||
func (w *worker) Version(context.Context) (api.Version, error) {
|
||||
return api.WorkerAPIVersion, nil
|
||||
}
|
||||
|
||||
func (w *worker) StorageAddLocal(ctx context.Context, path string) error {
|
||||
|
@ -6,6 +6,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/docker/go-units"
|
||||
"github.com/google/uuid"
|
||||
"github.com/mitchellh/go-homedir"
|
||||
"github.com/urfave/cli/v2"
|
||||
@ -46,6 +47,10 @@ var storageAttachCmd = &cli.Command{
|
||||
Name: "store",
|
||||
Usage: "(for init) use path for long-term storage",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "max-storage",
|
||||
Usage: "(for init) limit storage space for sectors (expensive for very large paths!)",
|
||||
},
|
||||
},
|
||||
Action: func(cctx *cli.Context) error {
|
||||
nodeApi, closer, err := lcli.GetWorkerAPI(cctx)
|
||||
@ -79,11 +84,20 @@ var storageAttachCmd = &cli.Command{
|
||||
return err
|
||||
}
|
||||
|
||||
var maxStor int64
|
||||
if cctx.IsSet("max-storage") {
|
||||
maxStor, err = units.RAMInBytes(cctx.String("max-storage"))
|
||||
if err != nil {
|
||||
return xerrors.Errorf("parsing max-storage: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
cfg := &stores.LocalStorageMeta{
|
||||
ID: stores.ID(uuid.New().String()),
|
||||
Weight: cctx.Uint64("weight"),
|
||||
CanSeal: cctx.Bool("seal"),
|
||||
CanStore: cctx.Bool("store"),
|
||||
ID: stores.ID(uuid.New().String()),
|
||||
Weight: cctx.Uint64("weight"),
|
||||
CanSeal: cctx.Bool("seal"),
|
||||
CanStore: cctx.Bool("store"),
|
||||
MaxStorage: uint64(maxStor),
|
||||
}
|
||||
|
||||
if !(cfg.CanStore || cfg.CanSeal) {
|
||||
|
@ -9,10 +9,10 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/chain/vm"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
|
||||
"github.com/filecoin-project/lotus/journal"
|
||||
"github.com/filecoin-project/lotus/lib/blockstore"
|
||||
"github.com/filecoin-project/lotus/node/modules/testing"
|
||||
"github.com/google/uuid"
|
||||
"github.com/mitchellh/go-homedir"
|
||||
@ -37,6 +37,8 @@ var genesisCmd = &cli.Command{
|
||||
genesisNewCmd,
|
||||
genesisAddMinerCmd,
|
||||
genesisAddMsigsCmd,
|
||||
genesisSetVRKCmd,
|
||||
genesisSetRemainderCmd,
|
||||
genesisCarCmd,
|
||||
},
|
||||
}
|
||||
@ -309,6 +311,200 @@ func parseMultisigCsv(csvf string) ([]GenAccountEntry, error) {
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
var genesisSetVRKCmd = &cli.Command{
|
||||
Name: "set-vrk",
|
||||
Usage: "Set the verified registry's root key",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "multisig",
|
||||
Usage: "CSV file to parse the multisig that will be set as the root key",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "account",
|
||||
Usage: "pubkey address that will be set as the root key (must NOT be declared anywhere else, since it must be given ID 80)",
|
||||
},
|
||||
},
|
||||
Action: func(cctx *cli.Context) error {
|
||||
if cctx.Args().Len() != 1 {
|
||||
return fmt.Errorf("must specify template file")
|
||||
}
|
||||
|
||||
genf, err := homedir.Expand(cctx.Args().First())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
csvf, err := homedir.Expand(cctx.Args().Get(1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var template genesis.Template
|
||||
b, err := ioutil.ReadFile(genf)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("read genesis template: %w", err)
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(b, &template); err != nil {
|
||||
return xerrors.Errorf("unmarshal genesis template: %w", err)
|
||||
}
|
||||
|
||||
if cctx.IsSet("account") {
|
||||
addr, err := address.NewFromString(cctx.String("account"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
am := genesis.AccountMeta{Owner: addr}
|
||||
|
||||
template.VerifregRootKey = genesis.Actor{
|
||||
Type: genesis.TAccount,
|
||||
Balance: big.Zero(),
|
||||
Meta: am.ActorMeta(),
|
||||
}
|
||||
} else if cctx.IsSet("multisig") {
|
||||
|
||||
entries, err := parseMultisigCsv(csvf)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("parsing multisig csv file: %w", err)
|
||||
}
|
||||
|
||||
if len(entries) == 0 {
|
||||
return xerrors.Errorf("no msig entries in csv file: %w", err)
|
||||
}
|
||||
|
||||
e := entries[0]
|
||||
if len(e.Addresses) != e.N {
|
||||
return fmt.Errorf("entry had mismatch between 'N' and number of addresses")
|
||||
}
|
||||
|
||||
msig := &genesis.MultisigMeta{
|
||||
Signers: e.Addresses,
|
||||
Threshold: e.M,
|
||||
VestingDuration: monthsToBlocks(e.VestingMonths),
|
||||
VestingStart: 0,
|
||||
}
|
||||
|
||||
act := genesis.Actor{
|
||||
Type: genesis.TMultisig,
|
||||
Balance: abi.TokenAmount(e.Amount),
|
||||
Meta: msig.ActorMeta(),
|
||||
}
|
||||
|
||||
template.VerifregRootKey = act
|
||||
} else {
|
||||
return xerrors.Errorf("must include either --account or --multisig flag")
|
||||
}
|
||||
|
||||
b, err = json.MarshalIndent(&template, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := ioutil.WriteFile(genf, b, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var genesisSetRemainderCmd = &cli.Command{
|
||||
Name: "set-remainder",
|
||||
Usage: "Set the remainder actor",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "multisig",
|
||||
Usage: "CSV file to parse the multisig that will be set as the remainder actor",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "account",
|
||||
Usage: "pubkey address that will be set as the remainder key (must NOT be declared anywhere else, since it must be given ID 90)",
|
||||
},
|
||||
},
|
||||
Action: func(cctx *cli.Context) error {
|
||||
if cctx.Args().Len() != 1 {
|
||||
return fmt.Errorf("must specify template file")
|
||||
}
|
||||
|
||||
genf, err := homedir.Expand(cctx.Args().First())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
csvf, err := homedir.Expand(cctx.Args().Get(1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var template genesis.Template
|
||||
b, err := ioutil.ReadFile(genf)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("read genesis template: %w", err)
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(b, &template); err != nil {
|
||||
return xerrors.Errorf("unmarshal genesis template: %w", err)
|
||||
}
|
||||
|
||||
if cctx.IsSet("account") {
|
||||
addr, err := address.NewFromString(cctx.String("account"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
am := genesis.AccountMeta{Owner: addr}
|
||||
|
||||
template.RemainderAccount = genesis.Actor{
|
||||
Type: genesis.TAccount,
|
||||
Balance: big.Zero(),
|
||||
Meta: am.ActorMeta(),
|
||||
}
|
||||
} else if cctx.IsSet("multisig") {
|
||||
|
||||
entries, err := parseMultisigCsv(csvf)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("parsing multisig csv file: %w", err)
|
||||
}
|
||||
|
||||
if len(entries) == 0 {
|
||||
return xerrors.Errorf("no msig entries in csv file: %w", err)
|
||||
}
|
||||
|
||||
e := entries[0]
|
||||
if len(e.Addresses) != e.N {
|
||||
return fmt.Errorf("entry had mismatch between 'N' and number of addresses")
|
||||
}
|
||||
|
||||
msig := &genesis.MultisigMeta{
|
||||
Signers: e.Addresses,
|
||||
Threshold: e.M,
|
||||
VestingDuration: monthsToBlocks(e.VestingMonths),
|
||||
VestingStart: 0,
|
||||
}
|
||||
|
||||
act := genesis.Actor{
|
||||
Type: genesis.TMultisig,
|
||||
Balance: abi.TokenAmount(e.Amount),
|
||||
Meta: msig.ActorMeta(),
|
||||
}
|
||||
|
||||
template.RemainderAccount = act
|
||||
} else {
|
||||
return xerrors.Errorf("must include either --account or --multisig flag")
|
||||
}
|
||||
|
||||
b, err = json.MarshalIndent(&template, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := ioutil.WriteFile(genf, b, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var genesisCarCmd = &cli.Command{
|
||||
Name: "car",
|
||||
Description: "write genesis car file",
|
||||
@ -327,7 +523,7 @@ var genesisCarCmd = &cli.Command{
|
||||
}
|
||||
ofile := c.String("out")
|
||||
jrnl := journal.NilJournal()
|
||||
bstor := blockstore.NewTemporarySync()
|
||||
bstor := blockstore.NewMemorySync()
|
||||
sbldr := vm.Syscalls(ffiwrapper.ProofVerifier)
|
||||
_, err := testing.MakeGenesis(ofile, c.Args().First())(bstor, sbldr, jrnl)()
|
||||
return err
|
||||
|
@ -175,7 +175,7 @@ var chainBalanceStateCmd = &cli.Command{
|
||||
|
||||
defer lkrepo.Close() //nolint:errcheck
|
||||
|
||||
bs, err := lkrepo.Blockstore(ctx, repo.BlockstoreChain)
|
||||
bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open blockstore: %w", err)
|
||||
}
|
||||
@ -396,7 +396,7 @@ var chainPledgeCmd = &cli.Command{
|
||||
|
||||
defer lkrepo.Close() //nolint:errcheck
|
||||
|
||||
bs, err := lkrepo.Blockstore(ctx, repo.BlockstoreChain)
|
||||
bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to open blockstore: %w", err)
|
||||
}
|
||||
|
@ -36,7 +36,7 @@ type consensusItem struct {
|
||||
targetTipset *types.TipSet
|
||||
headTipset *types.TipSet
|
||||
peerID peer.ID
|
||||
version api.Version
|
||||
version api.APIVersion
|
||||
api api.FullNode
|
||||
}
|
||||
|
||||
|
@ -319,7 +319,7 @@ var datastoreRewriteCmd = &cli.Command{
|
||||
)
|
||||
|
||||
// open the destination (to) store.
|
||||
opts, err := repo.BadgerBlockstoreOptions(repo.BlockstoreChain, toPath, false)
|
||||
opts, err := repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, toPath, false)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to get badger options: %w", err)
|
||||
}
|
||||
@ -329,7 +329,7 @@ var datastoreRewriteCmd = &cli.Command{
|
||||
}
|
||||
|
||||
// open the source (from) store.
|
||||
opts, err = repo.BadgerBlockstoreOptions(repo.BlockstoreChain, fromPath, true)
|
||||
opts, err = repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, fromPath, true)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to get badger options: %w", err)
|
||||
}
|
||||
|
@ -72,7 +72,7 @@ var exportChainCmd = &cli.Command{
|
||||
|
||||
defer fi.Close() //nolint:errcheck
|
||||
|
||||
bs, err := lr.Blockstore(ctx, repo.BlockstoreChain)
|
||||
bs, err := lr.Blockstore(ctx, repo.UniversalBlockstore)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open blockstore: %w", err)
|
||||
}
|
||||
|
@ -17,6 +17,7 @@ import (
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-state-types/big"
|
||||
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/actors/adt"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/account"
|
||||
@ -26,7 +27,6 @@ import (
|
||||
"github.com/filecoin-project/lotus/chain/stmgr"
|
||||
"github.com/filecoin-project/lotus/chain/store"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/lib/blockstore"
|
||||
)
|
||||
|
||||
type addrInfo struct {
|
||||
@ -50,7 +50,7 @@ var genesisVerifyCmd = &cli.Command{
|
||||
if !cctx.Args().Present() {
|
||||
return fmt.Errorf("must pass genesis car file")
|
||||
}
|
||||
bs := blockstore.NewBlockstore(datastore.NewMapDatastore())
|
||||
bs := blockstore.FromDatastore(datastore.NewMapDatastore())
|
||||
|
||||
cs := store.NewChainStore(bs, bs, datastore.NewMapDatastore(), nil, nil)
|
||||
defer cs.Close() //nolint:errcheck
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user