cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge branch 'master' into sbansal/nonce-coordination-and-consensus-for-chain-nodes

Browse Source
This commit is contained in:
Shrenuj Bansal
2022-11-11 14:41:38 -05:00
parent 09e9562cac 6104567fb1
commit 2fa21ff091
261 changed files with 7497 additions and 19074 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node/builder_chain.go
+3 -1
View File
@@ -31,6 +31,7 @@ import (
"github.com/filecoin-project/lotus/chain/wallet/remotewallet"
consensus2 "github.com/filecoin-project/lotus/lib/consensus/raft"
"github.com/filecoin-project/lotus/lib/peermgr"
"github.com/filecoin-project/lotus/markets/retrievaladapter"
"github.com/filecoin-project/lotus/markets/storageadapter"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/hello"
@@ -132,6 +133,7 @@ var ChainNode = Options(
Override(new(*market.FundManager), market.NewFundManager),
Override(new(dtypes.ClientDatastore), modules.NewClientDatastore),
Override(new(storagemarket.BlockstoreAccessor), modules.StorageBlockstoreAccessor),
Override(new(*retrievaladapter.APIBlockstoreAccessor), retrievaladapter.NewAPIBlockstoreAdapter),
Override(new(storagemarket.StorageClient), modules.StorageClient),
Override(new(storagemarket.StorageClientNode), storageadapter.NewClientNodeAdapter),
Override(HandleMigrateClientFundsKey, modules.HandleMigrateClientFunds),
@@ -184,7 +186,7 @@ func ConfigFullNode(c interface{}) Option {
Override(new(dtypes.UniversalBlockstore), modules.UniversalBlockstore),
If(cfg.Chainstore.EnableSplitstore,
If(cfg.Chainstore.Splitstore.ColdStoreType == "universal",
If(cfg.Chainstore.Splitstore.ColdStoreType == "universal" || cfg.Chainstore.Splitstore.ColdStoreType == "messages",
Override(new(dtypes.ColdBlockstore), From(new(dtypes.UniversalBlockstore)))),
If(cfg.Chainstore.Splitstore.ColdStoreType == "discard",
Override(new(dtypes.ColdBlockstore), modules.DiscardColdBlockstore)),
node/builder_miner.go
+2 -1
View File
@@ -224,7 +224,8 @@ func ConfigStorageMiner(c interface{}) Option {
Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(&cfg.Fees, &cfg.Dealmaking)),
),
Override(new(sectorstorage.Config), cfg.StorageManager()),
Override(new(config.SealerConfig), cfg.Storage),
Override(new(config.ProvingConfig), cfg.Proving),
Override(new(*ctladdr.AddressSelector), modules.AddressSelector(&cfg.Addresses)),
)
}
node/config/def.go
+17 -5
View File
@@ -15,7 +15,6 @@ import (
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/actors/policy"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/storage/sealer"
)
const (
@@ -91,12 +90,11 @@ func DefaultFullNode() *FullNode {
Chainstore: Chainstore{
EnableSplitstore: false,
Splitstore: Splitstore{
ColdStoreType: "universal",
ColdStoreType: "messages",
HotStoreType: "badger",
MarkSetType: "badger",
HotStoreFullGCFrequency: 20,
ColdStoreFullGCFrequency: 7,
HotStoreFullGCFrequency: 20,
},
},
Raft: *DefaultUserRaftConfig(),
@@ -164,7 +162,7 @@ func DefaultStorageMiner() *StorageMiner {
Assigner: "utilization",
// By default use the hardware resource filtering strategy.
ResourceFiltering: sealer.ResourceFilteringHardware,
ResourceFiltering: ResourceFilteringHardware,
},
Dealmaking: DealmakingConfig{
@@ -277,6 +275,20 @@ func (dur Duration) MarshalText() ([]byte, error) {
return []byte(d.String()), nil
}
// ResourceFilteringStrategy is an enum indicating the kinds of resource
// filtering strategies that can be configured for workers.
type ResourceFilteringStrategy string
const (
// ResourceFilteringHardware specifies that available hardware resources
// should be evaluated when scheduling a task against the worker.
ResourceFilteringHardware = ResourceFilteringStrategy("hardware")
// ResourceFilteringDisabled disables resource filtering against this
// worker. The scheduler may assign any task to this worker.
ResourceFilteringDisabled = ResourceFilteringStrategy("disabled")
)
var (
DefaultDataSubFolder = "raft"
DefaultWaitForLeaderTimeout = 15 * time.Second
node/config/dep_test.go
+34
View File
@@ -0,0 +1,34 @@
package config
import (
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
)
func goCmd() string {
var exeSuffix string
if runtime.GOOS == "windows" {
exeSuffix = ".exe"
}
path := filepath.Join(runtime.GOROOT(), "bin", "go"+exeSuffix)
if _, err := os.Stat(path); err == nil {
return path
}
return "go"
}
func TestDoesntDependOnFFI(t *testing.T) {
deps, err := exec.Command(goCmd(), "list", "-deps", "github.com/filecoin-project/lotus/node/config").Output()
if err != nil {
t.Fatal(err)
}
for _, pkg := range strings.Fields(string(deps)) {
if pkg == "github.com/filecoin-project/filecoin-ffi" {
t.Fatal("config depends on filecoin-ffi")
}
}
}
node/config/doc_gen.go
+2 -26
View File
@@ -900,7 +900,7 @@ If you see stuck Finalize tasks after enabling this setting, check
},
{
Name: "ResourceFiltering",
Type: "sealer.ResourceFilteringStrategy",
Type: "ResourceFilteringStrategy",
Comment: `ResourceFiltering instructs the system which resource filtering strategy
to use when evaluating tasks against this worker. An empty value defaults
@@ -1122,7 +1122,7 @@ submitting proofs to the chain individually`,
Type: "string",
Comment: `ColdStoreType specifies the type of the coldstore.
It can be "universal" (default) or "discard" for discarding cold blocks.`,
It can be "messages" (default) to store only messages, "universal" to store all chain state or "discard" for discarding cold blocks.`,
},
{
Name: "HotStoreType",
@@ -1153,30 +1153,6 @@ the compaction boundary; default is 0.`,
A value of 0 disables, while a value 1 will do full GC in every compaction.
Default is 20 (about once a week).`,
},
{
Name: "EnableColdStoreAutoPrune",
Type: "bool",
Comment: `EnableColdStoreAutoPrune turns on compaction of the cold store i.e. pruning
where hotstore compaction occurs every finality epochs pruning happens every 3 finalities
Default is false`,
},
{
Name: "ColdStoreFullGCFrequency",
Type: "uint64",
Comment: `ColdStoreFullGCFrequency specifies how often to performa a full (moving) GC on the coldstore.
Only applies if auto prune is enabled. A value of 0 disables while a value of 1 will do
full GC in every prune.
Default is 7 (about once every a week)`,
},
{
Name: "ColdStoreRetention",
Type: "int64",
Comment: `ColdStoreRetention specifies the retention policy for data reachable from the chain, in
finalities beyond the compaction boundary, default is 0, -1 retains everything`,
},
},
"StorageMiner": []DocField{
{
node/config/storage.go
+5 -31
View File
@@ -8,11 +8,10 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
func StorageFromFile(path string, def *paths.StorageConfig) (*paths.StorageConfig, error) {
func StorageFromFile(path string, def *storiface.StorageConfig) (*storiface.StorageConfig, error) {
file, err := os.Open(path)
switch {
case os.IsNotExist(err):
@@ -28,8 +27,8 @@ func StorageFromFile(path string, def *paths.StorageConfig) (*paths.StorageConfi
return StorageFromReader(file)
}
func StorageFromReader(reader io.Reader) (*paths.StorageConfig, error) {
var cfg paths.StorageConfig
func StorageFromReader(reader io.Reader) (*storiface.StorageConfig, error) {
var cfg storiface.StorageConfig
err := json.NewDecoder(reader).Decode(&cfg)
if err != nil {
return nil, err
@@ -38,7 +37,7 @@ func StorageFromReader(reader io.Reader) (*paths.StorageConfig, error) {
return &cfg, nil
}
func WriteStorageFile(path string, config paths.StorageConfig) error {
func WriteStorageFile(path string, config storiface.StorageConfig) error {
b, err := json.MarshalIndent(config, "", " ")
if err != nil {
return xerrors.Errorf("marshaling storage config: %w", err)
@@ -50,28 +49,3 @@ func WriteStorageFile(path string, config paths.StorageConfig) error {
return nil
}
func (c *StorageMiner) StorageManager() sealer.Config {
return sealer.Config{
ParallelFetchLimit: c.Storage.ParallelFetchLimit,
AllowSectorDownload: c.Storage.AllowSectorDownload,
AllowAddPiece: c.Storage.AllowAddPiece,
AllowPreCommit1: c.Storage.AllowPreCommit1,
AllowPreCommit2: c.Storage.AllowPreCommit2,
AllowCommit: c.Storage.AllowCommit,
AllowUnseal: c.Storage.AllowUnseal,
AllowReplicaUpdate: c.Storage.AllowReplicaUpdate,
AllowProveReplicaUpdate2: c.Storage.AllowProveReplicaUpdate2,
AllowRegenSectorKey: c.Storage.AllowRegenSectorKey,
ResourceFiltering: c.Storage.ResourceFiltering,
DisallowRemoteFinalize: c.Storage.DisallowRemoteFinalize,
LocalWorkerName: c.Storage.LocalWorkerName,
Assigner: c.Storage.Assigner,
ParallelCheckLimit: c.Proving.ParallelCheckLimit,
DisableBuiltinWindowPoSt: c.Proving.DisableBuiltinWindowPoSt,
DisableBuiltinWinningPoSt: c.Proving.DisableBuiltinWinningPoSt,
}
}
node/config/types.go
+2 -18
View File
@@ -4,7 +4,6 @@ import (
"github.com/ipfs/go-cid"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/storage/sealer"
)
// // NOTE: ONLY PUT STRUCT DEFINITIONS IN THIS FILE
@@ -453,7 +452,7 @@ type SealerConfig struct {
// ResourceFiltering instructs the system which resource filtering strategy
// to use when evaluating tasks against this worker. An empty value defaults
// to "hardware".
ResourceFiltering sealer.ResourceFilteringStrategy
ResourceFiltering ResourceFilteringStrategy
}
type BatchFeeConfig struct {
@@ -556,7 +555,7 @@ type Chainstore struct {
type Splitstore struct {
// ColdStoreType specifies the type of the coldstore.
// It can be "universal" (default) or "discard" for discarding cold blocks.
// It can be "messages" (default) to store only messages, "universal" to store all chain state or "discard" for discarding cold blocks.
ColdStoreType string
// HotStoreType specifies the type of the hotstore.
// Only currently supported value is "badger".
@@ -572,21 +571,6 @@ type Splitstore struct {
// A value of 0 disables, while a value 1 will do full GC in every compaction.
// Default is 20 (about once a week).
HotStoreFullGCFrequency uint64
// EnableColdStoreAutoPrune turns on compaction of the cold store i.e. pruning
// where hotstore compaction occurs every finality epochs pruning happens every 3 finalities
// Default is false
EnableColdStoreAutoPrune bool
// ColdStoreFullGCFrequency specifies how often to performa a full (moving) GC on the coldstore.
// Only applies if auto prune is enabled. A value of 0 disables while a value of 1 will do
// full GC in every prune.
// Default is 7 (about once every a week)
ColdStoreFullGCFrequency uint64
// ColdStoreRetention specifies the retention policy for data reachable from the chain, in
// finalities beyond the compaction boundary, default is 0, -1 retains everything
ColdStoreRetention int64
}
// // Full Node
node/impl/client/client.go
+37 -7
View File
@@ -10,6 +10,7 @@ import (
"os"
"sort"
"strings"
"sync"
"time"
"github.com/ipfs/go-blockservice"
@@ -52,7 +53,7 @@ import (
"github.com/filecoin-project/go-padreader"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
market8 "github.com/filecoin-project/go-state-types/builtin/v8/market"
markettypes "github.com/filecoin-project/go-state-types/builtin/v9/market"
"github.com/filecoin-project/go-state-types/dline"
"github.com/filecoin-project/lotus/api"
@@ -97,6 +98,7 @@ type API struct {
Imports dtypes.ClientImportMgr
StorageBlockstoreAccessor storagemarket.BlockstoreAccessor
RtvlBlockstoreAccessor rm.BlockstoreAccessor
ApiBlockstoreAccessor *retrievaladapter.APIBlockstoreAccessor
DataTransfer dtypes.ClientDataTransfer
Host host.Host
@@ -228,12 +230,12 @@ func (a *API) dealStarter(ctx context.Context, params *api.StartDealParams, isSt
// stateless flow from here to the end
//
label, err := market8.NewLabelFromString(params.Data.Root.Encode(multibase.MustNewEncoder('u')))
label, err := markettypes.NewLabelFromString(params.Data.Root.Encode(multibase.MustNewEncoder('u')))
if err != nil {
return nil, xerrors.Errorf("failed to encode label: %w", err)
}
dealProposal := &market8.DealProposal{
dealProposal := &markettypes.DealProposal{
PieceCID: *params.Data.PieceCid,
PieceSize: params.Data.PieceSize.Padded(),
Client: walletKey,
@@ -265,7 +267,7 @@ func (a *API) dealStarter(ctx context.Context, params *api.StartDealParams, isSt
return nil, xerrors.Errorf("failed to sign proposal : %w", err)
}
dealProposalSigned := &market8.ClientDealProposal{
dealProposalSigned := &markettypes.ClientDealProposal{
Proposal: *dealProposal,
ClientSignature: *dealProposalSig,
}
@@ -845,6 +847,13 @@ func (a *API) doRetrieval(ctx context.Context, order api.RetrievalOrder, sel dat
}
id := a.Retrieval.NextID()
if order.RemoteStore != nil {
if err := a.ApiBlockstoreAccessor.RegisterDealToRetrievalStore(id, *order.RemoteStore); err != nil {
return 0, xerrors.Errorf("registering api store: %w", err)
}
}
id, err = a.Retrieval.Retrieve(
ctx,
id,
@@ -999,6 +1008,8 @@ func (a *API) outputCAR(ctx context.Context, ds format.DAGService, bs bstore.Blo
roots[i] = dag.root
}
var lk sync.Mutex
return dest.doWrite(func(w io.Writer) error {
if err := car.WriteHeader(&car.CarHeader{
@@ -1011,13 +1022,29 @@ func (a *API) outputCAR(ctx context.Context, ds format.DAGService, bs bstore.Blo
cs := cid.NewSet()
for _, dagSpec := range dags {
dagSpec := dagSpec
if err := utils.TraverseDag(
ctx,
ds,
root,
dagSpec.selector,
func(node format.Node) error {
// if we're exporting merkle proofs for this dag, export all nodes read by the traversal
if dagSpec.exportAll {
lk.Lock()
defer lk.Unlock()
if cs.Visit(node.Cid()) {
err := util.LdWrite(w, node.Cid().Bytes(), node.RawData())
if err != nil {
return xerrors.Errorf("writing block data: %w", err)
}
}
}
return nil
},
func(p traversal.Progress, n ipld.Node, r traversal.VisitReason) error {
if r == traversal.VisitReason_SelectionMatch {
if !dagSpec.exportAll && r == traversal.VisitReason_SelectionMatch {
var c cid.Cid
if p.LastBlock.Link == nil {
c = root
@@ -1082,8 +1109,9 @@ func (a *API) outputUnixFS(ctx context.Context, root cid.Cid, ds format.DAGServi
}
type dagSpec struct {
root cid.Cid
selector ipld.Node
root cid.Cid
selector ipld.Node
exportAll bool
}
func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds format.DAGService, car bool) ([]dagSpec, error) {
@@ -1098,6 +1126,7 @@ func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds forma
out := make([]dagSpec, len(dsp))
for i, spec := range dsp {
out[i].exportAll = spec.ExportMerkleProof
if spec.DataSelector == nil {
return nil, xerrors.Errorf("invalid DagSpec at position %d: `DataSelector` can not be nil", i)
@@ -1131,6 +1160,7 @@ func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds forma
ds,
root,
rsn,
nil,
func(p traversal.Progress, n ipld.Node, r traversal.VisitReason) error {
if r == traversal.VisitReason_SelectionMatch {
if !car && p.LastBlock.Path.String() != p.Path.String() {
node/impl/full/state.go
+174 -11
View File
@@ -19,6 +19,7 @@ import (
actorstypes "github.com/filecoin-project/go-state-types/actors"
"github.com/filecoin-project/go-state-types/big"
minertypes "github.com/filecoin-project/go-state-types/builtin/v9/miner"
verifregtypes "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
"github.com/filecoin-project/go-state-types/cbor"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/go-state-types/dline"
@@ -30,6 +31,7 @@ import (
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/actors/builtin/datacap"
"github.com/filecoin-project/lotus/chain/actors/builtin/market"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/multisig"
@@ -769,6 +771,135 @@ func (m *StateModule) StateMarketStorageDeal(ctx context.Context, dealId abi.Dea
return stmgr.GetStorageDeal(ctx, m.StateManager, dealId, ts)
}
func (a *StateAPI) StateGetAllocationForPendingDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*verifregtypes.Allocation, error) {
ts, err := a.Chain.GetTipSetFromKey(ctx, tsk)
if err != nil {
return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
}
st, err := a.StateManager.GetMarketState(ctx, ts)
if err != nil {
return nil, err
}
allocationId, err := st.GetAllocationIdForPendingDeal(dealId)
if err != nil {
return nil, err
}
if allocationId == verifregtypes.NoAllocationID {
return nil, nil
}
dealState, err := a.StateMarketStorageDeal(ctx, dealId, tsk)
if err != nil {
return nil, err
}
return a.StateGetAllocation(ctx, dealState.Proposal.Client, allocationId, tsk)
}
func (a *StateAPI) StateGetAllocation(ctx context.Context, clientAddr address.Address, allocationId verifregtypes.AllocationId, tsk types.TipSetKey) (*verifregtypes.Allocation, error) {
idAddr, err := a.StateLookupID(ctx, clientAddr, tsk)
if err != nil {
return nil, err
}
ts, err := a.Chain.GetTipSetFromKey(ctx, tsk)
if err != nil {
return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
}
st, err := a.StateManager.GetVerifregState(ctx, ts)
if err != nil {
return nil, err
}
allocation, found, err := st.GetAllocation(idAddr, allocationId)
if err != nil {
return nil, xerrors.Errorf("getting allocation: %w", err)
}
if !found {
return nil, nil
}
return allocation, nil
}
func (a *StateAPI) StateGetAllocations(ctx context.Context, clientAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error) {
idAddr, err := a.StateLookupID(ctx, clientAddr, tsk)
if err != nil {
return nil, err
}
ts, err := a.Chain.GetTipSetFromKey(ctx, tsk)
if err != nil {
return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
}
st, err := a.StateManager.GetVerifregState(ctx, ts)
if err != nil {
return nil, xerrors.Errorf("loading verifreg state: %w", err)
}
allocations, err := st.GetAllocations(idAddr)
if err != nil {
return nil, xerrors.Errorf("getting allocations: %w", err)
}
return allocations, nil
}
func (a *StateAPI) StateGetClaim(ctx context.Context, providerAddr address.Address, claimId verifregtypes.ClaimId, tsk types.TipSetKey) (*verifregtypes.Claim, error) {
idAddr, err := a.StateLookupID(ctx, providerAddr, tsk)
if err != nil {
return nil, err
}
ts, err := a.Chain.GetTipSetFromKey(ctx, tsk)
if err != nil {
return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
}
st, err := a.StateManager.GetVerifregState(ctx, ts)
if err != nil {
return nil, err
}
claim, found, err := st.GetClaim(idAddr, claimId)
if err != nil {
return nil, xerrors.Errorf("getting claim: %w", err)
}
if !found {
return nil, nil
}
return claim, nil
}
func (a *StateAPI) StateGetClaims(ctx context.Context, providerAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error) {
idAddr, err := a.StateLookupID(ctx, providerAddr, tsk)
if err != nil {
return nil, err
}
ts, err := a.Chain.GetTipSetFromKey(ctx, tsk)
if err != nil {
return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
}
st, err := a.StateManager.GetVerifregState(ctx, ts)
if err != nil {
return nil, xerrors.Errorf("loading verifreg state: %w", err)
}
claims, err := st.GetClaims(idAddr)
if err != nil {
return nil, xerrors.Errorf("getting claims: %w", err)
}
return claims, nil
}
func (a *StateAPI) StateComputeDataCID(ctx context.Context, maddr address.Address, sectorType abi.RegisteredSealProof, deals []abi.DealID, tsk types.TipSetKey) (cid.Cid, error) {
nv, err := a.StateNetworkVersion(ctx, tsk)
if err != nil {
@@ -962,7 +1093,7 @@ func (a *StateAPI) StateListMessages(ctx context.Context, match *api.MessageMatc
_, err := a.StateLookupID(ctx, match.To, tsk)
// if the recipient doesn't exist at the start point, we're not gonna find any matches
if xerrors.Is(err, types.ErrActorNotFound) {
if xerrors.Is(err, &api.ErrActorNotFound{}) {
return nil, nil
}
@@ -973,7 +1104,7 @@ func (a *StateAPI) StateListMessages(ctx context.Context, match *api.MessageMatc
_, err := a.StateLookupID(ctx, match.From, tsk)
// if the sender doesn't exist at the start point, we're not gonna find any matches
if xerrors.Is(err, types.ErrActorNotFound) {
if xerrors.Is(err, &api.ErrActorNotFound{}) {
return nil, nil
}
@@ -1399,26 +1530,56 @@ func (a *StateAPI) StateVerifierStatus(ctx context.Context, addr address.Address
// Returns zero if there is no entry in the data cap table for the
// address.
func (m *StateModule) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) {
act, err := m.StateGetActor(ctx, verifreg.Address, tsk)
if err != nil {
return nil, err
}
aid, err := m.StateLookupID(ctx, addr, tsk)
if err != nil {
log.Warnf("lookup failure %v", err)
return nil, err
}
vrs, err := verifreg.Load(m.StateManager.ChainStore().ActorStore(ctx), act)
nv, err := m.StateNetworkVersion(ctx, tsk)
if err != nil {
return nil, xerrors.Errorf("failed to load verified registry state: %w", err)
return nil, err
}
verified, dcap, err := vrs.VerifiedClientDataCap(aid)
av, err := actorstypes.VersionForNetwork(nv)
if err != nil {
return nil, xerrors.Errorf("looking up verified client: %w", err)
return nil, err
}
var dcap abi.StoragePower
var verified bool
if av <= 8 {
act, err := m.StateGetActor(ctx, verifreg.Address, tsk)
if err != nil {
return nil, err
}
vrs, err := verifreg.Load(m.StateManager.ChainStore().ActorStore(ctx), act)
if err != nil {
return nil, xerrors.Errorf("failed to load verified registry state: %w", err)
}
verified, dcap, err = vrs.VerifiedClientDataCap(aid)
if err != nil {
return nil, xerrors.Errorf("looking up verified client: %w", err)
}
} else {
act, err := m.StateGetActor(ctx, datacap.Address, tsk)
if err != nil {
return nil, err
}
dcs, err := datacap.Load(m.StateManager.ChainStore().ActorStore(ctx), act)
if err != nil {
return nil, xerrors.Errorf("failed to load datacap actor state: %w", err)
}
verified, dcap, err = dcs.VerifiedClientDataCap(aid)
if err != nil {
return nil, xerrors.Errorf("looking up verified client: %w", err)
}
}
if !verified {
return nil, nil
}
@@ -1636,6 +1797,8 @@ func (a *StateAPI) StateGetNetworkParams(ctx context.Context) (*api.NetworkParam
UpgradeHyperdriveHeight: build.UpgradeHyperdriveHeight,
UpgradeChocolateHeight: build.UpgradeChocolateHeight,
UpgradeOhSnapHeight: build.UpgradeOhSnapHeight,
UpgradeSkyrHeight: build.UpgradeSkyrHeight,
UpgradeSharkHeight: build.UpgradeSharkHeight,
},
}, nil
}
node/modules/blockstore.go
+1 -3
View File
@@ -84,11 +84,9 @@ func SplitBlockstore(cfg *config.Chainstore) func(lc fx.Lifecycle, r repo.Locked
cfg := &splitstore.Config{
MarkSetType: cfg.Splitstore.MarkSetType,
DiscardColdBlocks: cfg.Splitstore.ColdStoreType == "discard",
UniversalColdBlocks: cfg.Splitstore.ColdStoreType == "universal",
HotStoreMessageRetention: cfg.Splitstore.HotStoreMessageRetention,
HotStoreFullGCFrequency: cfg.Splitstore.HotStoreFullGCFrequency,
EnableColdStoreAutoPrune: cfg.Splitstore.EnableColdStoreAutoPrune,
ColdStoreFullGCFrequency: cfg.Splitstore.ColdStoreFullGCFrequency,
ColdStoreRetention: cfg.Splitstore.ColdStoreRetention,
}
ss, err := splitstore.Open(path, ds, hot, cold, cfg)
if err != nil {
node/modules/client.go
+2 -2
View File
@@ -202,9 +202,9 @@ func StorageClient(lc fx.Lifecycle, h host.Host, dataTransfer dtypes.ClientDataT
// RetrievalClient creates a new retrieval client attached to the client blockstore
func RetrievalClient(forceOffChain bool) func(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, dt dtypes.ClientDataTransfer, payAPI payapi.PaychAPI, resolver discovery.PeerResolver,
ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI, accessor retrievalmarket.BlockstoreAccessor, j journal.Journal) (retrievalmarket.RetrievalClient, error) {
ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI, accessor *retrievaladapter.APIBlockstoreAccessor, j journal.Journal) (retrievalmarket.RetrievalClient, error) {
return func(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, dt dtypes.ClientDataTransfer, payAPI payapi.PaychAPI, resolver discovery.PeerResolver,
ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI, accessor retrievalmarket.BlockstoreAccessor, j journal.Journal) (retrievalmarket.RetrievalClient, error) {
ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI, accessor *retrievaladapter.APIBlockstoreAccessor, j journal.Journal) (retrievalmarket.RetrievalClient, error) {
adapter := retrievaladapter.NewRetrievalClientNode(forceOffChain, payAPI, chainAPI, stateAPI)
network := rmnet.NewFromLibp2pHost(h)
ds = namespace.Wrap(ds, datastore.NewKey("/retrievals/client"))
node/modules/storageminer.go
+3 -3
View File
@@ -787,17 +787,17 @@ func LocalStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls paths.LocalStorag
return paths.NewLocal(ctx, ls, si, urls)
}
func RemoteStorage(lstor *paths.Local, si paths.SectorIndex, sa sealer.StorageAuth, sc sealer.Config) *paths.Remote {
func RemoteStorage(lstor *paths.Local, si paths.SectorIndex, sa sealer.StorageAuth, sc config.SealerConfig) *paths.Remote {
return paths.NewRemote(lstor, si, http.Header(sa), sc.ParallelFetchLimit, &paths.DefaultPartialFileHandler{})
}
func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, lstor *paths.Local, stor paths.Store, ls paths.LocalStorage, si paths.SectorIndex, sc sealer.Config, ds dtypes.MetadataDS) (*sealer.Manager, error) {
func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, lstor *paths.Local, stor paths.Store, ls paths.LocalStorage, si paths.SectorIndex, sc config.SealerConfig, pc config.ProvingConfig, ds dtypes.MetadataDS) (*sealer.Manager, error) {
ctx := helpers.LifecycleCtx(mctx, lc)
wsts := statestore.New(namespace.Wrap(ds, WorkerCallsPrefix))
smsts := statestore.New(namespace.Wrap(ds, ManagerWorkPrefix))
sst, err := sealer.New(ctx, lstor, stor, ls, si, sc, wsts, smsts)
sst, err := sealer.New(ctx, lstor, stor, ls, si, sc, pc, wsts, smsts)
if err != nil {
return nil, err
}
node/repo/fsrepo.go
+6 -6
View File
@@ -25,8 +25,8 @@ import (
badgerbs "github.com/filecoin-project/lotus/blockstore/badger"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/fsutil"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
const (
@@ -572,26 +572,26 @@ func (fsr *fsLockedRepo) SetConfig(c func(interface{})) error {
return nil
}
func (fsr *fsLockedRepo) GetStorage() (paths.StorageConfig, error) {
func (fsr *fsLockedRepo) GetStorage() (storiface.StorageConfig, error) {
fsr.storageLk.Lock()
defer fsr.storageLk.Unlock()
return fsr.getStorage(nil)
}
func (fsr *fsLockedRepo) getStorage(def *paths.StorageConfig) (paths.StorageConfig, error) {
func (fsr *fsLockedRepo) getStorage(def *storiface.StorageConfig) (storiface.StorageConfig, error) {
c, err := config.StorageFromFile(fsr.join(fsStorageConfig), def)
if err != nil {
return paths.StorageConfig{}, err
return storiface.StorageConfig{}, err
}
return *c, nil
}
func (fsr *fsLockedRepo) SetStorage(c func(*paths.StorageConfig)) error {
func (fsr *fsLockedRepo) SetStorage(c func(*storiface.StorageConfig)) error {
fsr.storageLk.Lock()
defer fsr.storageLk.Unlock()
sc, err := fsr.getStorage(&paths.StorageConfig{})
sc, err := fsr.getStorage(&storiface.StorageConfig{})
if err != nil {
return xerrors.Errorf("get storage: %w", err)
}
node/repo/interface.go
+3 -3
View File
@@ -9,8 +9,8 @@ import (
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/fsutil"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
// BlockstoreDomain represents the domain of a blockstore.
@@ -73,8 +73,8 @@ type LockedRepo interface {
Config() (interface{}, error)
SetConfig(func(interface{})) error
GetStorage() (paths.StorageConfig, error)
SetStorage(func(*paths.StorageConfig)) error
GetStorage() (storiface.StorageConfig, error)
SetStorage(func(*storiface.StorageConfig)) error
Stat(path string) (fsutil.FsStat, error)
DiskUsage(path string) (int64, error)
node/repo/memrepo.go
+8 -9
View File
@@ -18,7 +18,6 @@ import (
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/fsutil"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
@@ -37,7 +36,7 @@ type MemRepo struct {
keystore map[string]types.KeyInfo
blockstore blockstore.Blockstore
sc *paths.StorageConfig
sc *storiface.StorageConfig
tempDir string
// holds the current config value
@@ -59,13 +58,13 @@ func (lmem *lockedMemRepo) RepoType() RepoType {
return lmem.t
}
func (lmem *lockedMemRepo) GetStorage() (paths.StorageConfig, error) {
func (lmem *lockedMemRepo) GetStorage() (storiface.StorageConfig, error) {
if err := lmem.checkToken(); err != nil {
return paths.StorageConfig{}, err
return storiface.StorageConfig{}, err
}
if lmem.mem.sc == nil {
lmem.mem.sc = &paths.StorageConfig{StoragePaths: []paths.LocalPath{
lmem.mem.sc = &storiface.StorageConfig{StoragePaths: []storiface.LocalPath{
{Path: lmem.Path()},
}}
}
@@ -73,7 +72,7 @@ func (lmem *lockedMemRepo) GetStorage() (paths.StorageConfig, error) {
return *lmem.mem.sc, nil
}
func (lmem *lockedMemRepo) SetStorage(c func(*paths.StorageConfig)) error {
func (lmem *lockedMemRepo) SetStorage(c func(*storiface.StorageConfig)) error {
if err := lmem.checkToken(); err != nil {
return err
}
@@ -126,14 +125,14 @@ func (lmem *lockedMemRepo) Path() string {
}
func (lmem *lockedMemRepo) initSectorStore(t string) {
if err := config.WriteStorageFile(filepath.Join(t, fsStorageConfig), paths.StorageConfig{
StoragePaths: []paths.LocalPath{
if err := config.WriteStorageFile(filepath.Join(t, fsStorageConfig), storiface.StorageConfig{
StoragePaths: []storiface.LocalPath{
{Path: t},
}}); err != nil {
panic(err)
}
b, err := json.MarshalIndent(&paths.LocalStorageMeta{
b, err := json.MarshalIndent(&storiface.LocalStorageMeta{
ID: storiface.ID(uuid.New().String()),
Weight: 10,
CanSeal: true,
node/rpc.go
+43
View File
@@ -3,13 +3,16 @@ package node
import (
"context"
"encoding/json"
"fmt"
"net"
"net/http"
_ "net/http/pprof"
"runtime"
"strconv"
"github.com/google/uuid"
"github.com/gorilla/mux"
"github.com/gorilla/websocket"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log/v2"
"github.com/multiformats/go-multiaddr"
@@ -23,6 +26,7 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/v0api"
"github.com/filecoin-project/lotus/api/v1api"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/lib/rpcenc"
"github.com/filecoin-project/lotus/metrics"
"github.com/filecoin-project/lotus/metrics/proxy"
@@ -92,6 +96,7 @@ func FullNodeHandler(a v1api.FullNode, permissioned bool, opts ...jsonrpc.Server
// Import handler
handleImportFunc := handleImport(a.(*impl.FullNodeAPI))
handleExportFunc := handleExport(a.(*impl.FullNodeAPI))
handleRemoteStoreFunc := handleRemoteStore(a.(*impl.FullNodeAPI))
if permissioned {
importAH := &auth.Handler{
Verify: a.AuthVerify,
@@ -104,9 +109,16 @@ func FullNodeHandler(a v1api.FullNode, permissioned bool, opts ...jsonrpc.Server
Next: handleExportFunc,
}
m.Handle("/rest/v0/export", exportAH)
storeAH := &auth.Handler{
Verify: a.AuthVerify,
Next: handleRemoteStoreFunc,
}
m.Handle("/rest/v0/store/{uuid}", storeAH)
} else {
m.HandleFunc("/rest/v0/import", handleImportFunc)
m.HandleFunc("/rest/v0/export", handleExportFunc)
m.HandleFunc("/rest/v0/store/{uuid}", handleRemoteStoreFunc)
}
// debugging
@@ -256,3 +268,34 @@ func handleFractionOpt(name string, setter func(int)) http.HandlerFunc {
setter(fr)
}
}
var upgrader = websocket.Upgrader{
CheckOrigin: func(r *http.Request) bool {
return true
},
}
func handleRemoteStore(a *impl.FullNodeAPI) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id, err := uuid.Parse(vars["uuid"])
if err != nil {
http.Error(w, fmt.Sprintf("parse uuid: %s", err), http.StatusBadRequest)
return
}
c, err := upgrader.Upgrade(w, r, nil)
if err != nil {
log.Error(err)
w.WriteHeader(500)
return
}
nstore := bstore.NewNetworkStoreWS(c)
if err := a.ApiBlockstoreAccessor.RegisterApiStore(id, nstore); err != nil {
log.Errorw("registering api bstore", "error", err)
_ = c.Close()
return
}
}
}