cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

integrate extern/sector-storage into lotus proper.

Browse Source
This commit is contained in:
Raúl Kripalani
2020-08-16 11:09:58 +01:00
parent fb0ccc9260
commit 3c17cd655e
113 changed files with 223 additions and 891 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node/builder.go
+10 -10
View File
@@ -61,10 +61,10 @@ import (
"github.com/filecoin-project/lotus/paychmgr/settler"
"github.com/filecoin-project/lotus/storage"
"github.com/filecoin-project/lotus/storage/sealing"
"github.com/filecoin-project/lotus/storage/sector"
"github.com/filecoin-project/lotus/storage/sector/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sector/stores"
"github.com/filecoin-project/lotus/storage/sectorblocks"
sectorstorage "github.com/filecoin-project/sector-storage"
"github.com/filecoin-project/sector-storage/ffiwrapper"
"github.com/filecoin-project/sector-storage/stores"
)
var log = logging.Logger("builder")
@@ -281,7 +281,7 @@ func Online() Option {
// miner
ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner },
Override(new(api.Common), From(new(common.CommonAPI))),
Override(new(sectorstorage.StorageAuth), modules.StorageAuth),
Override(new(sector.StorageAuth), modules.StorageAuth),
Override(new(*stores.Index), stores.NewIndex),
Override(new(stores.SectorIndex), From(new(*stores.Index))),
@@ -290,11 +290,11 @@ func Online() Option {
Override(new(*ffiwrapper.Config), modules.ProofsConfig),
Override(new(stores.LocalStorage), From(new(repo.LockedRepo))),
Override(new(sealing.SectorIDCounter), modules.SectorIDCounter),
Override(new(*sectorstorage.Manager), modules.SectorStorage),
Override(new(*sector.Manager), modules.SectorStorage),
Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier),
Override(new(sectorstorage.SectorManager), From(new(*sectorstorage.Manager))),
Override(new(storage2.Prover), From(new(sectorstorage.SectorManager))),
Override(new(sector.SectorManager), From(new(*sector.Manager))),
Override(new(storage2.Prover), From(new(sector.SectorManager))),
Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks),
Override(new(*storage.Miner), modules.StorageMiner(config.DefaultStorageMiner().Fees)),
@@ -369,10 +369,10 @@ func ConfigCommon(cfg *config.Common) Option {
Override(SetApiEndpointKey, func(lr repo.LockedRepo, e dtypes.APIEndpoint) error {
return lr.SetAPIEndpoint(e)
}),
Override(new(sectorstorage.URLs), func(e dtypes.APIEndpoint) (sectorstorage.URLs, error) {
Override(new(sector.URLs), func(e dtypes.APIEndpoint) (sector.URLs, error) {
ip := cfg.API.RemoteListenAddress
var urls sectorstorage.URLs
var urls sector.URLs
urls = append(urls, "http://"+ip+"/remote") // TODO: This makes no assumptions, and probably could...
return urls, nil
}),
@@ -430,7 +430,7 @@ func ConfigStorageMiner(c interface{}) Option {
Override(new(dtypes.DealFilter), modules.BasicDealFilter(dealfilter.CliDealFilter(cfg.Dealmaking.Filter))),
),
Override(new(sectorstorage.SealerConfig), cfg.Storage),
Override(new(sector.SealerConfig), cfg.Storage),
Override(new(*storage.Miner), modules.StorageMiner(cfg.Fees)),
)
}
node/config/def.go
+3 -4
View File
@@ -6,9 +6,8 @@ import (
"github.com/ipfs/go-cid"
sectorstorage "github.com/filecoin-project/sector-storage"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/storage/sector"
)
// Common is common config between full node and miner
@@ -32,7 +31,7 @@ type StorageMiner struct {
Common
Dealmaking DealmakingConfig
Storage sectorstorage.SealerConfig
Storage sector.SealerConfig
Fees MinerFeeConfig
SealingDelay Duration
@@ -132,7 +131,7 @@ func DefaultStorageMiner() *StorageMiner {
cfg := &StorageMiner{
Common: defCommon(),
Storage: sectorstorage.SealerConfig{
Storage: sector.SealerConfig{
AllowPreCommit1: true,
AllowPreCommit2: true,
AllowCommit: true,
node/config/storage.go
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/sector-storage/stores"
"github.com/filecoin-project/lotus/storage/sector/stores"
)
func StorageFromFile(path string, def *stores.StorageConfig) (*stores.StorageConfig, error) {
node/impl/client/client.go
+1 -1
View File
@@ -39,7 +39,7 @@ import (
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
"github.com/filecoin-project/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sector/ffiwrapper"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
node/impl/full/state.go
+1 -1
View File
@@ -15,7 +15,7 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
"github.com/filecoin-project/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sector/ffiwrapper"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/filecoin-project/specs-actors/actors/builtin"
node/impl/remoteworker.go
+2 -2
View File
@@ -12,7 +12,7 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/client"
"github.com/filecoin-project/sector-storage"
"github.com/filecoin-project/lotus/storage/sector"
)
type remoteWorker struct {
@@ -46,4 +46,4 @@ func (r *remoteWorker) Close() error {
return nil
}
var _ sectorstorage.Worker = &remoteWorker{}
var _ sector.Worker = &remoteWorker{}
node/impl/storminer.go
+7 -7
View File
@@ -8,7 +8,7 @@ import (
"strconv"
"time"
"github.com/filecoin-project/sector-storage/fsutil"
"github.com/filecoin-project/lotus/storage/sector/fsutil"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
@@ -19,10 +19,10 @@ import (
storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/filecoin-project/lotus/storage/sealing"
sectorstorage "github.com/filecoin-project/sector-storage"
"github.com/filecoin-project/sector-storage/ffiwrapper"
"github.com/filecoin-project/sector-storage/stores"
"github.com/filecoin-project/sector-storage/storiface"
"github.com/filecoin-project/lotus/storage/sector"
"github.com/filecoin-project/lotus/storage/sector/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sector/stores"
"github.com/filecoin-project/lotus/storage/sector/storiface"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/lotus/api"
@@ -47,8 +47,8 @@ type StorageMinerAPI struct {
Miner *storage.Miner
BlockMiner *miner.Miner
Full api.FullNode
StorageMgr *sectorstorage.Manager `optional:"true"`
IStorageMgr sectorstorage.SectorManager
StorageMgr *sector.Manager `optional:"true"`
IStorageMgr sector.SectorManager
*stores.Index
ConsiderOnlineStorageDealsConfigFunc dtypes.ConsiderOnlineStorageDealsConfigFunc
node/modules/chain.go
+1 -1
View File
@@ -16,7 +16,7 @@ import (
"go.uber.org/fx"
"golang.org/x/xerrors"
"github.com/filecoin-project/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sector/ffiwrapper"
"github.com/filecoin-project/lotus/chain"
"github.com/filecoin-project/lotus/chain/beacon"
node/modules/storageminer.go
+10 -10
View File
@@ -43,9 +43,9 @@ import (
paramfetch "github.com/filecoin-project/go-paramfetch"
"github.com/filecoin-project/go-storedcounter"
"github.com/filecoin-project/lotus/storage/sealing"
sectorstorage "github.com/filecoin-project/sector-storage"
"github.com/filecoin-project/sector-storage/ffiwrapper"
"github.com/filecoin-project/sector-storage/stores"
"github.com/filecoin-project/lotus/storage/sector"
"github.com/filecoin-project/lotus/storage/sector/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sector/stores"
"github.com/filecoin-project/specs-actors/actors/abi"
lapi "github.com/filecoin-project/lotus/api"
@@ -140,8 +140,8 @@ func SectorIDCounter(ds dtypes.MetadataDS) sealing.SectorIDCounter {
return &sidsc{sc}
}
func StorageMiner(fc config.MinerFeeConfig) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h host.Host, ds dtypes.MetadataDS, sealer sectorstorage.SectorManager, sc sealing.SectorIDCounter, verif ffiwrapper.Verifier, gsd dtypes.GetSealingDelayFunc) (*storage.Miner, error) {
return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h host.Host, ds dtypes.MetadataDS, sealer sectorstorage.SectorManager, sc sealing.SectorIDCounter, verif ffiwrapper.Verifier, gsd dtypes.GetSealingDelayFunc) (*storage.Miner, error) {
func StorageMiner(fc config.MinerFeeConfig) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h host.Host, ds dtypes.MetadataDS, sealer sector.SectorManager, sc sealing.SectorIDCounter, verif ffiwrapper.Verifier, gsd dtypes.GetSealingDelayFunc) (*storage.Miner, error) {
return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h host.Host, ds dtypes.MetadataDS, sealer sector.SectorManager, sc sealing.SectorIDCounter, verif ffiwrapper.Verifier, gsd dtypes.GetSealingDelayFunc) (*storage.Miner, error) {
maddr, err := minerAddrFromDS(ds)
if err != nil {
return nil, err
@@ -440,7 +440,7 @@ func StorageProvider(minerAddress dtypes.MinerAddress,
}
// RetrievalProvider creates a new retrieval provider attached to the provider blockstore
func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.SectorManager, full lapi.FullNode, ds dtypes.MetadataDS, pieceStore dtypes.ProviderPieceStore, mds dtypes.StagingMultiDstore, dt dtypes.ProviderDataTransfer, onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc, offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) (retrievalmarket.RetrievalProvider, error) {
func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sector.SectorManager, full lapi.FullNode, ds dtypes.MetadataDS, pieceStore dtypes.ProviderPieceStore, mds dtypes.StagingMultiDstore, dt dtypes.ProviderDataTransfer, onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc, offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) (retrievalmarket.RetrievalProvider, error) {
adapter := retrievaladapter.NewRetrievalProviderNode(miner, sealer, full)
maddr, err := minerAddrFromDS(ds)
@@ -476,10 +476,10 @@ func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.S
return retrievalimpl.NewProvider(maddr, adapter, netwk, pieceStore, mds, dt, namespace.Wrap(ds, datastore.NewKey("/retrievals/provider")), opt)
}
func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls stores.LocalStorage, si stores.SectorIndex, cfg *ffiwrapper.Config, sc sectorstorage.SealerConfig, urls sectorstorage.URLs, sa sectorstorage.StorageAuth) (*sectorstorage.Manager, error) {
func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls stores.LocalStorage, si stores.SectorIndex, cfg *ffiwrapper.Config, sc sector.SealerConfig, urls sector.URLs, sa sector.StorageAuth) (*sector.Manager, error) {
ctx := helpers.LifecycleCtx(mctx, lc)
sst, err := sectorstorage.New(ctx, ls, si, cfg, sc, urls, sa)
sst, err := sector.New(ctx, ls, si, cfg, sc, urls, sa)
if err != nil {
return nil, err
}
@@ -491,7 +491,7 @@ func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls stores.LocalStor
return sst, nil
}
func StorageAuth(ctx helpers.MetricsCtx, ca lapi.Common) (sectorstorage.StorageAuth, error) {
func StorageAuth(ctx helpers.MetricsCtx, ca lapi.Common) (sector.StorageAuth, error) {
token, err := ca.AuthNew(ctx, []auth.Permission{"admin"})
if err != nil {
return nil, xerrors.Errorf("creating storage auth header: %w", err)
@@ -499,7 +499,7 @@ func StorageAuth(ctx helpers.MetricsCtx, ca lapi.Common) (sectorstorage.StorageA
headers := http.Header{}
headers.Add("Authorization", "Bearer "+string(token))
return sectorstorage.StorageAuth(headers), nil
return sector.StorageAuth(headers), nil
}
func NewConsiderOnlineStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOnlineStorageDealsConfigFunc, error) {
node/node_test.go
+6 -6
View File
@@ -13,7 +13,7 @@ import (
"github.com/filecoin-project/lotus/lib/lotuslog"
"github.com/filecoin-project/lotus/storage/mockstorage"
"github.com/filecoin-project/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sector/ffiwrapper"
"github.com/filecoin-project/go-storedcounter"
"github.com/ipfs/go-datastore"
@@ -42,14 +42,14 @@ import (
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
genesis "github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node"
"github.com/filecoin-project/lotus/node/modules"
modtest "github.com/filecoin-project/lotus/node/modules/testing"
"github.com/filecoin-project/lotus/node/repo"
sectorstorage "github.com/filecoin-project/sector-storage"
"github.com/filecoin-project/sector-storage/mock"
"github.com/filecoin-project/lotus/storage/sector"
"github.com/filecoin-project/lotus/storage/sector/mock"
)
func init() {
@@ -422,11 +422,11 @@ func mockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test
}
storers[i] = testStorageNode(ctx, t, genms[i].Worker, maddrs[i], pidKeys[i], f, mn, node.Options(
node.Override(new(sectorstorage.SectorManager), func() (sectorstorage.SectorManager, error) {
node.Override(new(sector.SectorManager), func() (sector.SectorManager, error) {
return mock.NewMockSectorMgr(build.DefaultSectorSize(), sectors), nil
}),
node.Override(new(ffiwrapper.Verifier), mock.MockVerifier),
node.Unset(new(*sectorstorage.Manager)),
node.Unset(new(*sector.Manager)),
))
}
node/repo/fsrepo.go
+2 -2
View File
@@ -20,8 +20,8 @@ import (
"github.com/multiformats/go-multiaddr"
"golang.org/x/xerrors"
"github.com/filecoin-project/sector-storage/fsutil"
"github.com/filecoin-project/sector-storage/stores"
"github.com/filecoin-project/lotus/storage/sector/fsutil"
"github.com/filecoin-project/lotus/storage/sector/stores"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/config"
node/repo/interface.go
+2 -2
View File
@@ -6,8 +6,8 @@ import (
"github.com/ipfs/go-datastore"
"github.com/multiformats/go-multiaddr"
"github.com/filecoin-project/sector-storage/fsutil"
"github.com/filecoin-project/sector-storage/stores"
"github.com/filecoin-project/lotus/storage/sector/fsutil"
"github.com/filecoin-project/lotus/storage/sector/stores"
"github.com/filecoin-project/lotus/chain/types"
)
node/repo/memrepo.go
+2 -2
View File
@@ -14,11 +14,11 @@ import (
"github.com/multiformats/go-multiaddr"
"golang.org/x/xerrors"
"github.com/filecoin-project/sector-storage/fsutil"
"github.com/filecoin-project/lotus/storage/sector/fsutil"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/sector-storage/stores"
"github.com/filecoin-project/lotus/storage/sector/stores"
)
type MemRepo struct {