cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

sectorstorage: Untangle from lotus deps

Browse Source
This commit is contained in:
Łukasz Magiera
2020-03-27 22:35:33 +01:00
parent de1f2be505
commit c66d3dfad7
38 changed files with 192 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node/builder.go
+2 -1
View File
@@ -263,6 +263,7 @@ func Online() Option {
// Storage miner
ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner },
Override(new(api.Common), From(new(common.CommonAPI))),
Override(new(sectorstorage.StorageAuth), modules.StorageAuth),
Override(new(*stores.Index), stores.NewIndex),
Override(new(stores.SectorIndex), From(new(*stores.Index))),
@@ -384,7 +385,7 @@ func ConfigStorageMiner(c interface{}) Option {
return Options(
ConfigCommon(&cfg.Common),
Override(new(config.Storage), cfg.Storage),
Override(new(sectorstorage.SealerConfig), cfg.Storage),
)
}
node/config/def.go
+3 -10
View File
@@ -2,6 +2,7 @@ package config
import (
"encoding"
"github.com/filecoin-project/lotus/storage/sectorstorage"
"time"
)
@@ -23,7 +24,7 @@ type FullNode struct {
type StorageMiner struct {
Common
Storage Storage
Storage sectorstorage.SealerConfig
}
// API contains configs for API endpoint
@@ -51,14 +52,6 @@ type Metrics struct {
PubsubTracing bool
}
// // Storage Miner
type Storage struct {
// Local worker config
AllowPreCommit1 bool
AllowPreCommit2 bool
AllowCommit bool
}
func defCommon() Common {
return Common{
API: API{
@@ -90,7 +83,7 @@ func DefaultStorageMiner() *StorageMiner {
cfg := &StorageMiner{
Common: defCommon(),
Storage: Storage{
Storage: sectorstorage.SealerConfig{
AllowPreCommit1: true,
AllowPreCommit2: true,
AllowCommit: true,
node/config/storage.go
+5 -13
View File
@@ -2,6 +2,7 @@ package config
import (
"encoding/json"
"github.com/filecoin-project/lotus/storage/sectorstorage/stores"
"io"
"io/ioutil"
"os"
@@ -9,16 +10,7 @@ import (
"golang.org/x/xerrors"
)
type LocalPath struct {
Path string
}
// .lotusstorage/storage.json
type StorageConfig struct {
StoragePaths []LocalPath
}
func StorageFromFile(path string, def *StorageConfig) (*StorageConfig, error) {
func StorageFromFile(path string, def *stores.StorageConfig) (*stores.StorageConfig, error) {
file, err := os.Open(path)
switch {
case os.IsNotExist(err):
@@ -34,8 +26,8 @@ func StorageFromFile(path string, def *StorageConfig) (*StorageConfig, error) {
return StorageFromReader(file)
}
func StorageFromReader(reader io.Reader) (*StorageConfig, error) {
var cfg StorageConfig
func StorageFromReader(reader io.Reader) (*stores.StorageConfig, error) {
var cfg stores.StorageConfig
err := json.NewDecoder(reader).Decode(&cfg)
if err != nil {
return nil, err
@@ -44,7 +36,7 @@ func StorageFromReader(reader io.Reader) (*StorageConfig, error) {
return &cfg, nil
}
func WriteStorageFile(path string, config StorageConfig) error {
func WriteStorageFile(path string, config stores.StorageConfig) error {
b, err := json.MarshalIndent(config, "", " ")
if err != nil {
return xerrors.Errorf("marshaling storage config: %w", err)
node/impl/client/client.go
+2 -1
View File
@@ -3,6 +3,7 @@ package client
import (
"context"
"errors"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"io"
"os"
@@ -83,7 +84,7 @@ func (a *API) ClientStartDeal(ctx context.Context, params *api.StartDealParams)
return nil, xerrors.Errorf("failed checking miners sector size: %w", err)
}
rt, _, err := api.ProofTypeFromSectorSize(ssize)
rt, _, err := ffiwrapper.ProofTypeFromSectorSize(ssize)
if err != nil {
return nil, xerrors.Errorf("bad sector size: %w", err)
}
node/impl/remoteworker.go
+53
View File
@@ -0,0 +1,53 @@
package impl
import (
"context"
"net/http"
"golang.org/x/xerrors"
"github.com/filecoin-project/specs-actors/actors/abi"
storage2 "github.com/filecoin-project/specs-storage/storage"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/client"
"github.com/filecoin-project/lotus/lib/jsonrpc"
"github.com/filecoin-project/lotus/storage/sectorstorage"
)
type remoteWorker struct {
api.WorkerApi
closer jsonrpc.ClientCloser
}
func (r *remoteWorker) NewSector(ctx context.Context, sector abi.SectorID) error {
return xerrors.New("unsupported")
}
func (r *remoteWorker) AddPiece(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage2.Data) (abi.PieceInfo, error) {
return abi.PieceInfo{}, xerrors.New("unsupported")
}
func connectRemoteWorker(ctx context.Context, fa api.Common, url string) (*remoteWorker, error) {
token, err := fa.AuthNew(ctx, []api.Permission{"admin"})
if err != nil {
return nil, xerrors.Errorf("creating auth token for remote connection: %w", err)
}
headers := http.Header{}
headers.Add("Authorization", "Bearer "+string(token))
wapi, closer, err := client.NewWorkerRPC(url, headers)
if err != nil {
return nil, xerrors.Errorf("creating jsonrpc client: %w", err)
}
return &remoteWorker{wapi, closer}, nil
}
func (r *remoteWorker) Close() error {
r.closer()
return nil
}
var _ sectorstorage.Worker = &remoteWorker{}
node/impl/storminer.go
+2 -2
View File
@@ -50,7 +50,7 @@ func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) {
sm.StorageMgr.ServeHTTP(w, r)
}
func (sm *StorageMinerAPI) WorkerStats(context.Context) (map[uint64]api.WorkerStats, error) {
func (sm *StorageMinerAPI) WorkerStats(context.Context) (map[uint64]sectorstorage.WorkerStats, error) {
return sm.StorageMgr.WorkerStats(), nil
}
@@ -149,7 +149,7 @@ func (sm *StorageMinerAPI) SectorsUpdate(ctx context.Context, id abi.SectorNumbe
}
func (sm *StorageMinerAPI) WorkerConnect(ctx context.Context, url string) error {
w, err := sectorstorage.ConnectRemote(ctx, sm, url)
w, err := connectRemoteWorker(ctx, sm, url)
if err != nil {
return xerrors.Errorf("connecting remote storage failed: %w", err)
}
node/modules/storageminer.go
+17 -6
View File
@@ -2,6 +2,7 @@ package modules
import (
"context"
"net/http"
"reflect"
"github.com/ipfs/go-bitswap"
@@ -43,7 +44,6 @@ import (
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/markets/retrievaladapter"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/modules/helpers"
"github.com/filecoin-project/lotus/node/repo"
@@ -92,7 +92,7 @@ func ProofsConfig(maddr dtypes.MinerAddress, fnapi lapi.FullNode) (*ffiwrapper.C
return nil, err
}
ppt, spt, err := lapi.ProofTypeFromSectorSize(ssize)
ppt, spt, err := ffiwrapper.ProofTypeFromSectorSize(ssize)
if err != nil {
return nil, xerrors.Errorf("bad sector size: %w", err)
}
@@ -132,7 +132,7 @@ func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h
return nil, err
}
ppt, _, err := lapi.ProofTypeFromSectorSize(sealer.SectorSize())
ppt, _, err := ffiwrapper.ProofTypeFromSectorSize(sealer.SectorSize())
if err != nil {
return nil, xerrors.Errorf("bad sector size: %w", err)
}
@@ -320,7 +320,7 @@ func StorageProvider(ctx helpers.MetricsCtx, fapi lapi.FullNode, h host.Host, ds
return nil, err
}
rt, _, err := lapi.ProofTypeFromSectorSize(ssize)
rt, _, err := ffiwrapper.ProofTypeFromSectorSize(ssize)
if err != nil {
return nil, err
}
@@ -339,10 +339,10 @@ func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.S
return retrievalimpl.NewProvider(address, adapter, network, pieceStore, ibs, ds)
}
func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls stores.LocalStorage, si stores.SectorIndex, cfg *ffiwrapper.Config, sc config.Storage, urls sectorstorage.URLs, ca lapi.Common) (*sectorstorage.Manager, error) {
func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls stores.LocalStorage, si stores.SectorIndex, cfg *ffiwrapper.Config, sc sectorstorage.SealerConfig, urls sectorstorage.URLs, sa sectorstorage.StorageAuth) (*sectorstorage.Manager, error) {
ctx := helpers.LifecycleCtx(mctx, lc)
sst, err := sectorstorage.New(ctx, ls, si, cfg, sc, urls, ca)
sst, err := sectorstorage.New(ctx, ls, si, cfg, sc, urls, sa)
if err != nil {
return nil, err
}
@@ -359,3 +359,14 @@ func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls stores.LocalStor
return sst, nil
}
func StorageAuth(ctx helpers.MetricsCtx, ca lapi.Common) (sectorstorage.StorageAuth, error) {
token, err := ca.AuthNew(ctx, []lapi.Permission{"admin"})
if err != nil {
return nil, xerrors.Errorf("creating storage auth header: %w", err)
}
headers := http.Header{}
headers.Add("Authorization", "Bearer "+string(token))
return sectorstorage.StorageAuth(headers), nil
}
node/node_test.go
+2 -1
View File
@@ -5,6 +5,7 @@ import (
"context"
"crypto/rand"
"github.com/filecoin-project/lotus/lib/lotuslog"
"github.com/filecoin-project/lotus/storage/mockstorage"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"io/ioutil"
"net/http/httptest"
@@ -310,7 +311,7 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t
if err != nil {
t.Fatal(err)
}
genm, k, err := mock.PreSeal(2048, maddr, nPreseal)
genm, k, err := mockstorage.PreSeal(2048, maddr, nPreseal)
if err != nil {
t.Fatal(err)
}
node/repo/fsrepo.go
+6 -5
View File
@@ -3,6 +3,7 @@ package repo
import (
"encoding/json"
"fmt"
"github.com/filecoin-project/lotus/storage/sectorstorage/stores"
"io"
"io/ioutil"
"os"
@@ -282,26 +283,26 @@ func (fsr *fsLockedRepo) Config() (interface{}, error) {
return config.FromFile(fsr.join(fsConfig), defConfForType(fsr.repoType))
}
func (fsr *fsLockedRepo) GetStorage() (config.StorageConfig, error) {
func (fsr *fsLockedRepo) GetStorage() (stores.StorageConfig, error) {
fsr.storageLk.Lock()
defer fsr.storageLk.Unlock()
return fsr.getStorage(nil)
}
func (fsr *fsLockedRepo) getStorage(def *config.StorageConfig) (config.StorageConfig, error) {
func (fsr *fsLockedRepo) getStorage(def *stores.StorageConfig) (stores.StorageConfig, error) {
c, err := config.StorageFromFile(fsr.join(fsStorageConfig), def)
if err != nil {
return config.StorageConfig{}, err
return stores.StorageConfig{}, err
}
return *c, nil
}
func (fsr *fsLockedRepo) SetStorage(c func(*config.StorageConfig)) error {
func (fsr *fsLockedRepo) SetStorage(c func(*stores.StorageConfig)) error {
fsr.storageLk.Lock()
defer fsr.storageLk.Unlock()
sc, err := fsr.getStorage(&config.StorageConfig{})
sc, err := fsr.getStorage(&stores.StorageConfig{})
if err != nil {
return xerrors.Errorf("get storage: %w", err)
}
node/repo/interface.go
+3 -3
View File
@@ -2,12 +2,12 @@ package repo
import (
"errors"
"github.com/filecoin-project/lotus/storage/sectorstorage/stores"
"github.com/ipfs/go-datastore"
"github.com/multiformats/go-multiaddr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/config"
)
var (
@@ -38,8 +38,8 @@ type LockedRepo interface {
// Returns config in this repo
Config() (interface{}, error)
GetStorage() (config.StorageConfig, error)
SetStorage(func(*config.StorageConfig)) error
GetStorage() (stores.StorageConfig, error)
SetStorage(func(*stores.StorageConfig)) error
// SetAPIEndpoint sets the endpoint of the current API
// so it can be read by API clients
node/repo/memrepo.go
+7 -7
View File
@@ -41,12 +41,12 @@ type lockedMemRepo struct {
tempDir string
token *byte
sc *config.StorageConfig
sc *stores.StorageConfig
}
func (lmem *lockedMemRepo) GetStorage() (config.StorageConfig, error) {
func (lmem *lockedMemRepo) GetStorage() (stores.StorageConfig, error) {
if lmem.sc == nil {
lmem.sc = &config.StorageConfig{StoragePaths: []config.LocalPath{
lmem.sc = &stores.StorageConfig{StoragePaths: []stores.LocalPath{
{Path: lmem.Path()},
}}
}
@@ -54,7 +54,7 @@ func (lmem *lockedMemRepo) GetStorage() (config.StorageConfig, error) {
return *lmem.sc, nil
}
func (lmem *lockedMemRepo) SetStorage(c func(*config.StorageConfig)) error {
func (lmem *lockedMemRepo) SetStorage(c func(*stores.StorageConfig)) error {
_, _ = lmem.GetStorage()
c(lmem.sc)
@@ -75,8 +75,8 @@ func (lmem *lockedMemRepo) Path() string {
}
if lmem.t == StorageMiner {
if err := config.WriteStorageFile(filepath.Join(t, fsStorageConfig), config.StorageConfig{
StoragePaths: []config.LocalPath{
if err := config.WriteStorageFile(filepath.Join(t, fsStorageConfig), stores.StorageConfig{
StoragePaths: []stores.LocalPath{
{Path: t},
}}); err != nil {
panic(err)
@@ -220,7 +220,7 @@ func (lmem *lockedMemRepo) Config() (interface{}, error) {
return lmem.mem.configF(lmem.t), nil
}
func (lmem *lockedMemRepo) Storage() (config.StorageConfig, error) {
func (lmem *lockedMemRepo) Storage() (stores.StorageConfig, error) {
panic("implement me")
}