cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

storage: Support adding paths at runtime

Browse Source
This commit is contained in:
Łukasz Magiera
2020-03-05 23:02:01 +01:00
parent a5e5918fc5
commit 1461e475da
9 changed files with 112 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node/impl/storminer.go
+10
View File
@@ -21,6 +21,7 @@ import (
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/storage"
"github.com/filecoin-project/lotus/storage/sealmgr/advmgr"
"github.com/filecoin-project/lotus/storage/sectorblocks"
)
@@ -35,6 +36,7 @@ type StorageMinerAPI struct {
Miner *storage.Miner
BlockMiner *miner.Miner
Full api.FullNode
StorageMgr *advmgr.Manager `optional:"true"`
}
func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) {
@@ -300,4 +302,12 @@ func (sm *StorageMinerAPI) DealsImportData(ctx context.Context, deal cid.Cid, fn
return sm.StorageProvider.ImportDataForDeal(ctx, deal, fi)
}
func (sm *StorageMinerAPI) StorageAddLocal(ctx context.Context, path string) error {
if sm.StorageMgr == nil {
return xerrors.Errorf("no storage manager")
}
return sm.StorageMgr.AddLocalStorage(path)
}
var _ api.StorageMiner = &StorageMinerAPI{}
node/modules/services.go
+1 -1
View File
@@ -71,7 +71,7 @@ func HandleIncomingBlocks(mctx helpers.MetricsCtx, lc fx.Lifecycle, ps *pubsub.P
func HandleIncomingMessages(mctx helpers.MetricsCtx, lc fx.Lifecycle, ps *pubsub.PubSub, mpool *messagepool.MessagePool) {
ctx := helpers.LifecycleCtx(mctx, lc)
msgsub, err := ps.Subscribe(build.BlocksTopic)
msgsub, err := ps.Subscribe(build.MessagesTopic)
if err != nil {
panic(err)
}
node/node_test.go
+13 -28
View File
@@ -6,13 +6,10 @@ import (
"crypto/rand"
"io/ioutil"
"net/http/httptest"
"path/filepath"
"testing"
"time"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
badger "github.com/ipfs/go-ds-badger2"
logging "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p-core/crypto"
"github.com/libp2p/go-libp2p-core/peer"
@@ -39,11 +36,12 @@ import (
"github.com/filecoin-project/lotus/lib/jsonrpc"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node"
"github.com/filecoin-project/lotus/node/impl"
"github.com/filecoin-project/lotus/node/modules"
modtest "github.com/filecoin-project/lotus/node/modules/testing"
"github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/lotus/storage/sbmock"
"github.com/filecoin-project/lotus/storage/sealmgr"
"github.com/filecoin-project/lotus/storage/sealmgr/advmgr"
)
func init() {
@@ -240,40 +238,24 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
f := fulls[full]
if _, err := f.FullNode.WalletImport(ctx, &keys[i].KeyInfo); err != nil {
return nil, nil
t.Fatal(err)
}
if err := f.FullNode.WalletSetDefault(ctx, keys[i].Address); err != nil {
return nil, nil
t.Fatal(err)
}
genMiner := maddrs[i]
wa := genms[i].Worker
storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn, node.Options())
sma := storers[i].StorageMiner.(*impl.StorageMinerAPI)
psd := presealDirs[i]
mds, err := badger.NewDatastore(filepath.Join(psd, "badger"), nil)
if err != nil {
t.Fatal(err)
}
osb, err := sectorbuilder.New(&sectorbuilder.Config{
SealProofType: abi.RegisteredProof_StackedDRG2KiBSeal,
PoStProofType: abi.RegisteredProof_StackedDRG2KiBPoSt,
WorkerThreads: 2,
Miner: genMiner,
Paths: sectorbuilder.SimplePath(psd),
}, namespace.Wrap(mds, datastore.NewKey("/sectorbuilder")))
if err != nil {
t.Fatal(err)
}
if err := sma.SectorBuilder.(*sectorbuilder.SectorBuilder).ImportFrom(osb, false); err != nil {
if err := storers[i].StorageAddLocal(ctx, presealDirs[i]); err != nil {
t.Fatal(err)
}
/*
sma := storers[i].StorageMiner.(*impl.StorageMinerAPI)
psd := presealDirs[i]
*/
}
if err := mn.LinkAll(); err != nil {
@@ -394,7 +376,10 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t
wa := genms[i].Worker
storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn, node.Options(
node.Override(new(sectorbuilder.Interface), sbmock.NewMockSectorBuilder(5, build.SectorSizes[0])),
node.Override(new(sealmgr.Manager), func() (sealmgr.Manager, error) {
return sealmgr.NewSimpleManager(nil, genMiner, sbmock.NewMockSectorBuilder(5, build.SectorSizes[0]))
}),
node.Unset(new(*advmgr.Manager)),
))
}
node/repo/memrepo.go
+39 -3
View File
@@ -1,10 +1,13 @@
package repo
import (
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
"sync"
"github.com/google/uuid"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
dssync "github.com/ipfs/go-datastore/sync"
@@ -40,7 +43,9 @@ type lockedMemRepo struct {
}
func (lmem *lockedMemRepo) GetStorage() (config.StorageConfig, error) {
panic("implement me")
return config.StorageConfig{StoragePaths: []config.LocalPath{
{Path: lmem.Path()},
}}, nil
}
func (lmem *lockedMemRepo) SetStorage(config.StorageConfig) error {
@@ -48,14 +53,45 @@ func (lmem *lockedMemRepo) SetStorage(config.StorageConfig) error {
}
func (lmem *lockedMemRepo) Path() string {
lmem.Lock()
defer lmem.Unlock()
if lmem.tempDir != "" {
return lmem.tempDir
}
t, err := ioutil.TempDir(os.TempDir(), "lotus-memrepo-temp-")
if err != nil {
panic(err) // only used in tests, probably fine
}
lmem.Lock()
if lmem.t == StorageMiner {
if err := config.WriteStorageFile(filepath.Join(t, fsStorageConfig), config.StorageConfig{
StoragePaths: []config.LocalPath{
{Path: t},
}}); err != nil {
panic(err)
}
b, err := json.MarshalIndent(&config.StorageMeta{
ID: uuid.New().String(),
Weight: 10,
CanSeal: true,
CanStore: true,
}, "", " ")
if err != nil {
panic(err)
}
if err := ioutil.WriteFile(filepath.Join(t, "sectorstore.json"), b, 0644); err != nil {
panic(err)
}
}
lmem.tempDir = t
lmem.Unlock()
return t
}