add dagstore register-shard command

This commit is contained in:
LexLuthr 2022-05-12 19:46:53 +05:30
parent 33a05d33a9
commit 7e492f23b1
5 changed files with 104 additions and 0 deletions

View File

@ -227,6 +227,9 @@ type StorageMiner interface {
// DagstoreGC runs garbage collection on the DAG store.
DagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin
// DagstoreRegisterShard registers a shard manually with dagstore with given pieceCID
DagstoreRegisterShard(ctx context.Context, key string) error //perm:admin
// IndexerAnnounceDeal informs indexer nodes that a new deal was received,
// so they can download its index
IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error //perm:admin

View File

@ -663,6 +663,8 @@ type StorageMinerStruct struct {
DagstoreRecoverShard func(p0 context.Context, p1 string) error `perm:"write"`
DagstoreRegisterShard func(p0 context.Context, p1 string) error `perm:"admin"`
DealsConsiderOfflineRetrievalDeals func(p0 context.Context) (bool, error) `perm:"admin"`
DealsConsiderOfflineStorageDeals func(p0 context.Context) (bool, error) `perm:"admin"`
@ -3990,6 +3992,17 @@ func (s *StorageMinerStub) DagstoreRecoverShard(p0 context.Context, p1 string) e
return ErrNotSupported
}
func (s *StorageMinerStruct) DagstoreRegisterShard(p0 context.Context, p1 string) error {
if s.Internal.DagstoreRegisterShard == nil {
return ErrNotSupported
}
return s.Internal.DagstoreRegisterShard(p0, p1)
}
func (s *StorageMinerStub) DagstoreRegisterShard(p0 context.Context, p1 string) error {
return ErrNotSupported
}
func (s *StorageMinerStruct) DealsConsiderOfflineRetrievalDeals(p0 context.Context) (bool, error) {
if s.Internal.DealsConsiderOfflineRetrievalDeals == nil {
return false, ErrNotSupported

View File

@ -59,6 +59,45 @@ var dagstoreListShardsCmd = &cli.Command{
},
}
var dagstoreRegisterShardCmd = &cli.Command{
Name: "register-shard",
ArgsUsage: "[key/pieceCID]",
Usage: "Register a shard",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "color",
Usage: "use color in display output",
DefaultText: "depends on output being a TTY",
},
},
Action: func(cctx *cli.Context) error {
if cctx.IsSet("color") {
color.NoColor = !cctx.Bool("color")
}
if cctx.NArg() != 1 {
return fmt.Errorf("must provide a single shard key")
}
marketsAPI, closer, err := lcli.GetMarketsAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
shardKey := cctx.Args().First()
err = marketsAPI.DagstoreRegisterShard(ctx, shardKey)
if err != nil {
return err
}
fmt.Println("Registered shard " + shardKey)
return nil
},
}
var dagstoreInitializeShardCmd = &cli.Command{
Name: "initialize-shard",
ArgsUsage: "[key]",

View File

@ -22,6 +22,7 @@
* [CreateBackup](#CreateBackup)
* [Dagstore](#Dagstore)
* [DagstoreGC](#DagstoreGC)
* [DagstoreRegisterShard] (#DagstoreRegisterShard)
* [DagstoreInitializeAll](#DagstoreInitializeAll)
* [DagstoreInitializeShard](#DagstoreInitializeShard)
* [DagstoreListShards](#DagstoreListShards)
@ -509,6 +510,22 @@ Response:
]
```
### DagstoreRegisterShard
DagstoreRegisterShard registers a shard with the dagstore. It takes
a PieceCID as input and generates a success or error message.
Perms: admin
Inputs:
```json
[
"string value"
]
```
Response: `{}`
### DagstoreInitializeAll
DagstoreInitializeAll initializes all uninitialized shards in bulk,
according to the policy passed in the parameters.

View File

@ -37,8 +37,10 @@ import (
"github.com/filecoin-project/go-fil-markets/piecestore"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/storagemarket"
filmktsstore "github.com/filecoin-project/go-fil-markets/stores"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/network"
mktsdagstore "github.com/filecoin-project/lotus/markets/dagstore"
sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
@ -83,6 +85,7 @@ type StorageMinerAPI struct {
SectorBlocks *sectorblocks.SectorBlocks `optional:"true"`
Host host.Host `optional:"true"`
DAGStore *dagstore.DAGStore `optional:"true"`
DAGStoreWrapper *mktsdagstore.Wrapper `optional:"true"`
// Miner / storage
Miner *storage.Miner `optional:"true"`
@ -792,6 +795,35 @@ func (sm *StorageMinerAPI) DagstoreListShards(ctx context.Context) ([]api.Dagsto
return ret, nil
}
func (sm *StorageMinerAPI) DagstoreRegisterShard(ctx context.Context, key string) error {
if sm.DAGStore == nil {
return fmt.Errorf("dagstore not available on this node")
}
// First check if the shard has already been registered
k := shard.KeyFromString(key)
_, err := sm.DAGStore.GetShardInfo(k)
if err == nil {
// Shard already registered, nothing further to do
return nil
}
// If the shard is not registered we would expect ErrShardUnknown
if !errors.Is(err, dagstore.ErrShardUnknown) {
return fmt.Errorf("getting shard info from DAG store: %w", err)
}
pieceCid, err := cid.Parse(key)
if err != nil {
return fmt.Errorf("parsing shard key as piece cid: %w", err)
}
if err = filmktsstore.RegisterShardSync(ctx, sm.DAGStoreWrapper, pieceCid, "", true); err != nil {
return fmt.Errorf("failed to register shard: %w", err)
}
return nil
}
func (sm *StorageMinerAPI) DagstoreInitializeShard(ctx context.Context, key string) error {
if sm.DAGStore == nil {
return fmt.Errorf("dagstore not available on this node")