cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

update dagstore top level index db

Browse Source
This commit is contained in:
Aarsh Shah 2022-02-18 14:00:01 +04:00
parent a09f386537
commit e3f5b7c59f
13 changed files with 7 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
api/api_storage.go
View File

@ -226,9 +226,6 @@ type StorageMiner interface {
// IndexerAnnounceAllDeals informs the indexer nodes aboutall active deals.
IndexerAnnounceAllDeals(ctx context.Context) error //perm:admin
// DagstorePieceIndexSize returns the size of the piece index.
DagstorePieceIndexSize(ctx context.Context) (int64, error) //perm:admin
// DagstoreLookupPieces returns information about shards that contain the given CID.
DagstoreLookupPieces(ctx context.Context, cid cid.Cid) ([]DagstoreShardInfo, error) //perm:admin

13
api/proxy_gen.go
View File

@ -647,8 +647,6 @@ type StorageMinerStruct struct {
DagstoreLookupPieces func(p0 context.Context, p1 cid.Cid) ([]DagstoreShardInfo, error) `perm:"admin"`
DagstorePieceIndexSize func(p0 context.Context) (int64, error) `perm:"admin"`
DagstoreRecoverShard func(p0 context.Context, p1 string) error `perm:"write"`
DealsConsiderOfflineRetrievalDeals func(p0 context.Context) (bool, error) `perm:"admin"`
@ -3876,17 +3874,6 @@ func (s *StorageMinerStub) DagstoreLookupPieces(p0 context.Context, p1 cid.Cid)
return *new([]DagstoreShardInfo), ErrNotSupported
}
func (s *StorageMinerStruct) DagstorePieceIndexSize(p0 context.Context) (int64, error) {
if s.Internal.DagstorePieceIndexSize == nil {
return 0, ErrNotSupported
}
return s.Internal.DagstorePieceIndexSize(p0)
}
func (s *StorageMinerStub) DagstorePieceIndexSize(p0 context.Context) (int64, error) {
return 0, ErrNotSupported
}
func (s *StorageMinerStruct) DagstoreRecoverShard(p0 context.Context, p1 string) error {
if s.Internal.DagstoreRecoverShard == nil {
return ErrNotSupported

BIN
build/openrpc/full.json.gz
View File

Binary file not shown.

BIN
build/openrpc/miner.json.gz
View File

Binary file not shown.

BIN
build/openrpc/worker.json.gz
View File

Binary file not shown.

36
cmd/lotus-miner/dagstore.go
View File

@ -4,7 +4,6 @@ import (
"fmt"
"os"
"github.com/dustin/go-humanize"
"github.com/fatih/color"
"github.com/ipfs/go-cid"
"github.com/urfave/cli/v2"
@ -23,7 +22,6 @@ var dagstoreCmd = &cli.Command{
dagstoreRecoverShardCmd,
dagstoreInitializeAllCmd,
dagstoreGcCmd,
dagstorePieceIndexSizeCmd,
dagstoreLookupPiecesCmd,
},
}
@ -273,40 +271,6 @@ func printTableShards(shards []api.DagstoreShardInfo) error {
return tw.Flush(os.Stdout)
}
var dagstorePieceIndexSizeCmd = &cli.Command{
Name: "piece-index-size",
Usage: "Inspect the dagstore piece index size",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "color",
Usage: "use color in display output",
DefaultText: "depends on output being a TTY",
},
},
Action: func(cctx *cli.Context) error {
if cctx.IsSet("color") {
color.NoColor = !cctx.Bool("color")
}
marketsApi, closer, err := lcli.GetMarketsAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
size, err := marketsApi.DagstorePieceIndexSize(ctx)
if err != nil {
return err
}
fmt.Println(humanize.Bytes(uint64(size)))
return nil
},
}
var dagstoreLookupPiecesCmd = &cli.Command{
Name: "lookup-pieces",
Usage: "Lookup pieces that a given CID belongs to",

11
documentation/en/api-v0-methods-miner.md
View File

@ -24,7 +24,6 @@
* [DagstoreInitializeShard](#DagstoreInitializeShard)
* [DagstoreListShards](#DagstoreListShards)
* [DagstoreLookupPieces](#DagstoreLookupPieces)
* [DagstorePieceIndexSize](#DagstorePieceIndexSize)
* [DagstoreRecoverShard](#DagstoreRecoverShard)
* [Deals](#Deals)
* [DealsConsiderOfflineRetrievalDeals](#DealsConsiderOfflineRetrievalDeals)
@ -546,16 +545,6 @@ Response:
]
```
### DagstorePieceIndexSize
DagstorePieceIndexSize returns the size of the piece index.
Perms: admin
Inputs: `null`
Response: `9`
### DagstoreRecoverShard
DagstoreRecoverShard attempts to recover a failed shard.

15
documentation/en/cli-lotus-miner.md
View File

@ -1050,7 +1050,6 @@ COMMANDS:
recover-shard Attempt to recover a shard in errored state
initialize-all Initialize all uninitialized shards, streaming results as they're produced; only shards for unsealed pieces are initialized by default
gc Garbage collect the dagstore
piece-index-size Inspect the dagstore piece index size
lookup-pieces Lookup pieces that a given CID belongs to
help, h Shows a list of commands or help for one command
@ -1131,20 +1130,6 @@ OPTIONS:
```
### lotus-miner dagstore piece-index-size
```
NAME:
lotus-miner dagstore piece-index-size - Inspect the dagstore piece index size
USAGE:
lotus-miner dagstore piece-index-size [command options] [arguments...]
OPTIONS:
--color use color in display output (default: depends on output being a TTY)
--help, -h show help (default: false)
```
### lotus-miner dagstore lookup-pieces
```
NAME:

3
go.mod
View File

@ -26,7 +26,7 @@ require (
github.com/elastic/gosigar v0.14.1
github.com/etclabscore/go-openrpc-reflect v0.0.36
github.com/fatih/color v1.13.0
github.com/filecoin-project/dagstore v0.5.2-0.20220120115845-e07b050f48d3
github.com/filecoin-project/dagstore v0.5.2-0.20220218091406-7e3dada33cf4
github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200910194244-f640612a1a1f
github.com/filecoin-project/go-address v0.0.6
github.com/filecoin-project/go-bitfield v0.2.4
@ -37,7 +37,6 @@ require (
github.com/filecoin-project/go-fil-commcid v0.1.0
github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
github.com/filecoin-project/go-fil-markets v1.19.1-0.20220215120226-a0db7a73d4fd
github.com/filecoin-project/go-indexer-core v0.2.8
github.com/filecoin-project/go-jsonrpc v0.1.5
github.com/filecoin-project/go-legs v0.3.0
github.com/filecoin-project/go-padreader v0.0.1

8
go.sum
View File

@ -299,8 +299,8 @@ github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGE
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/filecoin-project/dagstore v0.5.1/go.mod h1:OdlK3x5m3Mol874WC2bI79H4H2+leN+FabwWdW2D/wY=
github.com/filecoin-project/dagstore v0.5.2-0.20220120115845-e07b050f48d3 h1:kuZsRFDjyrDAgoHNolJBnO7xX1EETHdJ090sgD3Al5E=
github.com/filecoin-project/dagstore v0.5.2-0.20220120115845-e07b050f48d3/go.mod h1:OdlK3x5m3Mol874WC2bI79H4H2+leN+FabwWdW2D/wY=
github.com/filecoin-project/dagstore v0.5.2-0.20220218091406-7e3dada33cf4 h1:zyE+eiIhkkTSi5zL5bDeHTfumikCER8uxKjyQJeEHV4=
github.com/filecoin-project/dagstore v0.5.2-0.20220218091406-7e3dada33cf4/go.mod h1:cVH6pCuJaOZzn2sCWQ2BtqnivvG/69acZJ0UA3mGagg=
github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8=
github.com/filecoin-project/go-address v0.0.5/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8=
github.com/filecoin-project/go-address v0.0.6 h1:DWQtj38ax+ogHwyH3VULRIoT8E6loyXqsk/p81xoY7M=
@ -346,7 +346,6 @@ github.com/filecoin-project/go-hamt-ipld/v3 v3.0.1/go.mod h1:gXpNmr3oQx8l3o7qkGy
github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0 h1:rVVNq0x6RGQIzCo1iiJlGFm9AGIZzeifggxtKMU7zmI=
github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0/go.mod h1:bxmzgT8tmeVQA1/gvBwFmYdT8SOFUwB3ovSUfG1Ux0g=
github.com/filecoin-project/go-indexer-core v0.2.4/go.mod h1:MSe5aRWmfRB+5syR4yDV+OApgJU+MUJ4rl9VUuzwCfc=
github.com/filecoin-project/go-indexer-core v0.2.8 h1:h1SRdZKTVcaXlzex3UevHh4OWDAhgpMzL4tHNAA7MQI=
github.com/filecoin-project/go-indexer-core v0.2.8/go.mod h1:IagNfTdFuX4057kla43PjRCn3yBuUiZgIxuA0hTUamY=
github.com/filecoin-project/go-jsonrpc v0.1.5 h1:ckxqZ09ivBAVf5CSmxxrqqNHC7PJm3GYGtYKiNQ+vGk=
github.com/filecoin-project/go-jsonrpc v0.1.5/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4=
@ -421,9 +420,7 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4
github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8=
github.com/gammazero/keymutex v0.0.2 h1:cmpLBJHdEwn+WlR5Z/o9/BN92znSZTp5AKPQDpu1QcI=
github.com/gammazero/keymutex v0.0.2/go.mod h1:qtzWCCLMisQUmVa4dvqHVgwfh4BP2YB7JxNDGXnsKrs=
github.com/gammazero/radixtree v0.2.5 h1:muPQ4eEgCkUymFWPiVQRuXOQv4IhWg8YXH2r71MoqPM=
github.com/gammazero/radixtree v0.2.5/go.mod h1:VPqqCDZ3YZZxAzUUsIF/ytFBigVWV7JIV1Stld8hri0=
github.com/gbrlsnchs/jwt/v3 v3.0.1 h1:lbUmgAKpxnClrKloyIwpxm4OuWeDl5wLk52G91ODPw4=
github.com/gbrlsnchs/jwt/v3 v3.0.1/go.mod h1:AncDcjXz18xetI3A6STfXq2w+LuTx8pQ8bGEwRN8zVM=
@ -928,7 +925,6 @@ github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd
github.com/ipld/go-ipld-selector-text-lite v0.0.1 h1:lNqFsQpBHc3p5xHob2KvEg/iM5dIFn6iw4L/Hh+kS1Y=
github.com/ipld/go-ipld-selector-text-lite v0.0.1/go.mod h1:U2CQmFb+uWzfIEF3I1arrDa5rwtj00PrpiwwCO+k1RM=
github.com/ipld/go-storethehash v0.0.0-20210915160027-d72ca9b0968c/go.mod h1:PwE6iq8TiWJRI3zMGA1RtkFAnrDMK93dLA5SUeu0lH8=
github.com/ipld/go-storethehash v0.0.1 h1:U9E9n3dE0IaImSNbRcUmm5Dk9UPiS9H0i+LmCFFGJBI=
github.com/ipld/go-storethehash v0.0.1/go.mod h1:w8cQfWInks8lvvbQTiKbCPusU9v0sqiViBihTHbavpQ=
github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=
github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4=

10
markets/dagstore/wrapper.go
View File

@ -10,7 +10,6 @@ import (
"sync"
"time"
"github.com/filecoin-project/go-indexer-core/store/storethehash"
"github.com/libp2p/go-libp2p-core/host"
carindex "github.com/ipld/go-car/v2/index"
@ -88,12 +87,7 @@ func NewDAGStore(cfg config.DAGStoreConfig, minerApi MinerAPI, h host.Host) (*da
return nil, nil, xerrors.Errorf("failed to initialise dagstore index repo: %w", err)
}
store, err := storethehash.New(indexDir)
if err != nil {
return nil, nil, xerrors.Errorf("failed to initialise store the index: %w", err)
}
topIndex := index.NewInverted(store, h.ID())
topIndex := index.NewInverted(dstore)
dcfg := dagstore.Config{
TransientsDir: transientsDir,
IndexRepo: irepo,
@ -414,7 +408,7 @@ func (w *Wrapper) markRegistrationComplete() error {
// Get all the pieces that contain a block
func (w *Wrapper) GetPiecesContainingBlock(blockCID cid.Cid) ([]cid.Cid, error) {
// Pieces are stored as "shards" in the DAG store
shardKeys, err := w.dagst.ShardsContainingMultihash(blockCID.Hash())
shardKeys, err := w.dagst.ShardsContainingMultihash(w.ctx, blockCID.Hash())
if err != nil {
return nil, xerrors.Errorf("getting pieces containing block %s: %w", blockCID, err)
}

2
markets/dagstore/wrapper_test.go
View File

@ -144,7 +144,7 @@ func (m *mockDagStore) GetIterableIndex(key shard.Key) (carindex.IterableIndex,
return nil, nil
}
func (m *mockDagStore) ShardsContainingMultihash(h mh.Multihash) ([]shard.Key, error) {
func (m *mockDagStore) ShardsContainingMultihash(ctx context.Context, h mh.Multihash) ([]shard.Key, error) {
return nil, nil
}

15
node/impl/storminer.go
View File

@ -1012,25 +1012,12 @@ func (sm *StorageMinerAPI) IndexerAnnounceAllDeals(ctx context.Context) error {
return sm.StorageProvider.AnnounceAllDealsToIndexer(ctx)
}
func (sm *StorageMinerAPI) DagstorePieceIndexSize(ctx context.Context) (int64, error) {
if sm.DAGStore == nil {
return 0, fmt.Errorf("dagstore not available on this node")
}
res, err := sm.DAGStore.TopLevelIndex.Size()
if err != nil {
return 0, fmt.Errorf("failed to get dagstore piece index size: %w", err)
}
return res, nil
}
func (sm *StorageMinerAPI) DagstoreLookupPieces(ctx context.Context, cid cid.Cid) ([]api.DagstoreShardInfo, error) {
if sm.DAGStore == nil {
return nil, fmt.Errorf("dagstore not available on this node")
}
keys, err := sm.DAGStore.TopLevelIndex.GetShardsForMultihash(cid.Hash())
keys, err := sm.DAGStore.TopLevelIndex.GetShardsForMultihash(ctx, cid.Hash())
if err != nil {
return nil, err
}