diff --git a/api/api_storage.go b/api/api_storage.go index e39acd034..55ed033aa 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -171,10 +171,32 @@ type StorageMiner interface { // DAG store. Only available on nodes running the markets subsystem. DagstoreListShards(ctx context.Context) ([]DagstoreShardInfo, error) //perm:read - // DagstoreInitializeShard initializes an uninitialized shard by acquiring - // it and releasing it as soon as it's ready. + // DagstoreInitializeShard initializes an uninitialized shard. + // + // Initialization consists of fetching the shard's data (deal payload) from + // the storage subsystem, generating an index, and persisting the index + // to facilitate later retrievals, and/or to publish to external sources. + // + // This operation is intended to complement the initial migration. The + // migration registers a shard for every unique piece CID, with lazy + // initialization. Thus, shards are not initialized immediately to avoid + // IO activity competing with proving. Instead, shard are initialized + // when first accessed. This method forces the initialization of a shard by + // accessing it and immediately releasing it. This is useful to warm up the + // cache to facilitate subsequent retrievals, and to generate the indexes + // to publish them externally. + // + // This operation fails if the shard is not in ShardStateNew state. + // It blocks until initialization finishes. DagstoreInitializeShard(ctx context.Context, key string) error //perm:write + // DagstoreRecoverShard attempts to recover a failed shard. + // + // This operation fails if the shard is not in ShardStateErrored state. + // It blocks until recovery finishes. If recovery failed, it returns the + // error. + DagstoreRecoverShard(ctx context.Context, key string) error //perm:write + // DagstoreGC runs garbage collection on the DAG store. DagstoreGC(ctx context.Context) ([]DagstoreGCResult, error) //perm:admin diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 416ea6a29..d32874890 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -609,6 +609,8 @@ type StorageMinerStruct struct { DagstoreListShards func(p0 context.Context) ([]DagstoreShardInfo, error) `perm:"read"` + DagstoreRecoverShard func(p0 context.Context, p1 string) error `perm:"write"` + DealsConsiderOfflineRetrievalDeals func(p0 context.Context) (bool, error) `perm:"admin"` DealsConsiderOfflineStorageDeals func(p0 context.Context) (bool, error) `perm:"admin"` @@ -3608,6 +3610,17 @@ func (s *StorageMinerStub) DagstoreListShards(p0 context.Context) ([]DagstoreSha return *new([]DagstoreShardInfo), ErrNotSupported } +func (s *StorageMinerStruct) DagstoreRecoverShard(p0 context.Context, p1 string) error { + if s.Internal.DagstoreRecoverShard == nil { + return ErrNotSupported + } + return s.Internal.DagstoreRecoverShard(p0, p1) +} + +func (s *StorageMinerStub) DagstoreRecoverShard(p0 context.Context, p1 string) error { + return ErrNotSupported +} + func (s *StorageMinerStruct) DealsConsiderOfflineRetrievalDeals(p0 context.Context) (bool, error) { if s.Internal.DealsConsiderOfflineRetrievalDeals == nil { return false, ErrNotSupported diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index 3e87a8914..725fc80f9 100644 Binary files a/build/openrpc/full.json.gz and b/build/openrpc/full.json.gz differ diff --git a/build/openrpc/miner.json.gz b/build/openrpc/miner.json.gz index 266112d07..93de3a98e 100644 Binary files a/build/openrpc/miner.json.gz and b/build/openrpc/miner.json.gz differ diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index 430601c03..c1f825ab3 100644 Binary files a/build/openrpc/worker.json.gz and b/build/openrpc/worker.json.gz differ diff --git a/documentation/en/api-v0-methods-miner.md b/documentation/en/api-v0-methods-miner.md index 537de3659..0bd212a3d 100644 --- a/documentation/en/api-v0-methods-miner.md +++ b/documentation/en/api-v0-methods-miner.md @@ -22,6 +22,7 @@ * [DagstoreGC](#DagstoreGC) * [DagstoreInitializeShard](#DagstoreInitializeShard) * [DagstoreListShards](#DagstoreListShards) + * [DagstoreRecoverShard](#DagstoreRecoverShard) * [Deals](#Deals) * [DealsConsiderOfflineRetrievalDeals](#DealsConsiderOfflineRetrievalDeals) * [DealsConsiderOfflineStorageDeals](#DealsConsiderOfflineStorageDeals) @@ -363,8 +364,23 @@ Inputs: `null` Response: `null` ### DagstoreInitializeShard -DagstoreInitializeShard initializes an uninitialized shard by acquiring -it and releasing it as soon as it's ready. +DagstoreInitializeShard initializes an uninitialized shard. + +Initialization consists of fetching the shard's data (deal payload) from +the storage subsystem, generating an index, and persisting the index +to facilitate later retrievals, and/or to publish to external sources. + +This operation is intended to complement the initial migration. The +migration registers a shard for every unique piece CID, with lazy +initialization. Thus, shards are not initialized immediately to avoid +IO activity competing with proving. Instead, shard are initialized +when first accessed. This method forces the initialization of a shard by +accessing it and immediately releasing it. This is useful to warm up the +cache to facilitate subsequent retrievals, and to generate the indexes +to publish them externally. + +This operation fails if the shard is not in ShardStateNew state. +It blocks until initialization finishes. Perms: write @@ -389,6 +405,25 @@ Inputs: `null` Response: `null` +### DagstoreRecoverShard +DagstoreRecoverShard attempts to recover a failed shard. + +This operation fails if the shard is not in ShardStateErrored state. +It blocks until recovery finishes. If recovery failed, it returns the +error. + + +Perms: write + +Inputs: +```json +[ + "string value" +] +``` + +Response: `{}` + ## Deals diff --git a/node/impl/storminer.go b/node/impl/storminer.go index b286b39f9..4b766b6f6 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -616,6 +616,36 @@ func (sm *StorageMinerAPI) DagstoreInitializeShard(ctx context.Context, key stri return nil } +func (sm *StorageMinerAPI) DagstoreRecoverShard(ctx context.Context, key string) error { + if sm.DAGStore == nil { + return fmt.Errorf("dagstore not available on this node") + } + + k := shard.KeyFromString(key) + + info, err := sm.DAGStore.GetShardInfo(k) + if err != nil { + return fmt.Errorf("failed to get shard info: %w", err) + } + if st := info.ShardState; st != dagstore.ShardStateErrored { + return fmt.Errorf("cannot recover shard; expected state ShardStateErrored, was: %s", st.String()) + } + + ch := make(chan dagstore.ShardResult, 1) + if err = sm.DAGStore.RecoverShard(ctx, k, ch, dagstore.RecoverOpts{}); err != nil { + return fmt.Errorf("failed to recover shard: %w", err) + } + + var res dagstore.ShardResult + select { + case res = <-ch: + case <-ctx.Done(): + return ctx.Err() + } + + return res.Error +} + func (sm *StorageMinerAPI) DagstoreGC(ctx context.Context) ([]api.DagstoreGCResult, error) { if sm.DAGStore == nil { return nil, fmt.Errorf("dagstore not available on this node")