2022-06-14 18:25:52 +00:00
package paths
2020-03-23 11:40:02 +00:00
import (
"context"
"encoding/json"
"math/bits"
2020-05-08 16:08:48 +00:00
"math/rand"
2020-03-23 11:40:02 +00:00
"os"
"path/filepath"
"sync"
2020-05-08 16:08:48 +00:00
"time"
2020-03-23 11:40:02 +00:00
2024-01-12 10:03:37 +00:00
"github.com/ipfs/go-cid"
2020-03-23 11:40:02 +00:00
"golang.org/x/xerrors"
2022-01-18 10:37:15 +00:00
ffi "github.com/filecoin-project/filecoin-ffi"
2020-09-07 03:49:10 +00:00
"github.com/filecoin-project/go-state-types/abi"
2022-06-14 15:00:51 +00:00
"github.com/filecoin-project/go-state-types/proof"
2020-08-16 10:40:35 +00:00
2022-11-09 11:27:45 +00:00
"github.com/filecoin-project/lotus/lib/result"
2022-06-14 18:03:38 +00:00
"github.com/filecoin-project/lotus/storage/sealer/fsutil"
2022-06-15 10:06:22 +00:00
"github.com/filecoin-project/lotus/storage/sealer/storiface"
2020-03-23 11:40:02 +00:00
)
type LocalStorage interface {
2022-11-01 11:01:31 +00:00
GetStorage ( ) ( storiface . StorageConfig , error )
SetStorage ( func ( * storiface . StorageConfig ) ) error
2020-05-20 16:36:46 +00:00
2020-07-08 14:58:09 +00:00
Stat ( path string ) ( fsutil . FsStat , error )
2020-08-04 14:20:59 +00:00
// returns real disk usage for a file/directory
// os.ErrNotExit when file doesn't exist
DiskUsage ( path string ) ( int64 , error )
2020-03-23 11:40:02 +00:00
}
const MetaFile = "sectorstore.json"
type Local struct {
localStorage LocalStorage
index SectorIndex
urls [ ] string
2022-01-18 10:57:04 +00:00
paths map [ storiface . ID ] * path
2020-03-23 11:40:02 +00:00
localLk sync . RWMutex
}
type path struct {
2021-02-18 15:44:34 +00:00
local string // absolute local path
maxStorage uint64
2020-07-06 16:36:44 +00:00
2020-07-06 17:19:24 +00:00
reserved int64
2020-09-06 16:54:00 +00:00
reservations map [ abi . SectorID ] storiface . SectorFileType
2020-07-06 16:36:44 +00:00
}
2020-07-08 14:58:09 +00:00
func ( p * path ) stat ( ls LocalStorage ) ( fsutil . FsStat , error ) {
2022-05-03 19:47:14 +00:00
start := time . Now ( )
2020-07-06 16:56:46 +00:00
stat , err := ls . Stat ( p . local )
2020-07-06 16:36:44 +00:00
if err != nil {
2020-08-04 14:20:59 +00:00
return fsutil . FsStat { } , xerrors . Errorf ( "stat %s: %w" , p . local , err )
2020-07-06 16:36:44 +00:00
}
stat . Reserved = p . reserved
2020-07-06 16:56:46 +00:00
for id , ft := range p . reservations {
2020-09-06 16:54:00 +00:00
for _ , fileType := range storiface . PathTypes {
2020-07-06 16:56:46 +00:00
if fileType & ft == 0 {
continue
}
2020-08-04 14:20:59 +00:00
sp := p . sectorPath ( id , fileType )
used , err := ls . DiskUsage ( sp )
if err == os . ErrNotExist {
p , ferr := tempFetchDest ( sp , false )
if ferr != nil {
return fsutil . FsStat { } , ferr
}
used , err = ls . DiskUsage ( p )
}
2020-07-06 16:56:46 +00:00
if err != nil {
2021-08-31 11:36:09 +00:00
// we don't care about 'not exist' errors, as storage can be
// reserved before any files are written, so this error is not
// unexpected
if ! os . IsNotExist ( err ) {
log . Warnf ( "getting disk usage of '%s': %+v" , p . sectorPath ( id , fileType ) , err )
}
2020-07-06 16:56:46 +00:00
continue
}
stat . Reserved -= used
}
}
if stat . Reserved < 0 {
log . Warnf ( "negative reserved storage: p.reserved=%d, reserved: %d" , p . reserved , stat . Reserved )
stat . Reserved = 0
}
stat . Available -= stat . Reserved
2020-07-06 16:36:44 +00:00
if stat . Available < 0 {
stat . Available = 0
}
2021-02-18 15:44:34 +00:00
if p . maxStorage > 0 {
used , err := ls . DiskUsage ( p . local )
if err != nil {
return fsutil . FsStat { } , err
}
stat . Max = int64 ( p . maxStorage )
stat . Used = used
avail := int64 ( p . maxStorage ) - used
if uint64 ( used ) > p . maxStorage {
avail = 0
}
if avail < stat . Available {
stat . Available = avail
}
}
2022-05-27 17:43:06 +00:00
if time . Now ( ) . Sub ( start ) > 5 * time . Second {
log . Warnw ( "slow storage stat" , "took" , time . Now ( ) . Sub ( start ) , "reservations" , len ( p . reservations ) )
}
2022-05-03 19:47:14 +00:00
2020-07-06 16:36:44 +00:00
return stat , err
2020-03-23 11:40:02 +00:00
}
2020-09-06 16:54:00 +00:00
func ( p * path ) sectorPath ( sid abi . SectorID , fileType storiface . SectorFileType ) string {
return filepath . Join ( p . local , fileType . String ( ) , storiface . SectorName ( sid ) )
2020-07-06 16:56:46 +00:00
}
2021-05-20 10:32:29 +00:00
type URLs [ ] string
2020-03-23 11:40:02 +00:00
func NewLocal ( ctx context . Context , ls LocalStorage , index SectorIndex , urls [ ] string ) ( * Local , error ) {
l := & Local {
2022-05-10 22:36:45 +00:00
localStorage : newCachedLocalStorage ( ls ) ,
2020-03-23 11:40:02 +00:00
index : index ,
urls : urls ,
2022-01-18 10:57:04 +00:00
paths : map [ storiface . ID ] * path { } ,
2020-03-23 11:40:02 +00:00
}
return l , l . open ( ctx )
}
func ( st * Local ) OpenPath ( ctx context . Context , p string ) error {
st . localLk . Lock ( )
defer st . localLk . Unlock ( )
2023-03-29 19:24:07 +00:00
mb , err := os . ReadFile ( filepath . Join ( p , MetaFile ) )
2020-03-23 11:40:02 +00:00
if err != nil {
return xerrors . Errorf ( "reading storage metadata for %s: %w" , p , err )
}
2022-11-01 11:01:31 +00:00
var meta storiface . LocalStorageMeta
2020-03-23 11:40:02 +00:00
if err := json . Unmarshal ( mb , & meta ) ; err != nil {
return xerrors . Errorf ( "unmarshalling storage metadata for %s: %w" , p , err )
}
2022-07-13 18:09:21 +00:00
if p , exists := st . paths [ meta . ID ] ; exists {
return xerrors . Errorf ( "path with ID %s already opened: '%s'" , meta . ID , p . local )
}
2020-03-23 11:40:02 +00:00
// TODO: Check existing / dedupe
out := & path {
local : p ,
2020-07-06 16:36:44 +00:00
2021-02-18 15:44:34 +00:00
maxStorage : meta . MaxStorage ,
2020-07-06 17:19:24 +00:00
reserved : 0 ,
2020-09-06 16:54:00 +00:00
reservations : map [ abi . SectorID ] storiface . SectorFileType { } ,
2020-03-23 11:40:02 +00:00
}
2020-07-06 16:56:46 +00:00
fst , err := out . stat ( st . localStorage )
2020-03-23 11:40:02 +00:00
if err != nil {
return err
}
2022-01-18 10:57:04 +00:00
err = st . index . StorageAttach ( ctx , storiface . StorageInfo {
2021-02-18 15:44:34 +00:00
ID : meta . ID ,
URLs : st . urls ,
Weight : meta . Weight ,
MaxStorage : meta . MaxStorage ,
CanSeal : meta . CanSeal ,
CanStore : meta . CanStore ,
2021-10-06 11:54:25 +00:00
Groups : meta . Groups ,
AllowTo : meta . AllowTo ,
2022-07-12 10:42:08 +00:00
AllowTypes : meta . AllowTypes ,
DenyTypes : meta . DenyTypes ,
2020-03-23 11:40:02 +00:00
} , fst )
if err != nil {
return xerrors . Errorf ( "declaring storage in index: %w" , err )
}
2022-07-15 09:40:03 +00:00
if err := st . declareSectors ( ctx , p , meta . ID , meta . CanStore , false ) ; err != nil {
2020-09-28 19:06:49 +00:00
return err
}
st . paths [ meta . ID ] = out
return nil
}
2022-07-13 18:09:21 +00:00
func ( st * Local ) ClosePath ( ctx context . Context , id storiface . ID ) error {
st . localLk . Lock ( )
defer st . localLk . Unlock ( )
if _ , exists := st . paths [ id ] ; ! exists {
2022-07-15 10:56:03 +00:00
return xerrors . Errorf ( "path with ID %s isn't opened" , id )
2022-07-13 18:09:21 +00:00
}
for _ , url := range st . urls {
if err := st . index . StorageDetach ( ctx , id , url ) ; err != nil {
2022-07-15 10:56:03 +00:00
return xerrors . Errorf ( "dropping path (id='%s' url='%s'): %w" , id , url , err )
2022-07-13 18:09:21 +00:00
}
}
delete ( st . paths , id )
return nil
}
2020-09-28 19:06:49 +00:00
func ( st * Local ) open ( ctx context . Context ) error {
cfg , err := st . localStorage . GetStorage ( )
if err != nil {
return xerrors . Errorf ( "getting local storage config: %w" , err )
}
for _ , path := range cfg . StoragePaths {
err := st . OpenPath ( ctx , path . Path )
if err != nil {
return xerrors . Errorf ( "opening path %s: %w" , path . Path , err )
}
}
go st . reportHealth ( ctx )
return nil
}
2022-07-15 09:40:03 +00:00
func ( st * Local ) Redeclare ( ctx context . Context , filterId * storiface . ID , dropMissingDecls bool ) error {
2020-09-28 19:06:49 +00:00
st . localLk . Lock ( )
defer st . localLk . Unlock ( )
for id , p := range st . paths {
2023-03-29 19:24:07 +00:00
mb , err := os . ReadFile ( filepath . Join ( p . local , MetaFile ) )
2020-09-28 19:06:49 +00:00
if err != nil {
2020-09-30 17:32:19 +00:00
return xerrors . Errorf ( "reading storage metadata for %s: %w" , p . local , err )
2020-09-28 19:06:49 +00:00
}
2022-11-01 11:01:31 +00:00
var meta storiface . LocalStorageMeta
2020-09-28 19:06:49 +00:00
if err := json . Unmarshal ( mb , & meta ) ; err != nil {
2020-09-30 17:32:19 +00:00
return xerrors . Errorf ( "unmarshalling storage metadata for %s: %w" , p . local , err )
2020-09-28 19:06:49 +00:00
}
fst , err := p . stat ( st . localStorage )
if err != nil {
return err
}
if id != meta . ID {
log . Errorf ( "storage path ID changed: %s; %s -> %s" , p . local , id , meta . ID )
continue
}
2022-07-15 09:40:03 +00:00
if filterId != nil && * filterId != id {
continue
}
2020-09-28 19:06:49 +00:00
2022-01-18 10:57:04 +00:00
err = st . index . StorageAttach ( ctx , storiface . StorageInfo {
2021-02-18 15:44:34 +00:00
ID : id ,
URLs : st . urls ,
Weight : meta . Weight ,
MaxStorage : meta . MaxStorage ,
CanSeal : meta . CanSeal ,
CanStore : meta . CanStore ,
2021-10-06 11:54:25 +00:00
Groups : meta . Groups ,
AllowTo : meta . AllowTo ,
2022-07-12 10:42:08 +00:00
AllowTypes : meta . AllowTypes ,
DenyTypes : meta . DenyTypes ,
2020-09-28 19:06:49 +00:00
} , fst )
if err != nil {
return xerrors . Errorf ( "redeclaring storage in index: %w" , err )
}
2022-07-15 09:40:03 +00:00
if err := st . declareSectors ( ctx , p . local , meta . ID , meta . CanStore , dropMissingDecls ) ; err != nil {
2020-09-28 19:06:49 +00:00
return xerrors . Errorf ( "redeclaring sectors: %w" , err )
}
}
return nil
}
2022-07-15 09:40:03 +00:00
func ( st * Local ) declareSectors ( ctx context . Context , p string , id storiface . ID , primary , dropMissing bool ) error {
indexed := map [ storiface . Decl ] struct { } { }
if dropMissing {
decls , err := st . index . StorageList ( ctx )
if err != nil {
return xerrors . Errorf ( "getting declaration list: %w" , err )
}
for _ , decl := range decls [ id ] {
2022-07-15 13:49:23 +00:00
for _ , fileType := range decl . SectorFileType . AllSet ( ) {
indexed [ storiface . Decl {
SectorID : decl . SectorID ,
SectorFileType : fileType ,
} ] = struct { } { }
}
2022-07-15 09:40:03 +00:00
}
}
2020-09-06 16:54:00 +00:00
for _ , t := range storiface . PathTypes {
2022-09-09 11:53:40 +00:00
ents , err := os . ReadDir ( filepath . Join ( p , t . String ( ) ) )
2020-03-23 11:40:02 +00:00
if err != nil {
if os . IsNotExist ( err ) {
2020-08-16 10:40:35 +00:00
if err := os . MkdirAll ( filepath . Join ( p , t . String ( ) ) , 0755 ) ; err != nil { // nolint
2020-03-23 11:40:02 +00:00
return xerrors . Errorf ( "openPath mkdir '%s': %w" , filepath . Join ( p , t . String ( ) ) , err )
}
continue
}
return xerrors . Errorf ( "listing %s: %w" , filepath . Join ( p , t . String ( ) ) , err )
}
for _ , ent := range ents {
2020-07-24 17:39:25 +00:00
if ent . Name ( ) == FetchTempSubdir {
continue
}
2020-09-06 16:54:00 +00:00
sid , err := storiface . ParseSectorID ( ent . Name ( ) )
2020-03-23 11:40:02 +00:00
if err != nil {
return xerrors . Errorf ( "parse sector id %s: %w" , ent . Name ( ) , err )
}
2022-07-15 09:40:03 +00:00
delete ( indexed , storiface . Decl {
SectorID : sid ,
SectorFileType : t ,
} )
2020-09-28 19:06:49 +00:00
if err := st . index . StorageDeclareSector ( ctx , id , sid , t , primary ) ; err != nil {
return xerrors . Errorf ( "declare sector %d(t:%d) -> %s: %w" , sid , t , id , err )
2020-03-23 11:40:02 +00:00
}
}
}
2022-07-15 09:40:03 +00:00
if len ( indexed ) > 0 {
log . Warnw ( "index contains sectors which are missing in the storage path" , "count" , len ( indexed ) , "dropMissing" , dropMissing )
}
if dropMissing {
for decl := range indexed {
if err := st . index . StorageDropSector ( ctx , id , decl . SectorID , decl . SectorFileType ) ; err != nil {
return xerrors . Errorf ( "dropping sector %v from index: %w" , decl , err )
}
}
}
2020-03-23 11:40:02 +00:00
return nil
}
2020-05-08 16:08:48 +00:00
func ( st * Local ) reportHealth ( ctx context . Context ) {
// randomize interval by ~10%
2020-05-08 16:54:06 +00:00
interval := ( HeartbeatInterval * 100_000 + time . Duration ( rand . Int63n ( 10_000 ) ) ) / 100_000
2020-05-08 16:08:48 +00:00
for {
select {
case <- time . After ( interval ) :
case <- ctx . Done ( ) :
return
}
2020-11-09 20:43:52 +00:00
st . reportStorage ( ctx )
}
}
2020-05-08 16:08:48 +00:00
2020-11-09 20:43:52 +00:00
func ( st * Local ) reportStorage ( ctx context . Context ) {
st . localLk . RLock ( )
2020-05-08 16:08:48 +00:00
2022-01-18 10:57:04 +00:00
toReport := map [ storiface . ID ] storiface . HealthReport { }
2020-11-09 20:43:52 +00:00
for id , p := range st . paths {
stat , err := p . stat ( st . localStorage )
2022-01-18 10:57:04 +00:00
r := storiface . HealthReport { Stat : stat }
2020-11-12 09:29:42 +00:00
if err != nil {
r . Err = err . Error ( )
2020-05-08 16:08:48 +00:00
}
2020-11-12 09:29:42 +00:00
2020-11-12 09:42:06 +00:00
toReport [ id ] = r
2020-11-09 20:43:52 +00:00
}
2020-05-08 16:08:48 +00:00
2020-11-09 20:43:52 +00:00
st . localLk . RUnlock ( )
2020-05-08 16:08:48 +00:00
2020-11-09 20:43:52 +00:00
for id , report := range toReport {
if err := st . index . StorageReportHealth ( ctx , id , report ) ; err != nil {
log . Warnf ( "error reporting storage health for %s (%+v): %+v" , id , report , err )
2020-05-08 16:08:48 +00:00
}
}
}
2022-06-17 11:31:05 +00:00
func ( st * Local ) Reserve ( ctx context . Context , sid storiface . SectorRef , ft storiface . SectorFileType , storageIDs storiface . SectorPaths , overheadTab map [ storiface . SectorFileType ] int ) ( func ( ) , error ) {
2020-11-04 20:29:08 +00:00
ssize , err := sid . ProofType . SectorSize ( )
if err != nil {
return nil , err
}
2020-07-06 16:36:44 +00:00
st . localLk . Lock ( )
2020-07-06 17:19:24 +00:00
done := func ( ) { }
2020-07-06 16:36:44 +00:00
deferredDone := func ( ) { done ( ) }
defer func ( ) {
st . localLk . Unlock ( )
deferredDone ( )
} ( )
2020-09-06 16:54:00 +00:00
for _ , fileType := range storiface . PathTypes {
2020-07-06 16:36:44 +00:00
if fileType & ft == 0 {
continue
}
2022-01-18 10:57:04 +00:00
id := storiface . ID ( storiface . PathByType ( storageIDs , fileType ) )
2020-07-06 16:36:44 +00:00
p , ok := st . paths [ id ]
if ! ok {
return nil , errPathNotFound
}
2020-07-06 16:56:46 +00:00
stat , err := p . stat ( st . localStorage )
2020-07-06 16:36:44 +00:00
if err != nil {
2020-08-04 14:20:59 +00:00
return nil , xerrors . Errorf ( "getting local storage stat: %w" , err )
2020-07-06 16:36:44 +00:00
}
2020-09-06 16:54:00 +00:00
overhead := int64 ( overheadTab [ fileType ] ) * int64 ( ssize ) / storiface . FSOverheadDen
2020-07-06 16:36:44 +00:00
if stat . Available < overhead {
2020-11-17 15:17:45 +00:00
return nil , storiface . Err ( storiface . ErrTempAllocateSpace , xerrors . Errorf ( "can't reserve %d bytes in '%s' (id:%s), only %d available" , overhead , p . local , id , stat . Available ) )
2020-07-06 16:36:44 +00:00
}
p . reserved += overhead
2021-03-31 06:30:24 +00:00
p . reservations [ sid . ID ] |= fileType
2020-07-06 16:36:44 +00:00
prevDone := done
2021-03-31 06:30:24 +00:00
saveFileType := fileType
2020-07-06 16:36:44 +00:00
done = func ( ) {
prevDone ( )
st . localLk . Lock ( )
defer st . localLk . Unlock ( )
p . reserved -= overhead
2021-03-31 06:30:24 +00:00
p . reservations [ sid . ID ] ^ = saveFileType
if p . reservations [ sid . ID ] == storiface . FTNone {
delete ( p . reservations , sid . ID )
}
2020-07-06 16:36:44 +00:00
}
}
deferredDone = func ( ) { }
return done , nil
}
2022-06-17 11:31:05 +00:00
func ( st * Local ) AcquireSector ( ctx context . Context , sid storiface . SectorRef , existing storiface . SectorFileType , allocate storiface . SectorFileType , pathType storiface . PathType , op storiface . AcquireMode ) ( storiface . SectorPaths , storiface . SectorPaths , error ) {
2020-03-23 11:40:02 +00:00
if existing | allocate != existing ^ allocate {
2020-09-06 16:54:00 +00:00
return storiface . SectorPaths { } , storiface . SectorPaths { } , xerrors . New ( "can't both find and allocate a sector" )
2020-03-23 11:40:02 +00:00
}
2020-11-04 20:29:08 +00:00
ssize , err := sid . ProofType . SectorSize ( )
if err != nil {
return storiface . SectorPaths { } , storiface . SectorPaths { } , err
}
2020-03-23 11:40:02 +00:00
st . localLk . RLock ( )
2020-06-04 19:00:16 +00:00
defer st . localLk . RUnlock ( )
2020-03-23 11:40:02 +00:00
2020-09-06 16:54:00 +00:00
var out storiface . SectorPaths
var storageIDs storiface . SectorPaths
2020-03-23 11:40:02 +00:00
2024-03-17 11:19:27 +00:00
// First find existing files
2020-09-06 16:54:00 +00:00
for _ , fileType := range storiface . PathTypes {
2024-03-17 11:19:27 +00:00
// also try to find existing sectors if we're allocating
if fileType & ( existing | allocate ) == 0 {
2020-03-23 11:40:02 +00:00
continue
}
2020-11-04 20:29:08 +00:00
si , err := st . index . StorageFindSector ( ctx , sid . ID , fileType , ssize , false )
2020-03-23 11:40:02 +00:00
if err != nil {
2024-03-17 11:19:27 +00:00
if fileType & existing != 0 {
log . Warnf ( "finding existing sector %d(t:%d) failed: %+v" , sid , fileType , err )
}
2020-03-23 11:40:02 +00:00
continue
}
for _ , info := range si {
p , ok := st . paths [ info . ID ]
if ! ok {
continue
}
if p . local == "" { // TODO: can that even be the case?
continue
}
2020-11-04 20:29:08 +00:00
spath := p . sectorPath ( sid . ID , fileType )
2020-09-06 16:54:00 +00:00
storiface . SetPathByType ( & out , fileType , spath )
storiface . SetPathByType ( & storageIDs , fileType , string ( info . ID ) )
2020-03-23 11:40:02 +00:00
2024-03-17 11:19:27 +00:00
existing = existing . Unset ( fileType )
allocate = allocate . Unset ( fileType )
2020-03-23 11:40:02 +00:00
break
}
}
2024-03-17 11:19:27 +00:00
// Then allocate for allocation requests
2020-09-06 16:54:00 +00:00
for _ , fileType := range storiface . PathTypes {
2020-03-23 11:40:02 +00:00
if fileType & allocate == 0 {
continue
}
2020-10-21 01:30:56 +00:00
sis , err := st . index . StorageBestAlloc ( ctx , fileType , ssize , pathType )
2020-03-23 11:40:02 +00:00
if err != nil {
2020-09-06 16:54:00 +00:00
return storiface . SectorPaths { } , storiface . SectorPaths { } , xerrors . Errorf ( "finding best storage for allocating : %w" , err )
2020-03-23 11:40:02 +00:00
}
var best string
2022-01-18 10:57:04 +00:00
var bestID storiface . ID
2020-03-23 11:40:02 +00:00
for _ , si := range sis {
p , ok := st . paths [ si . ID ]
if ! ok {
continue
}
if p . local == "" { // TODO: can that even be the case?
continue
}
2020-09-06 16:54:00 +00:00
if ( pathType == storiface . PathSealing ) && ! si . CanSeal {
2020-03-23 11:40:02 +00:00
continue
}
2020-09-06 16:54:00 +00:00
if ( pathType == storiface . PathStorage ) && ! si . CanStore {
2020-03-23 11:40:02 +00:00
continue
}
2022-07-01 16:02:10 +00:00
if ! fileType . Allowed ( si . AllowTypes , si . DenyTypes ) {
continue
}
2020-03-23 11:40:02 +00:00
// TODO: Check free space
2020-11-04 20:29:08 +00:00
best = p . sectorPath ( sid . ID , fileType )
2020-03-23 11:40:02 +00:00
bestID = si . ID
2020-07-28 10:20:58 +00:00
break
2020-03-23 11:40:02 +00:00
}
if best == "" {
2023-10-17 09:42:41 +00:00
return storiface . SectorPaths { } , storiface . SectorPaths { } , storiface . Err ( storiface . ErrTempAllocateSpace , xerrors . Errorf ( "couldn't find a suitable path for a sector" ) )
2020-03-23 11:40:02 +00:00
}
2020-09-06 16:54:00 +00:00
storiface . SetPathByType ( & out , fileType , best )
storiface . SetPathByType ( & storageIDs , fileType , string ( bestID ) )
2020-03-23 11:40:02 +00:00
allocate ^ = fileType
}
2020-06-04 19:00:16 +00:00
return out , storageIDs , nil
2020-03-23 11:40:02 +00:00
}
2022-01-18 10:57:04 +00:00
func ( st * Local ) Local ( ctx context . Context ) ( [ ] storiface . StoragePath , error ) {
2020-03-23 11:40:02 +00:00
st . localLk . RLock ( )
defer st . localLk . RUnlock ( )
2022-01-18 10:57:04 +00:00
var out [ ] storiface . StoragePath
2020-03-23 11:40:02 +00:00
for id , p := range st . paths {
if p . local == "" {
continue
}
si , err := st . index . StorageInfo ( ctx , id )
if err != nil {
return nil , xerrors . Errorf ( "get storage info for %s: %w" , id , err )
}
2022-01-18 10:57:04 +00:00
out = append ( out , storiface . StoragePath {
2020-03-23 11:40:02 +00:00
ID : id ,
Weight : si . Weight ,
LocalPath : p . local ,
CanSeal : si . CanSeal ,
CanStore : si . CanStore ,
} )
}
return out , nil
}
2022-01-18 10:57:04 +00:00
func ( st * Local ) Remove ( ctx context . Context , sid abi . SectorID , typ storiface . SectorFileType , force bool , keepIn [ ] storiface . ID ) error {
2020-03-23 11:40:02 +00:00
if bits . OnesCount ( uint ( typ ) ) != 1 {
return xerrors . New ( "delete expects one file type" )
}
2020-08-11 07:27:03 +00:00
si , err := st . index . StorageFindSector ( ctx , sid , typ , 0 , false )
2020-03-23 11:40:02 +00:00
if err != nil {
return xerrors . Errorf ( "finding existing sector %d(t:%d) failed: %w" , sid , typ , err )
}
2020-05-13 18:45:14 +00:00
if len ( si ) == 0 && ! force {
2020-03-23 15:40:36 +00:00
return xerrors . Errorf ( "can't delete sector %v(%d), not found" , sid , typ )
}
2021-10-11 19:05:05 +00:00
storeLoop :
2020-03-23 11:40:02 +00:00
for _ , info := range si {
2021-10-11 19:05:05 +00:00
for _ , id := range keepIn {
if id == info . ID {
continue storeLoop
}
}
2020-05-20 16:36:46 +00:00
if err := st . removeSector ( ctx , sid , typ , info . ID ) ; err != nil {
return err
2020-03-23 11:40:02 +00:00
}
2020-05-20 16:36:46 +00:00
}
return nil
}
2020-09-06 16:54:00 +00:00
func ( st * Local ) RemoveCopies ( ctx context . Context , sid abi . SectorID , typ storiface . SectorFileType ) error {
2020-05-20 16:36:46 +00:00
if bits . OnesCount ( uint ( typ ) ) != 1 {
return xerrors . New ( "delete expects one file type" )
}
2020-03-23 11:40:02 +00:00
2020-08-11 07:27:03 +00:00
si , err := st . index . StorageFindSector ( ctx , sid , typ , 0 , false )
2020-05-20 16:36:46 +00:00
if err != nil {
return xerrors . Errorf ( "finding existing sector %d(t:%d) failed: %w" , sid , typ , err )
}
var hasPrimary bool
for _ , info := range si {
if info . Primary {
hasPrimary = true
break
}
}
if ! hasPrimary {
log . Warnf ( "RemoveCopies: no primary copies of sector %v (%s), not removing anything" , sid , typ )
return nil
}
for _ , info := range si {
if info . Primary {
2020-03-23 11:40:02 +00:00
continue
}
2020-05-20 16:36:46 +00:00
if err := st . removeSector ( ctx , sid , typ , info . ID ) ; err != nil {
return err
2020-03-23 15:40:36 +00:00
}
2020-05-20 16:36:46 +00:00
}
2020-03-23 15:40:36 +00:00
2020-05-20 16:36:46 +00:00
return nil
}
2020-03-23 11:40:02 +00:00
2022-01-18 10:57:04 +00:00
func ( st * Local ) removeSector ( ctx context . Context , sid abi . SectorID , typ storiface . SectorFileType , storage storiface . ID ) error {
2020-05-20 16:36:46 +00:00
p , ok := st . paths [ storage ]
if ! ok {
return nil
}
if p . local == "" { // TODO: can that even be the case?
return nil
}
if err := st . index . StorageDropSector ( ctx , storage , sid , typ ) ; err != nil {
return xerrors . Errorf ( "dropping sector from index: %w" , err )
}
2020-07-06 16:56:46 +00:00
spath := p . sectorPath ( sid , typ )
2020-05-20 16:36:46 +00:00
log . Infof ( "remove %s" , spath )
if err := os . RemoveAll ( spath ) ; err != nil {
log . Errorf ( "removing sector (%v) from %s: %+v" , sid , spath , err )
2020-03-23 11:40:02 +00:00
}
2020-11-09 20:43:52 +00:00
st . reportStorage ( ctx ) // report freed space
2020-03-23 11:40:02 +00:00
return nil
}
2022-06-17 11:31:05 +00:00
func ( st * Local ) MoveStorage ( ctx context . Context , s storiface . SectorRef , types storiface . SectorFileType ) error {
2020-11-04 20:29:08 +00:00
dest , destIds , err := st . AcquireSector ( ctx , s , storiface . FTNone , types , storiface . PathStorage , storiface . AcquireMove )
2020-03-25 18:21:53 +00:00
if err != nil {
return xerrors . Errorf ( "acquire dest storage: %w" , err )
}
2020-11-04 20:29:08 +00:00
src , srcIds , err := st . AcquireSector ( ctx , s , types , storiface . FTNone , storiface . PathStorage , storiface . AcquireMove )
2020-03-25 18:21:53 +00:00
if err != nil {
return xerrors . Errorf ( "acquire src storage: %w" , err )
}
2020-09-06 16:54:00 +00:00
for _ , fileType := range storiface . PathTypes {
2020-03-25 18:21:53 +00:00
if fileType & types == 0 {
continue
}
2022-01-18 10:57:04 +00:00
sst , err := st . index . StorageInfo ( ctx , storiface . ID ( storiface . PathByType ( srcIds , fileType ) ) )
2020-03-25 18:21:53 +00:00
if err != nil {
return xerrors . Errorf ( "failed to get source storage info: %w" , err )
}
2022-01-18 10:57:04 +00:00
dst , err := st . index . StorageInfo ( ctx , storiface . ID ( storiface . PathByType ( destIds , fileType ) ) )
2020-03-25 18:21:53 +00:00
if err != nil {
return xerrors . Errorf ( "failed to get source storage info: %w" , err )
}
if sst . ID == dst . ID {
log . Debugf ( "not moving %v(%d); src and dest are the same" , s , fileType )
continue
}
if sst . CanStore {
log . Debugf ( "not moving %v(%d); source supports storage" , s , fileType )
continue
}
log . Debugf ( "moving %v(%d) to storage: %s(se:%t; st:%t) -> %s(se:%t; st:%t)" , s , fileType , sst . ID , sst . CanSeal , sst . CanStore , dst . ID , dst . CanSeal , dst . CanStore )
2022-01-18 10:57:04 +00:00
if err := st . index . StorageDropSector ( ctx , storiface . ID ( storiface . PathByType ( srcIds , fileType ) ) , s . ID , fileType ) ; err != nil {
2020-03-25 18:21:53 +00:00
return xerrors . Errorf ( "dropping source sector from index: %w" , err )
}
2023-06-28 11:56:00 +00:00
if err := Move ( storiface . PathByType ( src , fileType ) , storiface . PathByType ( dest , fileType ) ) ; err != nil {
2020-03-25 18:21:53 +00:00
// TODO: attempt some recovery (check if src is still there, re-declare)
return xerrors . Errorf ( "moving sector %v(%d): %w" , s , fileType , err )
}
2022-01-18 10:57:04 +00:00
if err := st . index . StorageDeclareSector ( ctx , storiface . ID ( storiface . PathByType ( destIds , fileType ) ) , s . ID , fileType , true ) ; err != nil {
return xerrors . Errorf ( "declare sector %d(t:%d) -> %s: %w" , s , fileType , storiface . ID ( storiface . PathByType ( destIds , fileType ) ) , err )
2020-03-25 18:21:53 +00:00
}
}
2020-11-09 20:43:52 +00:00
st . reportStorage ( ctx ) // report space use changes
2020-03-25 18:21:53 +00:00
return nil
}
2020-03-23 22:43:38 +00:00
var errPathNotFound = xerrors . Errorf ( "fsstat: path not found" )
2022-01-18 10:57:04 +00:00
func ( st * Local ) FsStat ( ctx context . Context , id storiface . ID ) ( fsutil . FsStat , error ) {
2020-03-23 11:40:02 +00:00
st . localLk . RLock ( )
defer st . localLk . RUnlock ( )
p , ok := st . paths [ id ]
if ! ok {
2020-07-08 14:58:09 +00:00
return fsutil . FsStat { } , errPathNotFound
2020-03-23 11:40:02 +00:00
}
2020-07-06 16:56:46 +00:00
return p . stat ( st . localStorage )
2020-03-23 11:40:02 +00:00
}
2020-03-24 20:28:07 +00:00
2022-01-14 13:11:04 +00:00
func ( st * Local ) GenerateSingleVanillaProof ( ctx context . Context , minerID abi . ActorID , si storiface . PostSectorChallenge , ppt abi . RegisteredPoStProof ) ( [ ] byte , error ) {
2022-06-17 11:31:05 +00:00
sr := storiface . SectorRef {
2022-01-14 13:11:04 +00:00
ID : abi . SectorID {
Miner : minerID ,
Number : si . SectorNumber ,
} ,
ProofType : si . SealProof ,
}
2022-11-09 11:49:52 +00:00
var cache , sealed , cacheID , sealedID string
2022-01-18 10:25:04 +00:00
if si . Update {
2022-11-09 11:49:52 +00:00
src , si , err := st . AcquireSector ( ctx , sr , storiface . FTUpdate | storiface . FTUpdateCache , storiface . FTNone , storiface . PathStorage , storiface . AcquireMove )
2022-01-18 10:25:04 +00:00
if err != nil {
return nil , xerrors . Errorf ( "acquire sector: %w" , err )
}
2022-01-20 13:59:57 +00:00
cache , sealed = src . UpdateCache , src . Update
2022-11-09 11:49:52 +00:00
cacheID , sealedID = si . UpdateCache , si . Update
2022-01-18 10:25:04 +00:00
} else {
2022-11-09 11:49:52 +00:00
src , si , err := st . AcquireSector ( ctx , sr , storiface . FTSealed | storiface . FTCache , storiface . FTNone , storiface . PathStorage , storiface . AcquireMove )
2022-01-18 10:25:04 +00:00
if err != nil {
return nil , xerrors . Errorf ( "acquire sector: %w" , err )
}
cache , sealed = src . Cache , src . Sealed
2022-11-09 11:49:52 +00:00
cacheID , sealedID = si . Cache , si . Sealed
2022-01-14 13:11:04 +00:00
}
2022-01-18 10:25:04 +00:00
if sealed == "" || cache == "" {
2022-01-14 13:11:04 +00:00
return nil , errPathNotFound
}
psi := ffi . PrivateSectorInfo {
SectorInfo : proof . SectorInfo {
SealProof : si . SealProof ,
SectorNumber : si . SectorNumber ,
SealedCID : si . SealedCID ,
} ,
2022-01-18 10:25:04 +00:00
CacheDirPath : cache ,
2022-01-14 13:11:04 +00:00
PoStProofType : ppt ,
2022-01-18 10:25:04 +00:00
SealedSectorPath : sealed ,
2022-01-14 13:11:04 +00:00
}
2022-11-09 11:49:52 +00:00
start := time . Now ( )
2022-11-09 11:27:45 +00:00
resCh := make ( chan result . Result [ [ ] byte ] , 1 )
go func ( ) {
resCh <- result . Wrap ( ffi . GenerateSingleVanillaProof ( psi , si . Challenge ) )
} ( )
select {
case r := <- resCh :
return r . Unwrap ( )
case <- ctx . Done ( ) :
2022-11-09 11:49:52 +00:00
log . Errorw ( "failed to generate valilla PoSt proof before context cancellation" , "err" , ctx . Err ( ) , "duration" , time . Now ( ) . Sub ( start ) , "cache-id" , cacheID , "sealed-id" , sealedID , "cache" , cache , "sealed" , sealed )
2022-11-09 11:27:45 +00:00
// this will leave the GenerateSingleVanillaProof goroutine hanging, but that's still less bad than failing PoSt
2022-11-14 20:04:18 +00:00
return nil , xerrors . Errorf ( "failed to generate vanilla proof before context cancellation: %w" , ctx . Err ( ) )
2022-11-09 11:27:45 +00:00
}
2021-07-27 03:15:53 +00:00
}
2024-02-17 10:20:49 +00:00
func ( st * Local ) GeneratePoRepVanillaProof ( ctx context . Context , sr storiface . SectorRef , sealed , unsealed cid . Cid , ticket abi . SealRandomness , seed abi . InteractiveSealRandomness ) ( [ ] byte , error ) {
2024-01-11 15:46:47 +00:00
src , _ , err := st . AcquireSector ( ctx , sr , storiface . FTSealed | storiface . FTCache , storiface . FTNone , storiface . PathStorage , storiface . AcquireMove )
if err != nil {
return nil , xerrors . Errorf ( "acquire sector: %w" , err )
}
if src . Sealed == "" || src . Cache == "" {
return nil , errPathNotFound
}
ssize , err := sr . ProofType . SectorSize ( )
if err != nil {
return nil , xerrors . Errorf ( "getting sector size: %w" , err )
}
secPiece := [ ] abi . PieceInfo { {
Size : abi . PaddedPieceSize ( ssize ) ,
PieceCID : unsealed ,
} }
return ffi . SealCommitPhase1 ( sr . ProofType , sealed , unsealed , src . Cache , src . Sealed , sr . ID . Number , sr . ID . Miner , ticket , seed , secPiece )
}
2020-03-24 20:28:07 +00:00
var _ Store = & Local { }