lotus/extern/sector-storage/stores/local.go

711 lines
17 KiB
Go
Raw Normal View History

2020-03-23 11:40:02 +00:00
package stores
import (
"context"
"encoding/json"
"io/ioutil"
"math/bits"
2020-05-08 16:08:48 +00:00
"math/rand"
2020-03-23 11:40:02 +00:00
"os"
"path/filepath"
"sync"
2020-05-08 16:08:48 +00:00
"time"
2020-03-23 11:40:02 +00:00
"golang.org/x/xerrors"
2020-09-07 03:49:10 +00:00
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/specs-storage/storage"
2020-08-16 10:40:35 +00:00
"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
2020-09-06 16:54:00 +00:00
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
2020-03-23 11:40:02 +00:00
)
type StoragePath struct {
ID ID
Weight uint64
LocalPath string
CanSeal bool
CanStore bool
}
2020-08-16 10:40:35 +00:00
// LocalStorageMeta [path]/sectorstore.json
2020-03-23 11:40:02 +00:00
type LocalStorageMeta struct {
ID ID
// A high weight means data is more likely to be stored in this path
2020-03-23 11:40:02 +00:00
Weight uint64 // 0 = readonly
// Intermediate data for the sealing process will be stored here
CanSeal bool
// Finalized sectors that will be proved over time will be stored here
2020-03-23 11:40:02 +00:00
CanStore bool
// MaxStorage specifies the maximum number of bytes to use for sector storage
// (0 = unlimited)
MaxStorage uint64
2020-03-23 11:40:02 +00:00
}
2020-08-16 10:40:35 +00:00
// StorageConfig .lotusstorage/storage.json
type StorageConfig struct {
StoragePaths []LocalPath
}
type LocalPath struct {
Path string
}
2020-03-23 11:40:02 +00:00
type LocalStorage interface {
GetStorage() (StorageConfig, error)
SetStorage(func(*StorageConfig)) error
2020-07-08 14:58:09 +00:00
Stat(path string) (fsutil.FsStat, error)
2020-08-04 14:20:59 +00:00
// returns real disk usage for a file/directory
// os.ErrNotExit when file doesn't exist
DiskUsage(path string) (int64, error)
2020-03-23 11:40:02 +00:00
}
const MetaFile = "sectorstore.json"
type Local struct {
localStorage LocalStorage
index SectorIndex
urls []string
paths map[ID]*path
localLk sync.RWMutex
}
type path struct {
local string // absolute local path
maxStorage uint64
2020-07-06 16:36:44 +00:00
2020-07-06 17:19:24 +00:00
reserved int64
2020-09-06 16:54:00 +00:00
reservations map[abi.SectorID]storiface.SectorFileType
2020-07-06 16:36:44 +00:00
}
2020-07-08 14:58:09 +00:00
func (p *path) stat(ls LocalStorage) (fsutil.FsStat, error) {
stat, err := ls.Stat(p.local)
2020-07-06 16:36:44 +00:00
if err != nil {
2020-08-04 14:20:59 +00:00
return fsutil.FsStat{}, xerrors.Errorf("stat %s: %w", p.local, err)
2020-07-06 16:36:44 +00:00
}
stat.Reserved = p.reserved
for id, ft := range p.reservations {
2020-09-06 16:54:00 +00:00
for _, fileType := range storiface.PathTypes {
if fileType&ft == 0 {
continue
}
2020-08-04 14:20:59 +00:00
sp := p.sectorPath(id, fileType)
used, err := ls.DiskUsage(sp)
if err == os.ErrNotExist {
p, ferr := tempFetchDest(sp, false)
if ferr != nil {
return fsutil.FsStat{}, ferr
}
used, err = ls.DiskUsage(p)
}
if err != nil {
// we don't care about 'not exist' errors, as storage can be
// reserved before any files are written, so this error is not
// unexpected
if !os.IsNotExist(err) {
log.Warnf("getting disk usage of '%s': %+v", p.sectorPath(id, fileType), err)
}
continue
}
stat.Reserved -= used
}
}
if stat.Reserved < 0 {
log.Warnf("negative reserved storage: p.reserved=%d, reserved: %d", p.reserved, stat.Reserved)
stat.Reserved = 0
}
stat.Available -= stat.Reserved
2020-07-06 16:36:44 +00:00
if stat.Available < 0 {
stat.Available = 0
}
if p.maxStorage > 0 {
used, err := ls.DiskUsage(p.local)
if err != nil {
return fsutil.FsStat{}, err
}
stat.Max = int64(p.maxStorage)
stat.Used = used
avail := int64(p.maxStorage) - used
if uint64(used) > p.maxStorage {
avail = 0
}
if avail < stat.Available {
stat.Available = avail
}
}
2020-07-06 16:36:44 +00:00
return stat, err
2020-03-23 11:40:02 +00:00
}
2020-09-06 16:54:00 +00:00
func (p *path) sectorPath(sid abi.SectorID, fileType storiface.SectorFileType) string {
return filepath.Join(p.local, fileType.String(), storiface.SectorName(sid))
}
type URLs []string
2020-03-23 11:40:02 +00:00
func NewLocal(ctx context.Context, ls LocalStorage, index SectorIndex, urls []string) (*Local, error) {
l := &Local{
localStorage: ls,
index: index,
urls: urls,
paths: map[ID]*path{},
}
return l, l.open(ctx)
}
func (st *Local) OpenPath(ctx context.Context, p string) error {
st.localLk.Lock()
defer st.localLk.Unlock()
mb, err := ioutil.ReadFile(filepath.Join(p, MetaFile))
if err != nil {
return xerrors.Errorf("reading storage metadata for %s: %w", p, err)
}
var meta LocalStorageMeta
if err := json.Unmarshal(mb, &meta); err != nil {
return xerrors.Errorf("unmarshalling storage metadata for %s: %w", p, err)
}
// TODO: Check existing / dedupe
out := &path{
local: p,
2020-07-06 16:36:44 +00:00
maxStorage: meta.MaxStorage,
2020-07-06 17:19:24 +00:00
reserved: 0,
2020-09-06 16:54:00 +00:00
reservations: map[abi.SectorID]storiface.SectorFileType{},
2020-03-23 11:40:02 +00:00
}
fst, err := out.stat(st.localStorage)
2020-03-23 11:40:02 +00:00
if err != nil {
return err
}
err = st.index.StorageAttach(ctx, StorageInfo{
ID: meta.ID,
URLs: st.urls,
Weight: meta.Weight,
MaxStorage: meta.MaxStorage,
CanSeal: meta.CanSeal,
CanStore: meta.CanStore,
2020-03-23 11:40:02 +00:00
}, fst)
if err != nil {
return xerrors.Errorf("declaring storage in index: %w", err)
}
2020-09-28 19:06:49 +00:00
if err := st.declareSectors(ctx, p, meta.ID, meta.CanStore); err != nil {
return err
}
st.paths[meta.ID] = out
return nil
}
func (st *Local) open(ctx context.Context) error {
cfg, err := st.localStorage.GetStorage()
if err != nil {
return xerrors.Errorf("getting local storage config: %w", err)
}
for _, path := range cfg.StoragePaths {
err := st.OpenPath(ctx, path.Path)
if err != nil {
return xerrors.Errorf("opening path %s: %w", path.Path, err)
}
}
go st.reportHealth(ctx)
return nil
}
func (st *Local) Redeclare(ctx context.Context) error {
st.localLk.Lock()
defer st.localLk.Unlock()
for id, p := range st.paths {
mb, err := ioutil.ReadFile(filepath.Join(p.local, MetaFile))
if err != nil {
2020-09-30 17:32:19 +00:00
return xerrors.Errorf("reading storage metadata for %s: %w", p.local, err)
2020-09-28 19:06:49 +00:00
}
var meta LocalStorageMeta
if err := json.Unmarshal(mb, &meta); err != nil {
2020-09-30 17:32:19 +00:00
return xerrors.Errorf("unmarshalling storage metadata for %s: %w", p.local, err)
2020-09-28 19:06:49 +00:00
}
fst, err := p.stat(st.localStorage)
if err != nil {
return err
}
if id != meta.ID {
log.Errorf("storage path ID changed: %s; %s -> %s", p.local, id, meta.ID)
continue
}
err = st.index.StorageAttach(ctx, StorageInfo{
ID: id,
URLs: st.urls,
Weight: meta.Weight,
MaxStorage: meta.MaxStorage,
CanSeal: meta.CanSeal,
CanStore: meta.CanStore,
2020-09-28 19:06:49 +00:00
}, fst)
if err != nil {
return xerrors.Errorf("redeclaring storage in index: %w", err)
}
if err := st.declareSectors(ctx, p.local, meta.ID, meta.CanStore); err != nil {
return xerrors.Errorf("redeclaring sectors: %w", err)
}
}
return nil
}
func (st *Local) declareSectors(ctx context.Context, p string, id ID, primary bool) error {
2020-09-06 16:54:00 +00:00
for _, t := range storiface.PathTypes {
2020-03-23 11:40:02 +00:00
ents, err := ioutil.ReadDir(filepath.Join(p, t.String()))
if err != nil {
if os.IsNotExist(err) {
2020-08-16 10:40:35 +00:00
if err := os.MkdirAll(filepath.Join(p, t.String()), 0755); err != nil { // nolint
2020-03-23 11:40:02 +00:00
return xerrors.Errorf("openPath mkdir '%s': %w", filepath.Join(p, t.String()), err)
}
continue
}
return xerrors.Errorf("listing %s: %w", filepath.Join(p, t.String()), err)
}
for _, ent := range ents {
2020-07-24 17:39:25 +00:00
if ent.Name() == FetchTempSubdir {
continue
}
2020-09-06 16:54:00 +00:00
sid, err := storiface.ParseSectorID(ent.Name())
2020-03-23 11:40:02 +00:00
if err != nil {
return xerrors.Errorf("parse sector id %s: %w", ent.Name(), err)
}
2020-09-28 19:06:49 +00:00
if err := st.index.StorageDeclareSector(ctx, id, sid, t, primary); err != nil {
return xerrors.Errorf("declare sector %d(t:%d) -> %s: %w", sid, t, id, err)
2020-03-23 11:40:02 +00:00
}
}
}
return nil
}
2020-05-08 16:08:48 +00:00
func (st *Local) reportHealth(ctx context.Context) {
// randomize interval by ~10%
interval := (HeartbeatInterval*100_000 + time.Duration(rand.Int63n(10_000))) / 100_000
2020-05-08 16:08:48 +00:00
for {
select {
case <-time.After(interval):
case <-ctx.Done():
return
}
2020-11-09 20:43:52 +00:00
st.reportStorage(ctx)
}
}
2020-05-08 16:08:48 +00:00
2020-11-09 20:43:52 +00:00
func (st *Local) reportStorage(ctx context.Context) {
st.localLk.RLock()
2020-05-08 16:08:48 +00:00
2020-11-09 20:43:52 +00:00
toReport := map[ID]HealthReport{}
for id, p := range st.paths {
stat, err := p.stat(st.localStorage)
2020-11-12 09:29:42 +00:00
r := HealthReport{Stat: stat}
if err != nil {
r.Err = err.Error()
2020-05-08 16:08:48 +00:00
}
2020-11-12 09:29:42 +00:00
2020-11-12 09:42:06 +00:00
toReport[id] = r
2020-11-09 20:43:52 +00:00
}
2020-05-08 16:08:48 +00:00
2020-11-09 20:43:52 +00:00
st.localLk.RUnlock()
2020-05-08 16:08:48 +00:00
2020-11-09 20:43:52 +00:00
for id, report := range toReport {
if err := st.index.StorageReportHealth(ctx, id, report); err != nil {
log.Warnf("error reporting storage health for %s (%+v): %+v", id, report, err)
2020-05-08 16:08:48 +00:00
}
}
}
func (st *Local) Reserve(ctx context.Context, sid storage.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int) (func(), error) {
ssize, err := sid.ProofType.SectorSize()
if err != nil {
return nil, err
}
2020-07-06 16:36:44 +00:00
st.localLk.Lock()
2020-07-06 17:19:24 +00:00
done := func() {}
2020-07-06 16:36:44 +00:00
deferredDone := func() { done() }
defer func() {
st.localLk.Unlock()
deferredDone()
}()
2020-09-06 16:54:00 +00:00
for _, fileType := range storiface.PathTypes {
2020-07-06 16:36:44 +00:00
if fileType&ft == 0 {
continue
}
2020-09-06 16:54:00 +00:00
id := ID(storiface.PathByType(storageIDs, fileType))
2020-07-06 16:36:44 +00:00
p, ok := st.paths[id]
if !ok {
return nil, errPathNotFound
}
stat, err := p.stat(st.localStorage)
2020-07-06 16:36:44 +00:00
if err != nil {
2020-08-04 14:20:59 +00:00
return nil, xerrors.Errorf("getting local storage stat: %w", err)
2020-07-06 16:36:44 +00:00
}
2020-09-06 16:54:00 +00:00
overhead := int64(overheadTab[fileType]) * int64(ssize) / storiface.FSOverheadDen
2020-07-06 16:36:44 +00:00
if stat.Available < overhead {
2020-11-17 15:17:45 +00:00
return nil, storiface.Err(storiface.ErrTempAllocateSpace, xerrors.Errorf("can't reserve %d bytes in '%s' (id:%s), only %d available", overhead, p.local, id, stat.Available))
2020-07-06 16:36:44 +00:00
}
p.reserved += overhead
2021-03-31 06:30:24 +00:00
p.reservations[sid.ID] |= fileType
2020-07-06 16:36:44 +00:00
prevDone := done
2021-03-31 06:30:24 +00:00
saveFileType := fileType
2020-07-06 16:36:44 +00:00
done = func() {
prevDone()
st.localLk.Lock()
defer st.localLk.Unlock()
p.reserved -= overhead
2021-03-31 06:30:24 +00:00
p.reservations[sid.ID] ^= saveFileType
if p.reservations[sid.ID] == storiface.FTNone {
delete(p.reservations, sid.ID)
}
2020-07-06 16:36:44 +00:00
}
}
deferredDone = func() {}
return done, nil
}
func (st *Local) AcquireSector(ctx context.Context, sid storage.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, pathType storiface.PathType, op storiface.AcquireMode) (storiface.SectorPaths, storiface.SectorPaths, error) {
2020-03-23 11:40:02 +00:00
if existing|allocate != existing^allocate {
2020-09-06 16:54:00 +00:00
return storiface.SectorPaths{}, storiface.SectorPaths{}, xerrors.New("can't both find and allocate a sector")
2020-03-23 11:40:02 +00:00
}
ssize, err := sid.ProofType.SectorSize()
if err != nil {
return storiface.SectorPaths{}, storiface.SectorPaths{}, err
}
2020-03-23 11:40:02 +00:00
st.localLk.RLock()
2020-06-04 19:00:16 +00:00
defer st.localLk.RUnlock()
2020-03-23 11:40:02 +00:00
2020-09-06 16:54:00 +00:00
var out storiface.SectorPaths
var storageIDs storiface.SectorPaths
2020-03-23 11:40:02 +00:00
2020-09-06 16:54:00 +00:00
for _, fileType := range storiface.PathTypes {
2020-03-23 11:40:02 +00:00
if fileType&existing == 0 {
continue
}
si, err := st.index.StorageFindSector(ctx, sid.ID, fileType, ssize, false)
2020-03-23 11:40:02 +00:00
if err != nil {
log.Warnf("finding existing sector %d(t:%d) failed: %+v", sid, fileType, err)
continue
}
for _, info := range si {
p, ok := st.paths[info.ID]
if !ok {
continue
}
if p.local == "" { // TODO: can that even be the case?
continue
}
spath := p.sectorPath(sid.ID, fileType)
2020-09-06 16:54:00 +00:00
storiface.SetPathByType(&out, fileType, spath)
storiface.SetPathByType(&storageIDs, fileType, string(info.ID))
2020-03-23 11:40:02 +00:00
existing ^= fileType
break
}
}
2020-09-06 16:54:00 +00:00
for _, fileType := range storiface.PathTypes {
2020-03-23 11:40:02 +00:00
if fileType&allocate == 0 {
continue
}
sis, err := st.index.StorageBestAlloc(ctx, fileType, ssize, pathType)
2020-03-23 11:40:02 +00:00
if err != nil {
2020-09-06 16:54:00 +00:00
return storiface.SectorPaths{}, storiface.SectorPaths{}, xerrors.Errorf("finding best storage for allocating : %w", err)
2020-03-23 11:40:02 +00:00
}
var best string
var bestID ID
for _, si := range sis {
p, ok := st.paths[si.ID]
if !ok {
continue
}
if p.local == "" { // TODO: can that even be the case?
continue
}
2020-09-06 16:54:00 +00:00
if (pathType == storiface.PathSealing) && !si.CanSeal {
2020-03-23 11:40:02 +00:00
continue
}
2020-09-06 16:54:00 +00:00
if (pathType == storiface.PathStorage) && !si.CanStore {
2020-03-23 11:40:02 +00:00
continue
}
// TODO: Check free space
best = p.sectorPath(sid.ID, fileType)
2020-03-23 11:40:02 +00:00
bestID = si.ID
break
2020-03-23 11:40:02 +00:00
}
if best == "" {
2020-09-06 16:54:00 +00:00
return storiface.SectorPaths{}, storiface.SectorPaths{}, xerrors.Errorf("couldn't find a suitable path for a sector")
2020-03-23 11:40:02 +00:00
}
2020-09-06 16:54:00 +00:00
storiface.SetPathByType(&out, fileType, best)
storiface.SetPathByType(&storageIDs, fileType, string(bestID))
2020-03-23 11:40:02 +00:00
allocate ^= fileType
}
2020-06-04 19:00:16 +00:00
return out, storageIDs, nil
2020-03-23 11:40:02 +00:00
}
func (st *Local) Local(ctx context.Context) ([]StoragePath, error) {
st.localLk.RLock()
defer st.localLk.RUnlock()
var out []StoragePath
for id, p := range st.paths {
if p.local == "" {
continue
}
si, err := st.index.StorageInfo(ctx, id)
if err != nil {
return nil, xerrors.Errorf("get storage info for %s: %w", id, err)
}
out = append(out, StoragePath{
ID: id,
Weight: si.Weight,
LocalPath: p.local,
CanSeal: si.CanSeal,
CanStore: si.CanStore,
})
}
return out, nil
}
func (st *Local) Remove(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType, force bool, keepIn []ID) error {
2020-03-23 11:40:02 +00:00
if bits.OnesCount(uint(typ)) != 1 {
return xerrors.New("delete expects one file type")
}
2020-08-11 07:27:03 +00:00
si, err := st.index.StorageFindSector(ctx, sid, typ, 0, false)
2020-03-23 11:40:02 +00:00
if err != nil {
return xerrors.Errorf("finding existing sector %d(t:%d) failed: %w", sid, typ, err)
}
2020-05-13 18:45:14 +00:00
if len(si) == 0 && !force {
return xerrors.Errorf("can't delete sector %v(%d), not found", sid, typ)
}
storeLoop:
2020-03-23 11:40:02 +00:00
for _, info := range si {
for _, id := range keepIn {
if id == info.ID {
continue storeLoop
}
}
if err := st.removeSector(ctx, sid, typ, info.ID); err != nil {
return err
2020-03-23 11:40:02 +00:00
}
}
return nil
}
2020-09-06 16:54:00 +00:00
func (st *Local) RemoveCopies(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType) error {
if bits.OnesCount(uint(typ)) != 1 {
return xerrors.New("delete expects one file type")
}
2020-03-23 11:40:02 +00:00
2020-08-11 07:27:03 +00:00
si, err := st.index.StorageFindSector(ctx, sid, typ, 0, false)
if err != nil {
return xerrors.Errorf("finding existing sector %d(t:%d) failed: %w", sid, typ, err)
}
var hasPrimary bool
for _, info := range si {
if info.Primary {
hasPrimary = true
break
}
}
if !hasPrimary {
log.Warnf("RemoveCopies: no primary copies of sector %v (%s), not removing anything", sid, typ)
return nil
}
for _, info := range si {
if info.Primary {
2020-03-23 11:40:02 +00:00
continue
}
if err := st.removeSector(ctx, sid, typ, info.ID); err != nil {
return err
}
}
return nil
}
2020-03-23 11:40:02 +00:00
2020-09-06 16:54:00 +00:00
func (st *Local) removeSector(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType, storage ID) error {
p, ok := st.paths[storage]
if !ok {
return nil
}
if p.local == "" { // TODO: can that even be the case?
return nil
}
if err := st.index.StorageDropSector(ctx, storage, sid, typ); err != nil {
return xerrors.Errorf("dropping sector from index: %w", err)
}
spath := p.sectorPath(sid, typ)
log.Infof("remove %s", spath)
if err := os.RemoveAll(spath); err != nil {
log.Errorf("removing sector (%v) from %s: %+v", sid, spath, err)
2020-03-23 11:40:02 +00:00
}
2020-11-09 20:43:52 +00:00
st.reportStorage(ctx) // report freed space
2020-03-23 11:40:02 +00:00
return nil
}
func (st *Local) MoveStorage(ctx context.Context, s storage.SectorRef, types storiface.SectorFileType) error {
dest, destIds, err := st.AcquireSector(ctx, s, storiface.FTNone, types, storiface.PathStorage, storiface.AcquireMove)
if err != nil {
return xerrors.Errorf("acquire dest storage: %w", err)
}
src, srcIds, err := st.AcquireSector(ctx, s, types, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove)
if err != nil {
return xerrors.Errorf("acquire src storage: %w", err)
}
2020-09-06 16:54:00 +00:00
for _, fileType := range storiface.PathTypes {
if fileType&types == 0 {
continue
}
2020-09-06 16:54:00 +00:00
sst, err := st.index.StorageInfo(ctx, ID(storiface.PathByType(srcIds, fileType)))
if err != nil {
return xerrors.Errorf("failed to get source storage info: %w", err)
}
2020-09-06 16:54:00 +00:00
dst, err := st.index.StorageInfo(ctx, ID(storiface.PathByType(destIds, fileType)))
if err != nil {
return xerrors.Errorf("failed to get source storage info: %w", err)
}
if sst.ID == dst.ID {
log.Debugf("not moving %v(%d); src and dest are the same", s, fileType)
continue
}
if sst.CanStore {
log.Debugf("not moving %v(%d); source supports storage", s, fileType)
continue
}
log.Debugf("moving %v(%d) to storage: %s(se:%t; st:%t) -> %s(se:%t; st:%t)", s, fileType, sst.ID, sst.CanSeal, sst.CanStore, dst.ID, dst.CanSeal, dst.CanStore)
if err := st.index.StorageDropSector(ctx, ID(storiface.PathByType(srcIds, fileType)), s.ID, fileType); err != nil {
return xerrors.Errorf("dropping source sector from index: %w", err)
}
2020-09-06 16:54:00 +00:00
if err := move(storiface.PathByType(src, fileType), storiface.PathByType(dest, fileType)); err != nil {
// TODO: attempt some recovery (check if src is still there, re-declare)
return xerrors.Errorf("moving sector %v(%d): %w", s, fileType, err)
}
if err := st.index.StorageDeclareSector(ctx, ID(storiface.PathByType(destIds, fileType)), s.ID, fileType, true); err != nil {
2020-09-06 16:54:00 +00:00
return xerrors.Errorf("declare sector %d(t:%d) -> %s: %w", s, fileType, ID(storiface.PathByType(destIds, fileType)), err)
}
}
2020-11-09 20:43:52 +00:00
st.reportStorage(ctx) // report space use changes
return nil
}
var errPathNotFound = xerrors.Errorf("fsstat: path not found")
2020-07-08 14:58:09 +00:00
func (st *Local) FsStat(ctx context.Context, id ID) (fsutil.FsStat, error) {
2020-03-23 11:40:02 +00:00
st.localLk.RLock()
defer st.localLk.RUnlock()
p, ok := st.paths[id]
if !ok {
2020-07-08 14:58:09 +00:00
return fsutil.FsStat{}, errPathNotFound
2020-03-23 11:40:02 +00:00
}
return p.stat(st.localStorage)
2020-03-23 11:40:02 +00:00
}
var _ Store = &Local{}