lotus/storage/sealmgr/stores/local.go

package stores

import (
	"context"
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"

	"github.com/filecoin-project/specs-actors/actors/abi"
	"golang.org/x/xerrors"

	"github.com/filecoin-project/go-sectorbuilder"
	"github.com/filecoin-project/lotus/node/config"
	"github.com/filecoin-project/lotus/storage/sealmgr/sectorutil"
)

type StoragePath struct {
	ID     ID
	Weight uint64

	LocalPath string

	CanSeal  bool
	CanStore bool
}

// [path]/sectorstore.json
type StorageMeta struct {
	ID     ID
	Weight uint64 // 0 = readonly

	CanSeal  bool
	CanStore bool
}

type LocalStorage interface {
	GetStorage() (config.StorageConfig, error)
	SetStorage(func(*config.StorageConfig)) error
}

const MetaFile = "sectorstore.json"

var pathTypes = []sectorbuilder.SectorFileType{sectorbuilder.FTUnsealed, sectorbuilder.FTSealed, sectorbuilder.FTCache}

type Local struct {
	localLk      sync.RWMutex
	localStorage LocalStorage

	paths []*path
}

type path struct {
	lk sync.Mutex

	meta  StorageMeta
	local string

	sectors map[abi.SectorID]sectorbuilder.SectorFileType
}

func NewLocal(ls LocalStorage) (*Local, error) {
	l := &Local{
		localStorage: ls,
	}
	return l, l.open()
}

func (st *Local) OpenPath(p string) error {
	mb, err := ioutil.ReadFile(filepath.Join(p, MetaFile))
	if err != nil {
		return xerrors.Errorf("reading storage metadata for %s: %w", p, err)
	}

	var meta StorageMeta
	if err := json.Unmarshal(mb, &meta); err != nil {
		return xerrors.Errorf("unmarshalling storage metadata for %s: %w", p, err)
	}

	// TODO: Check existing / dedupe

	out := &path{
		meta:    meta,
		local:   p,
		sectors: map[abi.SectorID]sectorbuilder.SectorFileType{},
	}

	for _, t := range pathTypes {
		ents, err := ioutil.ReadDir(filepath.Join(p, t.String()))
		if err != nil {
			if os.IsNotExist(err) {
				if err := os.MkdirAll(filepath.Join(p, t.String()), 0755); err != nil {
					return xerrors.Errorf("openPath mkdir '%s': %w", filepath.Join(p, t.String()), err)
				}

				continue
			}
			return xerrors.Errorf("listing %s: %w", filepath.Join(p, t.String()), err)
		}

		for _, ent := range ents {
			sid, err := sectorutil.ParseSectorID(ent.Name())
			if err != nil {
				return xerrors.Errorf("parse sector id %s: %w", ent.Name(), err)
			}

			out.sectors[sid] |= t
		}
	}

	st.paths = append(st.paths, out)

	return nil
}

func (st *Local) open() error {
	st.localLk.Lock()
	defer st.localLk.Unlock()

	cfg, err := st.localStorage.GetStorage()
	if err != nil {
		return xerrors.Errorf("getting local storage config: %w", err)
	}

	for _, path := range cfg.StoragePaths {
		err := st.OpenPath(path.Path)
		if err != nil {
			return xerrors.Errorf("opening path %s: %w", path.Path, err)
		}
	}

	return nil
}

func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, sectorbuilder.SectorPaths, func(), error) {
	if existing|allocate != existing^allocate {
		return sectorbuilder.SectorPaths{}, sectorbuilder.SectorPaths{}, nil, xerrors.New("can't both find and allocate a sector")
	}

	st.localLk.RLock()

	var out sectorbuilder.SectorPaths
	var storageIDs sectorbuilder.SectorPaths

	for _, fileType := range pathTypes {
		if fileType&existing == 0 {
			continue
		}

		for _, p := range st.paths {
			p.lk.Lock()
			s, ok := p.sectors[sid]
			p.lk.Unlock()
			if !ok {
				continue
			}
			if s&fileType == 0 {
				continue
			}
			if p.local == "" {
				continue
			}

			spath := filepath.Join(p.local, fileType.String(), sectorutil.SectorName(sid))
			sectorutil.SetPathByType(&out, fileType, spath)
			sectorutil.SetPathByType(&storageIDs, fileType, string(p.meta.ID))

			existing ^= fileType
		}
	}

	for _, fileType := range pathTypes {
		if fileType&allocate == 0 {
			continue
		}

		var best string
		var bestID ID

		for _, p := range st.paths {
			if sealing && !p.meta.CanSeal {
				continue
			}
			if !sealing && !p.meta.CanStore {
				continue
			}

			p.lk.Lock()
			p.sectors[sid] |= fileType
			p.lk.Unlock()

			// TODO: Check free space
			// TODO: Calc weights

			best = filepath.Join(p.local, fileType.String(), sectorutil.SectorName(sid))
			bestID = p.meta.ID
			break // todo: the first path won't always be the best
		}

		if best == "" {
			st.localLk.RUnlock()
			return sectorbuilder.SectorPaths{}, sectorbuilder.SectorPaths{}, nil, xerrors.Errorf("couldn't find a suitable path for a sector")
		}

		sectorutil.SetPathByType(&out, fileType, best)
		sectorutil.SetPathByType(&storageIDs, fileType, string(bestID))
		allocate ^= fileType
	}

	return out, storageIDs, st.localLk.RUnlock, nil
}

func (st *Local) FindBestAllocStorage(allocate sectorbuilder.SectorFileType, sealing bool) ([]StorageMeta, error) {
	st.localLk.RLock()
	defer st.localLk.RUnlock()

	var out []StorageMeta

	for _, p := range st.paths {
		if sealing && !p.meta.CanSeal {
			log.Debugf("alloc: not considering %s; can't seal", p.meta.ID)
			continue
		}
		if !sealing && !p.meta.CanStore {
			log.Debugf("alloc: not considering %s; can't store", p.meta.ID)
			continue
		}

		// TODO: filter out of space

		out = append(out, p.meta)
	}

	if len(out) == 0 {
		return nil, xerrors.New("no good path found")
	}

	// todo: sort by some kind of preference
	return out, nil
}

func (st *Local) FindSector(id abi.SectorID, typ sectorbuilder.SectorFileType) ([]StorageMeta, error) {
	st.localLk.RLock()
	defer st.localLk.RUnlock()

	var out []StorageMeta
	for _, p := range st.paths {
		p.lk.Lock()
		t := p.sectors[id]
		if t|typ == 0 {
			continue
		}
		p.lk.Unlock()
		out = append(out, p.meta)
	}
	if len(out) == 0 {
		return nil, xerrors.Errorf("sector %s/s-t0%d-%d not found", typ, id.Miner, id.Number)
	}

	return out, nil
}

func (st *Local) Local() []StoragePath {
	st.localLk.RLock()
	defer st.localLk.RUnlock()

	var out []StoragePath
	for _, p := range st.paths {
		if p.local == "" {
			continue
		}

		out = append(out, StoragePath{
			ID:        p.meta.ID,
			Weight:    p.meta.Weight,
			LocalPath: p.local,
			CanSeal:   p.meta.CanSeal,
			CanStore:  p.meta.CanStore,
		})
	}

	return out
}

func (st *Local) List(id ID) map[abi.SectorID]sectorbuilder.SectorFileType {
	out := map[abi.SectorID]sectorbuilder.SectorFileType{}
	for _, p := range st.paths {
		if p.meta.ID != id { // TODO: not very efficient
			continue
		}

		for id, fileType := range p.sectors {
			out[id] |= fileType
		}
	}
	return out
}