cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

WIP: sector index yugabyte impl

Browse Source
This commit is contained in:
Shrenuj Bansal 2023-08-04 11:35:31 -04:00
parent 1d58bf09ca
commit 864e8219ae
5 changed files with 782 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmd/lotus-miner/init.go
View File

@ -7,6 +7,8 @@ import (
"encoding/binary"
"encoding/json"
"fmt"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
logging "github.com/ipfs/go-log/v2"
"net/http"
"os"
"path/filepath"
@ -463,7 +465,10 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode
wsts := statestore.New(namespace.Wrap(mds, modules.WorkerCallsPrefix))
smsts := statestore.New(namespace.Wrap(mds, modules.ManagerWorkPrefix))
si := paths.NewIndex(nil)
harmonyDB, err := harmonydb.New([]string{"127.0.0.1"}, "yugabyte", "yugabyte", "yugabyte", "5433", "",
func(s string) { logging.Logger("harmonydb").Error(s) })
si := paths.NewIndex(nil, harmonyDB)
lstor, err := paths.NewLocal(ctx, lr, si, nil)
if err != nil {

3
itests/harmonydb_test.go
View File

@ -42,7 +42,8 @@ func TestCrud(t *testing.T) {
Animal string `db:"content"`
Unpopulated int
}
err = cdb.Select(ctx, &ints, "SELECT content, some_int FROM itest_scratch")
contentfilter := []string{"cows", "cats"}
err = cdb.Select(ctx, &ints, "SELECT content, some_int FROM itest_scratch where content = ANY($1)", contentfilter)
if err != nil {
t.Fatal("Could not select: ", err)
}

2
lib/harmony/harmonydb/userfuncs.go
View File

@ -88,7 +88,7 @@ Ex:
pet := "cat"
err := db.Select(ctx, &users, "SELECT name, id, tel_no FROM customers WHERE pet=?", pet)
*/
func (db *DB) Select(ctx context.Context, sliceOfStructPtr any, sql rawStringOnly, arguments ...any) error {
func (db *DB) Select(ctx context.Context, sliceOfStructPtr any, sql string, arguments ...any) error {
return pgxscan.Select(ctx, db.pgx, sliceOfStructPtr, string(sql), arguments...)
}

7
node/builder_miner.go
View File

@ -126,6 +126,10 @@ func ConfigStorageMiner(c interface{}) Option {
Override(new(sectorblocks.SectorBuilder), From(new(*sealing.Sealing))),
),
Override(new(*harmonydb.DB), func(cfg config.HarmonyDB, id harmonydb.ITestID) (*harmonydb.DB, error) {
return harmonydb.NewFromConfigWithITestID(cfg)(id)
}),
If(cfg.Subsystems.EnableSectorStorage,
// Sector storage
Override(new(*paths.Index), paths.NewIndex),
@ -234,9 +238,6 @@ func ConfigStorageMiner(c interface{}) Option {
Override(new(config.HarmonyDB), cfg.HarmonyDB),
Override(new(harmonydb.ITestID), harmonydb.ITestID("")),
Override(new(*ctladdr.AddressSelector), modules.AddressSelector(&cfg.Addresses)),
Override(new(*harmonydb.DB), func(cfg config.HarmonyDB, id harmonydb.ITestID) (*harmonydb.DB, error) {
return harmonydb.NewFromConfigWithITestID(cfg)(id)
}),
)
}

772
storage/paths/index.go
View File

@ -2,11 +2,14 @@ package paths
import (
"context"
"database/sql"
"errors"
"fmt"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"net/url"
gopath "path"
"sort"
"strings"
"sync"
"time"
@ -71,9 +74,27 @@ type Index struct {
sectors map[storiface.Decl][]*declMeta
stores map[storiface.ID]*storageEntry
dbi *DBIndex
}
func NewIndex(al *alerting.Alerting) *Index {
type DBIndex struct {
alerting *alerting.Alerting
pathAlerts map[storiface.ID]alerting.AlertType
harmonyDB *harmonydb.DB
}
func NewDBIndex(al *alerting.Alerting, db *harmonydb.DB) *DBIndex {
return &DBIndex{
harmonyDB: db,
alerting: al,
pathAlerts: map[storiface.ID]alerting.AlertType{},
}
}
func NewIndex(al *alerting.Alerting, db *harmonydb.DB) *Index {
return &Index{
indexLocks: &indexLocks{
locks: map[abi.SectorID]*sectorLock{},
@ -84,9 +105,88 @@ func NewIndex(al *alerting.Alerting) *Index {
sectors: map[storiface.Decl][]*declMeta{},
stores: map[storiface.ID]*storageEntry{},
dbi: NewDBIndex(al, db),
}
}
func (dbi *DBIndex) StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) {
var sectorEntries []struct {
Storage_id string
Miner_id sql.NullInt64
Sector_num sql.NullInt64
Sector_filetype sql.NullInt32
Is_primary sql.NullBool
}
err := dbi.harmonyDB.Select(ctx, &sectorEntries,
"select stor.storage_id, miner_id, sector_num, sector_filetype, is_primary from storagelocation stor left join sectorlocation sec on stor.storage_id=sec.storage_id")
if err != nil {
return nil, xerrors.Errorf("StorageList DB query fails: ", err)
}
log.Errorf("Sector entries: ", sectorEntries)
byID := map[storiface.ID]map[abi.SectorID]storiface.SectorFileType{}
for _, entry := range sectorEntries {
id := storiface.ID(entry.Storage_id)
_, ok := byID[id]
if !ok {
byID[id] = map[abi.SectorID]storiface.SectorFileType{}
}
// skip sector info for storage paths with no sectors
if !entry.Miner_id.Valid {
continue
}
sectorId := abi.SectorID{
Miner: abi.ActorID(entry.Miner_id.Int64),
Number: abi.SectorNumber(entry.Sector_num.Int64),
}
byID[id][sectorId] |= storiface.SectorFileType(entry.Sector_filetype.Int32)
}
out := map[storiface.ID][]storiface.Decl{}
for id, m := range byID {
out[id] = []storiface.Decl{}
for sectorID, fileType := range m {
out[id] = append(out[id], storiface.Decl{
SectorID: sectorID,
SectorFileType: fileType,
})
}
}
//for _, entry := range sectorEntries {
//
// id := storiface.ID(entry.Storage_id)
// _, ok := out[id]
// if !ok {
// out[id] = []storiface.Decl{}
// }
//
// // skip sector info for storage paths with no sectors
// if !entry.Miner_id.Valid {
// continue
// }
//
// sectorId := abi.SectorID{
// Miner: abi.ActorID(entry.Miner_id.Int64),
// Number: abi.SectorNumber(entry.Sector_num.Int64),
// }
// decl := storiface.Decl{
// SectorID: sectorId,
// SectorFileType: storiface.SectorFileType(entry.Sector_filetype.Int32),
// }
//
// out[id] = append(out[id], decl)
//}
return out, nil
}
func (i *Index) StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) {
i.lk.RLock()
defer i.lk.RUnlock()
@ -113,7 +213,160 @@ func (i *Index) StorageList(ctx context.Context) (map[storiface.ID][]storiface.D
}
}
return out, nil
log.Errorf("In memory index storage list: ", out)
dbout, err := i.dbi.StorageList(ctx)
if err != nil {
log.Errorf("DB Index storage list error: ", err)
return nil, err
} else {
log.Errorf("DB index storage list: ", dbout)
}
return dbout, nil
}
func union(a, b []string) []string {
m := make(map[string]bool)
for _, elem := range a {
m[elem] = true
}
for _, elem := range b {
if _, ok := m[elem]; !ok {
a = append(a, elem)
}
}
return a
}
func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo, st fsutil.FsStat) error {
var allow, deny = make([]string, 0, len(si.AllowTypes)), make([]string, 0, len(si.DenyTypes))
//if _, hasAlert := dbi.pathAlerts[si.ID]; i.alerting != nil && !hasAlert {
// i.pathAlerts[si.ID] = i.alerting.AddAlertType("sector-index", "pathconf-"+string(si.ID))
//}
//
//var hasConfigIssues bool
for _, typ := range si.AllowTypes {
_, err := storiface.TypeFromString(typ)
if err != nil {
// No need to hard-fail here, just warn the user
// (note that even with all-invalid entries we'll deny all types, so nothing unexpected should enter the path)
//hasConfigIssues = true
//if i.alerting != nil {
// i.alerting.Raise(i.pathAlerts[si.ID], map[string]interface{}{
// "message": "bad path type in AllowTypes",
// "path": string(si.ID),
// "idx": id,
// "path_type": typ,
// "error": err.Error(),
// })
//}
continue
}
allow = append(allow, typ)
}
for _, typ := range si.DenyTypes {
_, err := storiface.TypeFromString(typ)
if err != nil {
// No need to hard-fail here, just warn the user
//hasConfigIssues = true
//if i.alerting != nil {
// i.alerting.Raise(i.pathAlerts[si.ID], map[string]interface{}{
// "message": "bad path type in DenyTypes",
// "path": string(si.ID),
// "idx": id,
// "path_type": typ,
// "error": err.Error(),
// })
//}
continue
}
deny = append(deny, typ)
}
si.AllowTypes = allow
si.DenyTypes = deny
//if i.alerting != nil && !hasConfigIssues && i.alerting.IsRaised(i.pathAlerts[si.ID]) {
// i.alerting.Resolve(i.pathAlerts[si.ID], map[string]string{
// "message": "path config is now correct",
// })
//}
for _, u := range si.URLs {
if _, err := url.Parse(u); err != nil {
return xerrors.Errorf("failed to parse url %s: %w", si.URLs, err)
}
}
var urls sql.NullString
var storageId sql.NullString
err := dbi.harmonyDB.QueryRow(ctx,
"Select storage_id, urls from StorageLocation where storage_id = $1", string(si.ID)).Scan(&storageId, &urls)
if err != nil && !strings.Contains(err.Error(), "no rows in result set") {
return xerrors.Errorf("storage attach select fails: ", err)
}
// Storage ID entry exists
if storageId.Valid {
var currUrls []string
if urls.Valid {
currUrls = strings.Split(urls.String, ",")
}
currUrls = union(currUrls, si.URLs)
//updatedUrls := strings.Join(currUrls, ",")
_, err = dbi.harmonyDB.Exec(ctx,
"update StorageLocation set urls=$1, weight=$2, max_storage=$3, can_seal=$4, can_store=$5, groups=$6, allow_to=$7, allow_types=$8, deny_types=$9 where storage_id=$10",
strings.Join(currUrls, ","),
si.Weight,
si.MaxStorage,
si.CanSeal,
si.CanStore,
strings.Join(si.Groups, ","),
strings.Join(si.AllowTo, ","),
strings.Join(si.AllowTypes, ","),
strings.Join(si.DenyTypes, ","),
si.ID)
if err != nil {
return xerrors.Errorf("storage attach update fails: ", err)
}
return nil
}
// Insert storage id
_, err = dbi.harmonyDB.Exec(ctx,
"insert into StorageLocation "+
"Values($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)",
si.ID,
strings.Join(si.URLs, ","),
si.Weight,
si.MaxStorage,
si.CanSeal,
si.CanStore,
strings.Join(si.Groups, ","),
strings.Join(si.AllowTo, ","),
strings.Join(si.AllowTypes, ","),
strings.Join(si.DenyTypes, ","),
st.Capacity,
st.Available,
st.FSAvailable,
st.Reserved,
st.Used,
time.Now())
if err != nil {
log.Errorf("StorageAttach insert fails: ", err)
return xerrors.Errorf("StorageAttach insert fails: ", err)
}
return nil
}
func (i *Index) StorageAttach(ctx context.Context, si storiface.StorageInfo, st fsutil.FsStat) error {
@ -215,6 +468,68 @@ func (i *Index) StorageAttach(ctx context.Context, si storiface.StorageInfo, st
lastHeartbeat: time.Now(),
}
err := i.dbi.StorageAttach(ctx, si, st)
if err != nil {
print("db index storage attach error: ", err)
return err
}
return nil
}
func (dbi *DBIndex) StorageDetach(ctx context.Context, id storiface.ID, url string) error {
// If url not in path urls, error out
// if this is only path url for this storage path, drop storage path and sector decls which have this as a storage path
// record stats: droppedEntries, primaryEntries, droppedDecls
var qUrls sql.NullString
err := dbi.harmonyDB.QueryRow(ctx, "select urls from storagelocation where storage_id=$1", string(id)).Scan(&qUrls)
if err != nil {
return err
}
if !qUrls.Valid {
return xerrors.Errorf("No urls available for storage id: ", id)
}
urls := strings.Split(qUrls.String, ",")
var modUrls []string
for _, u := range urls {
if u != url {
modUrls = append(modUrls, u)
}
}
// noop if url doesn't exist in urls
if len(modUrls) == len(urls) {
return nil
}
if len(modUrls) > 0 {
newUrls := strings.Join(modUrls, ",")
_, err := dbi.harmonyDB.Exec(ctx, "update storagelocation set urls=$1 where storage_id=$2", newUrls, id)
if err != nil {
return err
}
log.Warnw("Dropping sector path endpoint", "path", id, "url", url)
} else {
// Drop storage path completely
_, err := dbi.harmonyDB.Exec(ctx, "delete from storagelocation where storage_id=$1", id)
if err != nil {
return err
}
// Drop all sectors entries which use this storage path
_, err = dbi.harmonyDB.Exec(ctx, "delete from sectorlocation where storage_id=$1", id)
if err != nil {
return err
}
log.Warnw("Dropping sector storage", "path", id)
}
return nil
}
@ -303,6 +618,58 @@ func (i *Index) StorageDetach(ctx context.Context, id storiface.ID, url string)
log.Warnw("Dropping sector path endpoint", "path", id, "url", url)
}
err := i.dbi.StorageDetach(ctx, id, url)
if err != nil {
return xerrors.Errorf("storage detach fails with the error: ", err)
}
return nil
}
func (dbi *DBIndex) StorageReportHealth(ctx context.Context, id storiface.ID, report storiface.HealthReport) error {
var canSeal, canStore bool
err := dbi.harmonyDB.QueryRow(ctx,
"select can_seal, can_store from storagelocation where storage_id=$1", id).Scan(&canSeal, &canStore)
if err != nil {
return xerrors.Errorf("Querying for storage id %d fails with err %w", id, err)
}
_, err = dbi.harmonyDB.Exec(ctx,
"update storagelocation set capacity=$1, available=$2, fs_available=$3, reserved=$4, used=$5, last_heartbeat=$6",
report.Stat.Capacity,
report.Stat.Available,
report.Stat.FSAvailable,
report.Stat.Reserved,
report.Stat.Used,
time.Now())
if err != nil {
return xerrors.Errorf("updating storage health in DB fails with err: ", err)
}
if report.Stat.Capacity > 0 {
ctx, _ = tag.New(ctx,
tag.Upsert(metrics.StorageID, string(id)),
tag.Upsert(metrics.PathStorage, fmt.Sprint(canStore)),
tag.Upsert(metrics.PathSeal, fmt.Sprint(canSeal)),
)
stats.Record(ctx, metrics.StorageFSAvailable.M(float64(report.Stat.FSAvailable)/float64(report.Stat.Capacity)))
stats.Record(ctx, metrics.StorageAvailable.M(float64(report.Stat.Available)/float64(report.Stat.Capacity)))
stats.Record(ctx, metrics.StorageReserved.M(float64(report.Stat.Reserved)/float64(report.Stat.Capacity)))
stats.Record(ctx, metrics.StorageCapacityBytes.M(report.Stat.Capacity))
stats.Record(ctx, metrics.StorageFSAvailableBytes.M(report.Stat.FSAvailable))
stats.Record(ctx, metrics.StorageAvailableBytes.M(report.Stat.Available))
stats.Record(ctx, metrics.StorageReservedBytes.M(report.Stat.Reserved))
if report.Stat.Max > 0 {
stats.Record(ctx, metrics.StorageLimitUsed.M(float64(report.Stat.Used)/float64(report.Stat.Max)))
stats.Record(ctx, metrics.StorageLimitUsedBytes.M(report.Stat.Used))
stats.Record(ctx, metrics.StorageLimitMaxBytes.M(report.Stat.Max))
}
}
return nil
}
@ -349,10 +716,60 @@ func (i *Index) StorageReportHealth(ctx context.Context, id storiface.ID, report
return nil
}
func (dbi *DBIndex) StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error {
ftValid := false
for _, fileType := range storiface.PathTypes {
if fileType&ft == 0 {
ftValid = true
break
}
}
if !ftValid {
return xerrors.Errorf("Invalid filetype")
}
// TODO: make the belowpart of a single transaction
var currPrimary sql.NullBool
err := dbi.harmonyDB.QueryRow(ctx,
"select is_primary from SectorLocation where miner_id=$1 and sector_num=$2 and sector_filetype=$3 and storage_id=$4",
uint64(s.Miner), uint64(s.Number), int(ft), string(storageID)).Scan(&currPrimary)
if err != nil && !strings.Contains(err.Error(), "no rows in result set") {
return xerrors.Errorf("DB select fails: ", err)
}
// If storage id already exists for this sector, update primary if need be
if currPrimary.Valid {
if !currPrimary.Bool && primary {
_, err = dbi.harmonyDB.Exec(ctx,
"update SectorLocation set is_primary = TRUE where miner_id=$1 and sector_num=$2 and sector_filetype=$3 and storage_id=$4",
s.Miner, s.Number, ft, storageID)
if err != nil {
return xerrors.Errorf("DB update fails: ", err)
}
} else {
log.Warnf("sector %v redeclared in %s", s, storageID)
}
} else {
_, err = dbi.harmonyDB.Exec(ctx,
"insert into SectorLocation "+
"values($1, $2, $3, $4, $5)",
s.Miner, s.Number, ft, storageID, primary)
if err != nil {
return xerrors.Errorf("DB insert fails: ", err)
}
}
return nil
}
func (i *Index) StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error {
i.lk.Lock()
defer i.lk.Unlock()
log.Errorf("StorageDeclareSector: %v, %v, %v, %v", storageID, s, ft, primary)
loop:
for _, fileType := range storiface.PathTypes {
if fileType&ft == 0 {
@ -378,6 +795,35 @@ loop:
})
}
err := i.dbi.StorageDeclareSector(ctx, storageID, s, ft, primary)
if err != nil {
print("DB index StorageDeclareSector err: ", err)
return err
}
return nil
}
func (dbi *DBIndex) StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error {
ftValid := false
for _, fileType := range storiface.PathTypes {
if fileType&ft == 0 {
ftValid = true
break
}
}
if !ftValid {
return xerrors.Errorf("Invalid filetype")
}
_, err := dbi.harmonyDB.Exec(ctx,
"delete from sectorlocation where miner_id=$1 and sector_num=$2 and sector_filetype=$3 and storage_id=$4",
int(s.Miner), int(s.Number), int(ft), string(storageID))
if err != nil {
return xerrors.Errorf("StorageDropSector delete query fails: ", err)
}
return nil
}
@ -385,6 +831,8 @@ func (i *Index) StorageDropSector(ctx context.Context, storageID storiface.ID, s
i.lk.Lock()
defer i.lk.Unlock()
log.Errorf("StorageDropSector: %v, %v, %v", storageID, s, ft)
for _, fileType := range storiface.PathTypes {
if fileType&ft == 0 {
continue
@ -412,13 +860,207 @@ func (i *Index) StorageDropSector(ctx context.Context, storageID storiface.ID, s
i.sectors[d] = rewritten
}
err := i.dbi.StorageDropSector(ctx, storageID, s, ft)
if err != nil {
return xerrors.Errorf("DB StorageDropSector fails with err: ", err)
}
return nil
}
func (dbi *DBIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error) {
var result []storiface.SectorStorageInfo
allowList := make(map[string]struct{})
storageWithSector := map[string]bool{}
type dbRes struct {
Storage_id string
Count uint64
Is_primary bool
Urls string
Weight uint64
Can_seal bool
Can_store bool
Groups string
Allow_to string
Allow_types string
Deny_types string
}
var rows []dbRes
fts := ft.AllSet()
// Find all storage info which already hold this sector + filetype
err := dbi.harmonyDB.Select(ctx, &rows,
` SELECT DISTINCT ON (stor.storage_id)
stor.storage_id,
COUNT(*) OVER(PARTITION BY stor.storage_id) as count,
BOOL_OR(is_primary) OVER(PARTITION BY stor.storage_id) AS is_primary,
urls,
weight,
can_seal,
can_store,
groups,
allow_to,
allow_types,
deny_types
FROM sectorlocation sec
JOIN storagelocation stor ON sec.storage_id = stor.storage_id
WHERE sec.miner_id = $1
AND sec.sector_num = $2
AND sec.sector_filetype = ANY($3)
ORDER BY stor.storage_id`,
s.Miner, s.Number, fts)
if err != nil {
return nil, xerrors.Errorf("Finding sector storage from DB fails with err: ", err)
}
for _, row := range rows {
// Parse all urls
var urls, burls []string
for _, u := range strings.Split(row.Urls, ",") {
rl, err := url.Parse(u)
if err != nil {
return nil, xerrors.Errorf("failed to parse url: %w", err)
}
rl.Path = gopath.Join(rl.Path, ft.String(), storiface.SectorName(s))
urls = append(urls, rl.String())
burls = append(burls, u)
}
result = append(result, storiface.SectorStorageInfo{
ID: storiface.ID(row.Storage_id),
URLs: urls,
BaseURLs: burls,
Weight: row.Weight * row.Count,
CanSeal: row.Can_seal,
CanStore: row.Can_store,
Primary: row.Is_primary,
AllowTypes: strings.Split(row.Allow_types, ","),
DenyTypes: strings.Split(row.Deny_types, ","),
})
storageWithSector[row.Storage_id] = true
allowTo := strings.Split(row.Allow_to, ",")
if allowList != nil && len(allowTo) > 0 {
for _, group := range allowTo {
allowList[group] = struct{}{}
}
} else {
allowList = nil // allow to any
}
}
// Find all storage paths which can hold this sector if allowFetch is true
if allowFetch {
spaceReq, err := ft.SealSpaceUse(ssize)
if err != nil {
return nil, xerrors.Errorf("estimating required space: %w", err)
}
// Conditions to satisfy when choosing a sector
// 1. CanSeal is true
// 2. Available >= spaceReq
// 3. curr_time - last_heartbeat < SkippedHeartbeatThresh
// 4. heartbeat_err is NULL
// 5. not one of the earlier picked storage ids
// 6. !ft.AnyAllowed(st.info.AllowTypes, st.info.DenyTypes)
// 7. Storage path is part of the groups which are allowed from the storage paths which already hold the sector
var rows []struct {
Storage_id string
Urls string
Weight uint64
Can_seal bool
Can_store bool
Groups string
Allow_types string
Deny_types string
}
err = dbi.harmonyDB.Select(ctx, &rows,
`select storage_id,
urls,
weight,
can_seal,
can_store,
groups,
allow_types,
deny_types
from storagelocation
where can_seal=true
and available >= $1
and NOW()-last_heartbeat < $2
and heartbeat_err is null`,
spaceReq, SkippedHeartbeatThresh)
if err != nil {
return nil, xerrors.Errorf("Selecting allowfetch storage paths from DB fails err: ", err)
}
for _, row := range rows {
if ok := storageWithSector[row.Storage_id]; ok {
continue
}
if !ft.AnyAllowed(strings.Split(row.Allow_types, ","), strings.Split(row.Deny_types, ",")) {
log.Debugf("not selecting on %s, not allowed by file type filters", row.Storage_id)
continue
}
if allowList != nil {
groups := strings.Split(row.Groups, ",")
allow := false
for _, group := range groups {
if _, found := allowList[group]; found {
log.Debugf("path %s in allowed group %s", row.Storage_id, group)
allow = true
break
}
}
if !allow {
log.Debugf("not selecting on %s, not in allowed group, allow %+v; path has %+v", row.Storage_id, allowList, groups)
continue
}
}
var urls, burls []string
for _, u := range strings.Split(row.Urls, ",") {
rl, err := url.Parse(u)
if err != nil {
return nil, xerrors.Errorf("failed to parse url: %w", err)
}
rl.Path = gopath.Join(rl.Path, ft.String(), storiface.SectorName(s))
urls = append(urls, rl.String())
burls = append(burls, u)
}
result = append(result, storiface.SectorStorageInfo{
ID: storiface.ID(row.Storage_id),
URLs: urls,
BaseURLs: burls,
Weight: row.Weight,
CanSeal: row.Can_seal,
CanStore: row.Can_store,
Primary: false,
AllowTypes: strings.Split(row.Allow_types, ","),
DenyTypes: strings.Split(row.Deny_types, ","),
})
}
}
return result, nil
}
func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error) {
i.lk.RLock()
defer i.lk.RUnlock()
log.Errorf("StorageFindSector: %v, %v, %v, %v, %v", s, ft, ssize, allowFetch, i.sectors)
storageIDs := map[storiface.ID]uint64{}
isprimary := map[storiface.ID]bool{}
@ -560,7 +1202,52 @@ func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft storif
}
}
return out, nil
log.Errorf("StorageFindSector out: ", out)
dbout, err := i.dbi.StorageFindSector(ctx, s, ft, ssize, allowFetch)
if err != nil {
return nil, xerrors.Errorf("DB StorageFindSector fails err: ", err)
}
log.Errorf("StorageFindSector dbout: ", dbout)
return dbout, nil
}
func (dbi *DBIndex) StorageInfo(ctx context.Context, id storiface.ID) (storiface.StorageInfo, error) {
var qResults []struct {
Urls string
Weight uint64
Max_storage uint64
Can_seal bool
Can_store bool
Groups string
Allow_to string
Allow_types string
Deny_types string
}
err := dbi.harmonyDB.Select(ctx, &qResults,
"select urls, weight, max_storage, can_seal, can_store, groups, allow_to, allow_types, deny_types "+
"from StorageLocation where storage_id=$1", string(id))
if err != nil {
return storiface.StorageInfo{}, xerrors.Errorf("StorageInfo query fails: ", err)
}
var sinfo storiface.StorageInfo
sinfo.ID = id
sinfo.URLs = strings.Split(qResults[0].Urls, ",")
sinfo.Weight = qResults[0].Weight
sinfo.MaxStorage = qResults[0].Max_storage
sinfo.CanSeal = qResults[0].Can_seal
sinfo.CanStore = qResults[0].Can_store
sinfo.Groups = strings.Split(qResults[0].Groups, ",")
sinfo.AllowTo = strings.Split(qResults[0].Allow_to, ",")
sinfo.AllowTypes = strings.Split(qResults[0].Allow_types, ",")
sinfo.DenyTypes = strings.Split(qResults[0].Deny_types, ",")
return sinfo, nil
}
func (i *Index) StorageInfo(ctx context.Context, id storiface.ID) (storiface.StorageInfo, error) {
@ -575,6 +1262,83 @@ func (i *Index) StorageInfo(ctx context.Context, id storiface.ID) (storiface.Sto
return *si.info, nil
}
func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) {
var err error
var spaceReq uint64
switch pathType {
case storiface.PathSealing:
spaceReq, err = allocate.SealSpaceUse(ssize)
case storiface.PathStorage:
spaceReq, err = allocate.StoreSpaceUse(ssize)
default:
return nil, xerrors.Errorf("Unexpected path type")
}
if err != nil {
return nil, xerrors.Errorf("estimating required space: %w", err)
}
var checkString string
if pathType == storiface.PathSealing {
checkString = "can_seal = TRUE"
} else if pathType == storiface.PathStorage {
checkString = "can_store = TRUE"
}
var rows []struct {
Storage_id string
Urls string
Weight uint64
Max_storage uint64
Can_seal bool
Can_store bool
Groups string
Allow_to string
Allow_types string
Deny_types string
}
sql := fmt.Sprintf(`select storage_id,
urls,
weight,
max_storage,
can_seal,
can_store,
groups,
allow_to,
allow_types,
deny_types
from storagelocation
where %s and available >= $1
and NOW()-last_heartbeat < $2
and heartbeat_err is null
order by available * weight desc`, checkString)
err = dbi.harmonyDB.Select(ctx, &rows,
sql, spaceReq, SkippedHeartbeatThresh)
if err != nil {
return nil, xerrors.Errorf("Querying for best storage sectors fails with sql: %s and err %w: ", sql, err)
}
var result []storiface.StorageInfo
for _, row := range rows {
result = append(result, storiface.StorageInfo{
ID: storiface.ID(row.Storage_id),
URLs: strings.Split(row.Urls, ","),
Weight: row.Weight,
MaxStorage: row.Max_storage,
CanSeal: row.Can_seal,
CanStore: row.Can_store,
Groups: strings.Split(row.Groups, ","),
AllowTo: strings.Split(row.Allow_to, ","),
AllowTypes: strings.Split(row.Allow_types, ","),
DenyTypes: strings.Split(row.Deny_types, ","),
})
}
log.Errorf("StorageBestAlloc result: ", result)
return result, nil
}
func (i *Index) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) {
i.lk.RLock()
defer i.lk.RUnlock()
@ -637,6 +1401,8 @@ func (i *Index) StorageBestAlloc(ctx context.Context, allocate storiface.SectorF
out[i] = *candidate.info
}
return i.dbi.StorageBestAlloc(ctx, allocate, ssize, pathType)
return out, nil
}