lotus/storage/paths/db_index.go

package paths

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"net/url"
	gopath "path"
	"strings"
	"time"

	"github.com/google/uuid"
	"go.opencensus.io/stats"
	"go.opencensus.io/tag"
	"golang.org/x/xerrors"

	"github.com/filecoin-project/go-state-types/abi"

	"github.com/filecoin-project/lotus/journal/alerting"
	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
	"github.com/filecoin-project/lotus/metrics"
	"github.com/filecoin-project/lotus/storage/sealer/fsutil"
	"github.com/filecoin-project/lotus/storage/sealer/storiface"
)

var errAlreadyLocked = errors.New("already locked")

type DBIndex struct {
	alerting   *alerting.Alerting
	pathAlerts map[storiface.ID]alerting.AlertType

	harmonyDB *harmonydb.DB
}

func NewDBIndex(al *alerting.Alerting, db *harmonydb.DB) *DBIndex {
	return &DBIndex{
		harmonyDB: db,

		alerting:   al,
		pathAlerts: map[storiface.ID]alerting.AlertType{},
	}
}

func (dbi *DBIndex) StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) {

	var sectorEntries []struct {
		StorageId      string
		MinerId        sql.NullInt64
		SectorNum      sql.NullInt64
		SectorFiletype sql.NullInt32 `db:"sector_filetype"`
		IsPrimary      sql.NullBool
	}

	err := dbi.harmonyDB.Select(ctx, &sectorEntries,
		"SELECT stor.storage_id, miner_id, sector_num, sector_filetype, is_primary FROM storage_path stor LEFT JOIN sector_location sec on stor.storage_id=sec.storage_id")
	if err != nil {
		return nil, xerrors.Errorf("StorageList DB query fails: %v", err)
	}

	byID := map[storiface.ID]map[abi.SectorID]storiface.SectorFileType{}
	for _, entry := range sectorEntries {
		id := storiface.ID(entry.StorageId)
		_, ok := byID[id]
		if !ok {
			byID[id] = map[abi.SectorID]storiface.SectorFileType{}
		}

		// skip sector info for storage paths with no sectors
		if !entry.MinerId.Valid {
			continue
		}

		sectorId := abi.SectorID{
			Miner:  abi.ActorID(entry.MinerId.Int64),
			Number: abi.SectorNumber(entry.SectorNum.Int64),
		}

		byID[id][sectorId] |= storiface.SectorFileType(entry.SectorFiletype.Int32)
	}

	out := map[storiface.ID][]storiface.Decl{}
	for id, m := range byID {
		out[id] = []storiface.Decl{}
		for sectorID, fileType := range m {
			out[id] = append(out[id], storiface.Decl{
				SectorID:       sectorID,
				SectorFileType: fileType,
			})
		}
	}

	return out, nil
}

func union(a, b []string) []string {
	m := make(map[string]bool)

	for _, elem := range a {
		m[elem] = true
	}

	for _, elem := range b {
		if _, ok := m[elem]; !ok {
			a = append(a, elem)
		}
	}
	return a
}

func splitString(str string) []string {
	if str == "" {
		return []string{}
	}
	return strings.Split(str, ",")
}

func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo, st fsutil.FsStat) error {
	var allow, deny = make([]string, 0, len(si.AllowTypes)), make([]string, 0, len(si.DenyTypes))

	if _, hasAlert := dbi.pathAlerts[si.ID]; dbi.alerting != nil && !hasAlert {
		dbi.pathAlerts[si.ID] = dbi.alerting.AddAlertType("sector-index", "pathconf-"+string(si.ID))
	}

	var hasConfigIssues bool

	for id, typ := range si.AllowTypes {
		_, err := storiface.TypeFromString(typ)
		if err != nil {
			//No need to hard-fail here, just warn the user
			//(note that even with all-invalid entries we'll deny all types, so nothing unexpected should enter the path)
			hasConfigIssues = true

			if dbi.alerting != nil {
				dbi.alerting.Raise(dbi.pathAlerts[si.ID], map[string]interface{}{
					"message":   "bad path type in AllowTypes",
					"path":      string(si.ID),
					"idx":       id,
					"path_type": typ,
					"error":     err.Error(),
				})
			}

			continue
		}
		allow = append(allow, typ)
	}
	for id, typ := range si.DenyTypes {
		_, err := storiface.TypeFromString(typ)
		if err != nil {
			//No need to hard-fail here, just warn the user
			hasConfigIssues = true

			if dbi.alerting != nil {
				dbi.alerting.Raise(dbi.pathAlerts[si.ID], map[string]interface{}{
					"message":   "bad path type in DenyTypes",
					"path":      string(si.ID),
					"idx":       id,
					"path_type": typ,
					"error":     err.Error(),
				})
			}

			continue
		}
		deny = append(deny, typ)
	}
	si.AllowTypes = allow
	si.DenyTypes = deny

	if dbi.alerting != nil && !hasConfigIssues && dbi.alerting.IsRaised(dbi.pathAlerts[si.ID]) {
		dbi.alerting.Resolve(dbi.pathAlerts[si.ID], map[string]string{
			"message": "path config is now correct",
		})
	}

	for _, u := range si.URLs {
		if _, err := url.Parse(u); err != nil {
			return xerrors.Errorf("failed to parse url %s: %w", si.URLs, err)
		}
	}

	retryWait := time.Millisecond * 20
retryAttachStorage:
	// Single transaction to attach storage which is not present in the DB
	_, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
		var urls sql.NullString
		var storageId sql.NullString
		err = tx.QueryRow(
			"SELECT storage_id, urls FROM storage_path WHERE storage_id = $1", string(si.ID)).Scan(&storageId, &urls)
		if err != nil && !strings.Contains(err.Error(), "no rows in result set") {
			return false, xerrors.Errorf("storage attach select fails: %v", err)
		}

		// Storage ID entry exists
		// TODO: Consider using insert into .. on conflict do update set ... below
		if storageId.Valid {
			var currUrls []string
			if urls.Valid {
				currUrls = strings.Split(urls.String, ",")
			}
			currUrls = union(currUrls, si.URLs)

			_, err = tx.Exec(
				"UPDATE storage_path set urls=$1, weight=$2, max_storage=$3, can_seal=$4, can_store=$5, groups=$6, allow_to=$7, allow_types=$8, deny_types=$9, last_heartbeat=NOW() WHERE storage_id=$10",
				strings.Join(currUrls, ","),
				si.Weight,
				si.MaxStorage,
				si.CanSeal,
				si.CanStore,
				strings.Join(si.Groups, ","),
				strings.Join(si.AllowTo, ","),
				strings.Join(si.AllowTypes, ","),
				strings.Join(si.DenyTypes, ","),
				si.ID)
			if err != nil {
				return false, xerrors.Errorf("storage attach UPDATE fails: %v", err)
			}

			return true, nil
		}

		// Insert storage id
		_, err = tx.Exec(
			"INSERT INTO storage_path "+
				"Values($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, NOW())",
			si.ID,
			strings.Join(si.URLs, ","),
			si.Weight,
			si.MaxStorage,
			si.CanSeal,
			si.CanStore,
			strings.Join(si.Groups, ","),
			strings.Join(si.AllowTo, ","),
			strings.Join(si.AllowTypes, ","),
			strings.Join(si.DenyTypes, ","),
			st.Capacity,
			st.Available,
			st.FSAvailable,
			st.Reserved,
			st.Used)
		if err != nil {
			return false, xerrors.Errorf("StorageAttach insert fails: %v", err)
		}
		return true, nil
	})
	if err != nil {
		if harmonydb.IsErrSerialization(err) {
			time.Sleep(retryWait)
			retryWait *= 2
			goto retryAttachStorage
		}
		return err
	}

	return nil
}

func (dbi *DBIndex) StorageDetach(ctx context.Context, id storiface.ID, url string) error {

	// If url not in path urls, error out
	// if this is only path url for this storage path, drop storage path and sector decls which have this as a storage path

	var qUrls string
	err := dbi.harmonyDB.QueryRow(ctx, "SELECT COALESCE(urls,'') FROM storage_path WHERE storage_id=$1", string(id)).Scan(&qUrls)
	if err != nil {
		return err
	}
	urls := splitString(qUrls)

	var modUrls []string
	for _, u := range urls {
		if u != url {
			modUrls = append(modUrls, u)
		}
	}

	// noop if url doesn't exist in urls
	if len(modUrls) == len(urls) {
		return nil
	}

	if len(modUrls) > 0 {
		newUrls := strings.Join(modUrls, ",")
		_, err := dbi.harmonyDB.Exec(ctx, "UPDATE storage_path set urls=$1 WHERE storage_id=$2", newUrls, id)
		if err != nil {
			return err
		}

		log.Warnw("Dropping sector path endpoint", "path", id, "url", url)
	} else {
		retryWait := time.Millisecond * 20
	retryDropPath:
		// Single transaction to drop storage path and sector decls which have this as a storage path
		_, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
			// Drop storage path completely
			_, err = tx.Exec("DELETE FROM storage_path WHERE storage_id=$1", id)
			if err != nil {
				return false, err
			}

			// Drop all sectors entries which use this storage path
			_, err = tx.Exec("DELETE FROM sector_location WHERE storage_id=$1", id)
			if err != nil {
				return false, err
			}
			return true, nil
		})
		if err != nil {
			if harmonydb.IsErrSerialization(err) {
				time.Sleep(retryWait)
				retryWait *= 2
				goto retryDropPath
			}
			return err
		}
		log.Warnw("Dropping sector storage", "path", id)
	}

	return nil
}

func (dbi *DBIndex) StorageReportHealth(ctx context.Context, id storiface.ID, report storiface.HealthReport) error {
	retryWait := time.Millisecond * 20
retryReportHealth:
	_, err := dbi.harmonyDB.Exec(ctx,
		"UPDATE storage_path set capacity=$1, available=$2, fs_available=$3, reserved=$4, used=$5, last_heartbeat=NOW() where storage_id=$6",
		report.Stat.Capacity,
		report.Stat.Available,
		report.Stat.FSAvailable,
		report.Stat.Reserved,
		report.Stat.Used,
		id)
	if err != nil {
		//return xerrors.Errorf("updating storage health in DB fails with err: %v", err)
		if harmonydb.IsErrSerialization(err) {
			time.Sleep(retryWait)
			retryWait *= 2
			goto retryReportHealth
		}
		return err
	}

	var canSeal, canStore bool
	err = dbi.harmonyDB.QueryRow(ctx,
		"SELECT can_seal, can_store FROM storage_path WHERE storage_id=$1", id).Scan(&canSeal, &canStore)
	if err != nil {
		return xerrors.Errorf("Querying for storage id %s fails with err %v", id, err)
	}

	if report.Stat.Capacity > 0 {
		ctx, _ = tag.New(ctx,
			tag.Upsert(metrics.StorageID, string(id)),
			tag.Upsert(metrics.PathStorage, fmt.Sprint(canStore)),
			tag.Upsert(metrics.PathSeal, fmt.Sprint(canSeal)),
		)

		stats.Record(ctx, metrics.StorageFSAvailable.M(float64(report.Stat.FSAvailable)/float64(report.Stat.Capacity)))
		stats.Record(ctx, metrics.StorageAvailable.M(float64(report.Stat.Available)/float64(report.Stat.Capacity)))
		stats.Record(ctx, metrics.StorageReserved.M(float64(report.Stat.Reserved)/float64(report.Stat.Capacity)))

		stats.Record(ctx, metrics.StorageCapacityBytes.M(report.Stat.Capacity))
		stats.Record(ctx, metrics.StorageFSAvailableBytes.M(report.Stat.FSAvailable))
		stats.Record(ctx, metrics.StorageAvailableBytes.M(report.Stat.Available))
		stats.Record(ctx, metrics.StorageReservedBytes.M(report.Stat.Reserved))

		if report.Stat.Max > 0 {
			stats.Record(ctx, metrics.StorageLimitUsed.M(float64(report.Stat.Used)/float64(report.Stat.Max)))
			stats.Record(ctx, metrics.StorageLimitUsedBytes.M(report.Stat.Used))
			stats.Record(ctx, metrics.StorageLimitMaxBytes.M(report.Stat.Max))
		}
	}

	return nil
}

// function to check if a filetype is valid
func (dbi *DBIndex) checkFileType(fileType storiface.SectorFileType) bool {
	ftValid := false
	for _, fileTypeValid := range storiface.PathTypes {
		if fileTypeValid&fileType == 0 {
			ftValid = true
			break
		}
	}
	return ftValid
}

func (dbi *DBIndex) StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error {

	if !dbi.checkFileType(ft) {
		return xerrors.Errorf("invalid filetype")
	}

	retryWait := time.Millisecond * 100
retryStorageDeclareSector:
	_, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
		var currPrimary sql.NullBool
		err = tx.QueryRow(
			"SELECT is_primary FROM sector_location WHERE miner_id=$1 and sector_num=$2 and sector_filetype=$3 and storage_id=$4",
			uint64(s.Miner), uint64(s.Number), int(ft), string(storageID)).Scan(&currPrimary)
		if err != nil && !strings.Contains(err.Error(), "no rows in result set") {
			return false, xerrors.Errorf("DB SELECT fails: %v", err)
		}

		// If storage id already exists for this sector, update primary if need be
		if currPrimary.Valid {
			if !currPrimary.Bool && primary {
				_, err = tx.Exec(
					"UPDATE sector_location set is_primary = TRUE WHERE miner_id=$1 and sector_num=$2 and sector_filetype=$3 and storage_id=$4",
					s.Miner, s.Number, ft, storageID)
				if err != nil {
					return false, xerrors.Errorf("DB update fails: %v", err)
				}
			} else {
				log.Warnf("sector %v redeclared in %s", s, storageID)
			}
		} else {
			_, err = tx.Exec(
				"INSERT INTO sector_location "+
					"values($1, $2, $3, $4, $5)",
				s.Miner, s.Number, ft, storageID, primary)
			if err != nil {
				return false, xerrors.Errorf("DB insert fails: %v", err)
			}
		}

		return true, nil
	})
	if err != nil {
		if harmonydb.IsErrSerialization(err) {
			time.Sleep(retryWait)
			retryWait *= 2
			goto retryStorageDeclareSector
		}
		return err
	}

	return nil
}

func (dbi *DBIndex) StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error {

	if !dbi.checkFileType(ft) {
		return xerrors.Errorf("invalid filetype")
	}

	_, err := dbi.harmonyDB.Exec(ctx,
		"DELETE FROM sector_location WHERE miner_id=$1 and sector_num=$2 and sector_filetype=$3 and storage_id=$4",
		int(s.Miner), int(s.Number), int(ft), string(storageID))
	if err != nil {
		return xerrors.Errorf("StorageDropSector DELETE query fails: %v", err)
	}

	return nil
}

func (dbi *DBIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error) {

	var result []storiface.SectorStorageInfo

	allowList := make(map[string]struct{})
	storageWithSector := map[string]bool{}

	type dbRes struct {
		StorageId  string
		Count      uint64
		IsPrimary  bool
		Urls       string
		Weight     uint64
		CanSeal    bool
		CanStore   bool
		Groups     string
		AllowTo    string
		AllowTypes string
		DenyTypes  string
	}

	var rows []dbRes

	fts := ft.AllSet()
	// Find all storage info which already hold this sector + filetype
	err := dbi.harmonyDB.Select(ctx, &rows,
		`		SELECT DISTINCT ON (stor.storage_id)
						  stor.storage_id,
						  COUNT(*) OVER(PARTITION BY stor.storage_id) as count, 
						  BOOL_OR(is_primary) OVER(PARTITION BY stor.storage_id) AS is_primary,
						  urls,
						  weight,
						  can_seal,
						  can_store,
						  groups,
						  allow_to,
						  allow_types,
						  deny_types
						FROM sector_location sec
						JOIN storage_path stor ON sec.storage_id = stor.storage_id 
						WHERE sec.miner_id = $1
						  AND sec.sector_num = $2 
						  AND sec.sector_filetype = ANY($3)  
						ORDER BY stor.storage_id`,
		s.Miner, s.Number, fts)
	if err != nil {
		return nil, xerrors.Errorf("Finding sector storage from DB fails with err: %v", err)
	}

	for _, row := range rows {

		// Parse all urls
		var urls, burls []string
		for _, u := range splitString(row.Urls) {
			rl, err := url.Parse(u)
			if err != nil {
				return nil, xerrors.Errorf("failed to parse url: %w", err)
			}
			rl.Path = gopath.Join(rl.Path, ft.String(), storiface.SectorName(s))
			urls = append(urls, rl.String())
			burls = append(burls, u)
		}

		result = append(result, storiface.SectorStorageInfo{
			ID:         storiface.ID(row.StorageId),
			URLs:       urls,
			BaseURLs:   burls,
			Weight:     row.Weight * row.Count,
			CanSeal:    row.CanSeal,
			CanStore:   row.CanStore,
			Primary:    row.IsPrimary,
			AllowTypes: splitString(row.AllowTypes),
			DenyTypes:  splitString(row.DenyTypes),
		})

		storageWithSector[row.StorageId] = true

		allowTo := splitString(row.AllowTo)
		if allowList != nil && len(allowTo) > 0 {
			for _, group := range allowTo {
				allowList[group] = struct{}{}
			}
		} else {
			allowList = nil // allow to any
		}
	}

	// Find all storage paths which can hold this sector if allowFetch is true
	if allowFetch {
		spaceReq, err := ft.SealSpaceUse(ssize)
		if err != nil {
			return nil, xerrors.Errorf("estimating required space: %w", err)
		}

		// Conditions to satisfy when choosing a sector
		// 1. CanSeal is true
		// 2. Available >= spaceReq
		// 3. curr_time - last_heartbeat < SkippedHeartbeatThresh
		// 4. heartbeat_err is NULL
		// 5. not one of the earlier picked storage ids
		// 6. !ft.AnyAllowed(st.info.AllowTypes, st.info.DenyTypes)
		// 7. Storage path is part of the groups which are allowed from the storage paths which already hold the sector

		var rows []struct {
			StorageId  string
			Urls       string
			Weight     uint64
			CanSeal    bool
			CanStore   bool
			Groups     string
			AllowTypes string
			DenyTypes  string
		}
		err = dbi.harmonyDB.Select(ctx, &rows,
			`SELECT storage_id, 
					  	urls, 
					  	weight,
					  	can_seal,
					  	can_store,
					  	groups, 
					  	allow_types, 
					  	deny_types
				FROM storage_path 
				WHERE can_seal=true 
				  and available >= $1 
				  and NOW()-($2 * INTERVAL '1 second') < last_heartbeat
				  and heartbeat_err is null`,
			spaceReq, SkippedHeartbeatThresh.Seconds())
		if err != nil {
			return nil, xerrors.Errorf("Selecting allowfetch storage paths from DB fails err: %v", err)
		}

		for _, row := range rows {
			if ok := storageWithSector[row.StorageId]; ok {
				continue
			}

			if !ft.AnyAllowed(splitString(row.AllowTypes), splitString(row.DenyTypes)) {
				log.Debugf("not selecting on %s, not allowed by file type filters", row.StorageId)
				continue
			}

			if allowList != nil {
				groups := splitString(row.Groups)
				allow := false
				for _, group := range groups {
					if _, found := allowList[group]; found {
						log.Debugf("path %s in allowed group %s", row.StorageId, group)
						allow = true
						break
					}
				}

				if !allow {
					log.Debugf("not selecting on %s, not in allowed group, allow %+v; path has %+v", row.StorageId, allowList, groups)
					continue
				}
			}

			var urls, burls []string
			for _, u := range splitString(row.Urls) {
				rl, err := url.Parse(u)
				if err != nil {
					return nil, xerrors.Errorf("failed to parse url: %w", err)
				}
				rl.Path = gopath.Join(rl.Path, ft.String(), storiface.SectorName(s))
				urls = append(urls, rl.String())
				burls = append(burls, u)
			}

			result = append(result, storiface.SectorStorageInfo{
				ID:         storiface.ID(row.StorageId),
				URLs:       urls,
				BaseURLs:   burls,
				Weight:     row.Weight * 0,
				CanSeal:    row.CanSeal,
				CanStore:   row.CanStore,
				Primary:    false,
				AllowTypes: splitString(row.AllowTypes),
				DenyTypes:  splitString(row.DenyTypes),
			})
		}
	}

	return result, nil
}

func (dbi *DBIndex) StorageInfo(ctx context.Context, id storiface.ID) (storiface.StorageInfo, error) {

	var qResults []struct {
		Urls       string
		Weight     uint64
		MaxStorage uint64
		CanSeal    bool
		CanStore   bool
		Groups     string
		AllowTo    string
		AllowTypes string
		DenyTypes  string
	}

	err := dbi.harmonyDB.Select(ctx, &qResults,
		"SELECT urls, weight, max_storage, can_seal, can_store, groups, allow_to, allow_types, deny_types "+
			"FROM storage_path WHERE storage_id=$1", string(id))
	if err != nil {
		return storiface.StorageInfo{}, xerrors.Errorf("StorageInfo query fails: %v", err)
	}

	var sinfo storiface.StorageInfo
	sinfo.ID = id
	sinfo.URLs = splitString(qResults[0].Urls)
	sinfo.Weight = qResults[0].Weight
	sinfo.MaxStorage = qResults[0].MaxStorage
	sinfo.CanSeal = qResults[0].CanSeal
	sinfo.CanStore = qResults[0].CanStore
	sinfo.Groups = splitString(qResults[0].Groups)
	sinfo.AllowTo = splitString(qResults[0].AllowTo)
	sinfo.AllowTypes = splitString(qResults[0].AllowTypes)
	sinfo.DenyTypes = splitString(qResults[0].DenyTypes)

	return sinfo, nil
}

func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) {
	var err error
	var spaceReq uint64
	switch pathType {
	case storiface.PathSealing:
		spaceReq, err = allocate.SealSpaceUse(ssize)
	case storiface.PathStorage:
		spaceReq, err = allocate.StoreSpaceUse(ssize)
	default:
		return nil, xerrors.Errorf("unexpected path type")
	}
	if err != nil {
		return nil, xerrors.Errorf("estimating required space: %w", err)
	}

	var rows []struct {
		StorageId  string
		Urls       string
		Weight     uint64
		MaxStorage uint64
		CanSeal    bool
		CanStore   bool
		Groups     string
		AllowTo    string
		AllowTypes string
		DenyTypes  string
	}

	err = dbi.harmonyDB.Select(ctx, &rows,
		`SELECT storage_id, 
								urls, 
								weight,
								max_storage, 
								can_seal, 
								can_store, 
								groups, 
								allow_to, 
								allow_types, 
								deny_types 
						 FROM storage_path 
						 WHERE available >= $1
						 and NOW()-($2 * INTERVAL '1 second') < last_heartbeat
						 and heartbeat_err is null
						 and (($3 and can_seal = TRUE) or ($4 and can_store = TRUE))
						order by (available::numeric * weight) desc`,
		spaceReq,
		SkippedHeartbeatThresh.Seconds(),
		pathType == storiface.PathSealing,
		pathType == storiface.PathStorage,
	)
	if err != nil {
		return nil, xerrors.Errorf("Querying for best storage sectors fails with err %w: ", err)
	}

	var result []storiface.StorageInfo
	for _, row := range rows {
		result = append(result, storiface.StorageInfo{
			ID:         storiface.ID(row.StorageId),
			URLs:       splitString(row.Urls),
			Weight:     row.Weight,
			MaxStorage: row.MaxStorage,
			CanSeal:    row.CanSeal,
			CanStore:   row.CanStore,
			Groups:     splitString(row.Groups),
			AllowTo:    splitString(row.AllowTo),
			AllowTypes: splitString(row.AllowTypes),
			DenyTypes:  splitString(row.DenyTypes),
		})
	}

	return result, nil
}

/*
2024-02-17T01:24:10.243+0100    WARN    stores  paths/remote.go:258     acquireFromRemote encountered errors when fetching sector from remote   {"errors": "1 error occurred:\n\t* fetch error http://10.99.16.4:3456/remote/cache/s-t02620-1 (storage 0bd90137-1219-4fb6-82d4-138a72c692b7) -> /data/3/store/cache/fetching/s-t02620-1: do request: Get \"http://10.99.16.4:3456/remote/cache/s-t02620-1\": dial tcp 10.99.16.4:3456: connect: connection refused\n\n"}
2024-02-17T01:24:10.248+0100    INFO    stores  paths/remote.go:359     Delete http://10.99.16.4:12300/remote/cache/s-t02620-1?keep=0ff4da63-7e0d-477e-a2ee-814c9fd7ef71

==> /var/log/lotus/provider1.log <==
2024-02-17T01:25:32.820+0100    INFO    harmonytask     harmonytask/task_type_handler.go:121    Beginning work on Task  {"id": 7815, "from": "poller", "name": "WinPost"}

==> /var/log/lotus/provider4.log <==
2024-02-17T01:25:45.058+0100    ERROR   harmonytask     harmonytask/task_type_handler.go:131    Recovered from a serious error while processing SDRTrees task 7702: todo Stack: goroutine 334 [running]:
github.com/filecoin-project/lotus/lib/harmony/harmonytask.(*taskTypeHandler).considerWork.func1.1()
        /home/magik6k/github.com/filecoin-project/go-lotus/lib/harmony/harmonytask/task_type_handler.go:130 +0xc5
panic({0x2210100?, 0x4557680?})
        /usr/lib/go/src/runtime/panic.go:770 +0x132
github.com/filecoin-project/lotus/provider/lpffi.(*SealCalls).makePhase1Out(0x10?, {{0xc0009ba120?, 0xf?}}, 0x8)
        /home/magik6k/github.com/filecoin-project/go-lotus/provider/lpffi/sdr_funcs.go:272 +0x445
github.com/filecoin-project/lotus/provider/lpffi.(*SealCalls).TreeRC(0xc000884118, {0x457c9b8, 0xa4c7d40}, {{0xa3c, 0x1}, 0x8}, {{0xc0009ba120?, 0x800000000?}})
        /home/magik6k/github.com/filecoin-project/go-lotus/provider/lpffi/sdr_funcs.go:133 +0x95
github.com/filecoin-project/lotus/provider/lpseal.(*TreesTask).Do(0xc00017c5e0, 0x1e16, 0x6?)
        /home/magik6k/github.com/filecoin-project/go-lotus/provider/lpseal/task_trees.go:149 +0x5ca
github.com/filecoin-project/lotus/lib/harmony/harmonytask.(*taskTypeHandler).considerWork.func1()
        /home/magik6k/github.com/filecoin-project/go-lotus/lib/harmony/harmonytask/task_type_handler.go:147 +0x2a6
created by github.com/filecoin-project/lotus/lib/harmony/harmonytask.(*taskTypeHandler).considerWork in goroutine 230
        /home/magik6k/github.com/filecoin-project/go-lotus/lib/harmony/harmonytask/task_type_handler.go:120 +0x465

*/

// timeout after which we consider a lock to be stale
const LockTimeOut = 300 * time.Second

func isLocked(ts sql.NullTime) bool {
	return ts.Valid && ts.Time.After(time.Now().Add(-LockTimeOut))
}

func (dbi *DBIndex) lock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType, lockUuid uuid.UUID) (bool, error) {
	if read|write == 0 {
		return false, nil
	}

	if read|write > (1<<storiface.FileTypes)-1 {
		return false, xerrors.Errorf("unknown file types specified")
	}

	var rows []struct {
		SectorFileType storiface.SectorFileType `db:"sector_filetype"`
		ReadTs         sql.NullTime
		ReadRefs       int
		WriteTs        sql.NullTime
	}
	_, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {

		fts := (read | write).AllSet()
		err = tx.Select(&rows,
			`SELECT sector_filetype, read_ts, read_refs, write_ts 
								 FROM sector_location 
								 WHERE miner_id=$1 
								   AND sector_num=$2
								   AND sector_filetype = ANY($3)`,
			sector.Miner, sector.Number, fts)
		if err != nil {
			return false, xerrors.Errorf("StorageLock SELECT fails: %v", err)
		}

		type locks struct {
			readTs   sql.NullTime
			readRefs int
			writeTs  sql.NullTime
		}
		lockMap := make(map[storiface.SectorFileType]locks)
		for _, row := range rows {
			lockMap[row.SectorFileType] = locks{
				readTs:   row.ReadTs,
				readRefs: row.ReadRefs,
				writeTs:  row.WriteTs,
			}
		}

		// Check if we can acquire write locks
		// Conditions: No write lock or write lock is stale, No read lock or read lock is stale
		for _, wft := range write.AllSet() {
			if isLocked(lockMap[wft].writeTs) || isLocked(lockMap[wft].readTs) {
				return false, xerrors.Errorf("cannot acquire writelock for sector %v filetype %d already locked: %w", sector, wft, errAlreadyLocked)
			}
		}

		// Check if we can acquire read locks
		// Conditions: No write lock or write lock is stale
		for _, rft := range read.AllSet() {
			if isLocked(lockMap[rft].writeTs) {
				return false, xerrors.Errorf("cannot acquire read lock for sector %v filetype %d already locked for writing: %w", sector, rft, errAlreadyLocked)
			}
		}

		// Acquire write locks
		_, err = tx.Exec(
			`UPDATE sector_location
							 	SET write_ts = NOW(), write_lock_owner = $1
							 	WHERE miner_id=$2
							 		AND sector_num=$3
							 		AND sector_filetype = ANY($4)`,
			lockUuid.String(),
			sector.Miner,
			sector.Number,
			write.AllSet())
		if err != nil {
			return false, xerrors.Errorf("acquiring write locks for sector %v fails with err: %v", sector, err)
		}

		// Acquire read locks
		_, err = tx.Exec(
			`UPDATE sector_location
							 	SET read_ts = NOW(), read_refs = read_refs + 1
							 	WHERE miner_id=$1
							 		AND sector_num=$2
							 		AND sector_filetype = ANY($3)`,
			sector.Miner,
			sector.Number,
			read.AllSet())
		if err != nil {
			return false, xerrors.Errorf("acquiring read locks for sector %v fails with err: %v", sector, err)
		}

		return true, nil
	})
	if err != nil {
		return false, err
	}

	return true, nil
}

func (dbi *DBIndex) unlock(sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType, lockUuid uuid.UUID) (bool, error) {
	ctx := context.Background()

	if read|write == 0 {
		return false, nil
	}

	if read|write > (1<<storiface.FileTypes)-1 {
		return false, xerrors.Errorf("unknown file types specified")
	}

	// Relinquish write locks
	_, err := dbi.harmonyDB.Exec(ctx,
		`UPDATE sector_location
				SET write_ts = NULL, write_lock_owner = NULL
				WHERE miner_id=$1
					AND sector_num=$2
					AND write_lock_owner=$3
					AND sector_filetype = ANY($4)`,
		sector.Miner,
		sector.Number,
		lockUuid.String(),
		write.AllSet())
	if err != nil {
		return false, xerrors.Errorf("relinquishing write locks for sector %v fails with err: %v", sector, err)
	}

	// Relinquish read locks
	_, err = dbi.harmonyDB.Exec(ctx,
		`UPDATE sector_location
				SET read_refs = read_refs-1,
					read_ts = CASE WHEN read_refs - 1 = 0 THEN NULL ELSE read_ts END
				WHERE miner_id=$1
					AND sector_num=$2
					AND sector_filetype = ANY($3)`,
		sector.Miner,
		sector.Number,
		read.AllSet())
	if err != nil {
		return false, xerrors.Errorf("relinquishing read locks for sector %v fails with err: %v", sector, err)
	}

	return true, nil

}

func (dbi *DBIndex) StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error {

	retries := 5
	maxWaitTime := 300 // Set max wait time to 5 minutes

	waitTime := 1
	// generate uuid for this lock owner
	lockUuid := uuid.New()

	// retry with exponential backoff and block until lock is acquired
	for {
		locked, err := dbi.lock(ctx, sector, read, write, lockUuid)
		// if err is not nil and is not because we cannot acquire lock, retry
		if err != nil && !errors.Is(err, errAlreadyLocked) {
			retries--
			if retries == 0 {
				return err
			}
		}

		if locked {
			break
		}

		select {
		case <-time.After(time.Duration(waitTime) * time.Second):
			if waitTime < maxWaitTime {
				waitTime *= 2
			}
		case <-ctx.Done():
			return ctx.Err()
		}
	}

	go func() {
		<-ctx.Done()
		_, err := dbi.unlock(sector, read, write, lockUuid)
		if err != nil {
			log.Errorf("unlocking sector %v for filetypes: read=%d, write=%d, fails with err: %v", sector, read, write, err)
		}

	}()

	return nil
}

func (dbi *DBIndex) StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) {
	lockUuid := uuid.New()
	locked, err := dbi.lock(ctx, sector, read, write, lockUuid)
	if err != nil {
		return false, err
	}

	if locked {
		go func() {
			<-ctx.Done()
			_, err := dbi.unlock(sector, read, write, lockUuid)
			if err != nil {
				log.Errorf("unlocking sector %v for filetypes: read=%d, write=%d, fails with err: %v", sector, read, write, err)
			}
		}()
	}

	return locked, nil
}

func (dbi *DBIndex) StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error) {

	var rows []struct {
		MinerId        uint64
		SectorNum      uint64
		SectorFileType int `db:"sector_filetype"`
		ReadTs         sql.NullTime
		ReadRefs       int
		WriteTs        sql.NullTime
	}

	err := dbi.harmonyDB.Select(ctx, &rows,
		"SELECT miner_id, sector_num, sector_filetype, read_ts, read_refs, write_ts FROM sector_location")
	if err != nil {
		return storiface.SectorLocks{}, err
	}

	type locks struct {
		sectorFileType storiface.SectorFileType
		readRefs       uint
		writeLk        bool
	}
	sectorLocks := make(map[abi.SectorID]locks)
	for _, row := range rows {
		sector := abi.SectorID{
			Miner:  abi.ActorID(row.MinerId),
			Number: abi.SectorNumber(row.SectorNum),
		}

		var readRefs uint
		if isLocked(row.ReadTs) {
			readRefs = uint(row.ReadRefs)
		}

		sectorLocks[sector] = locks{
			sectorFileType: storiface.SectorFileType(row.SectorFileType),
			readRefs:       readRefs,
			writeLk:        isLocked(row.WriteTs),
		}
	}

	var result storiface.SectorLocks
	for sector, locks := range sectorLocks {
		var lock storiface.SectorLock
		lock.Sector = sector
		lock.Read[locks.sectorFileType] = locks.readRefs
		if locks.writeLk {
			lock.Write[locks.sectorFileType] = 1
		} else {
			lock.Write[locks.sectorFileType] = 0
		}
		result.Locks = append(result.Locks, lock)
	}

	return result, nil
}

var _ SectorIndex = &DBIndex{}