Merge branch 'feat/testnet2' into feat/event-states
This commit is contained in:
Łukasz Magiera
+1
-1
@@ -5,7 +5,7 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
logging "github.com/ipfs/go-log"
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/api/apistruct"
|
||||
|
||||
+1
-2
@@ -5,10 +5,9 @@ import (
|
||||
"reflect"
|
||||
"sync"
|
||||
|
||||
"github.com/filecoin-project/go-statestore"
|
||||
"github.com/ipfs/go-datastore"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/lotus/lib/statestore"
|
||||
)
|
||||
|
||||
type StateHandler interface {
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
logging "github.com/ipfs/go-log"
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
"go.opencensus.io/trace"
|
||||
"go.opencensus.io/trace/propagation"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
package lotuslog
|
||||
|
||||
import logging "github.com/ipfs/go-log/v2"
|
||||
|
||||
func SetupLogLevels() {
|
||||
logging.SetLogLevel("*", "INFO")
|
||||
logging.SetLogLevel("dht", "ERROR")
|
||||
logging.SetLogLevel("swarm2", "WARN")
|
||||
logging.SetLogLevel("bitswap", "WARN")
|
||||
logging.SetLogLevel("pubsub", "WARN")
|
||||
logging.SetLogLevel("connmgr", "WARN")
|
||||
}
|
||||
@@ -1,126 +0,0 @@
|
||||
package sectorbuilder
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
func (sb *SectorBuilder) SectorName(sectorID uint64) string {
|
||||
return fmt.Sprintf("s-%s-%d", sb.Miner, sectorID)
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) StagedSectorPath(sectorID uint64) string {
|
||||
return filepath.Join(sb.filesystem.pathFor(dataStaging), sb.SectorName(sectorID))
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) unsealedSectorPath(sectorID uint64) string {
|
||||
return filepath.Join(sb.filesystem.pathFor(dataUnsealed), sb.SectorName(sectorID))
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) stagedSectorFile(sectorID uint64) (*os.File, error) {
|
||||
return os.OpenFile(sb.StagedSectorPath(sectorID), os.O_RDWR|os.O_CREATE, 0644)
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) SealedSectorPath(sectorID uint64) (string, error) {
|
||||
path := filepath.Join(sb.filesystem.pathFor(dataSealed), sb.SectorName(sectorID))
|
||||
|
||||
return path, nil
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) sectorCacheDir(sectorID uint64) (string, error) {
|
||||
dir := filepath.Join(sb.filesystem.pathFor(dataCache), sb.SectorName(sectorID))
|
||||
|
||||
err := os.Mkdir(dir, 0755)
|
||||
if os.IsExist(err) {
|
||||
err = nil
|
||||
}
|
||||
|
||||
return dir, err
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) GetPath(typ string, sectorName string) (string, error) {
|
||||
_, found := overheadMul[dataType(typ)]
|
||||
if !found {
|
||||
return "", xerrors.Errorf("unknown sector type: %s", typ)
|
||||
}
|
||||
|
||||
return filepath.Join(sb.filesystem.pathFor(dataType(typ)), sectorName), nil
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) TrimCache(sectorID uint64) error {
|
||||
dir, err := sb.sectorCacheDir(sectorID)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("getting cache dir: %w", err)
|
||||
}
|
||||
|
||||
files, err := ioutil.ReadDir(dir)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("readdir: %w", err)
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
if !strings.HasSuffix(file.Name(), ".dat") { // _aux probably
|
||||
continue
|
||||
}
|
||||
if strings.HasSuffix(file.Name(), "-data-tree-r-last.dat") { // Want to keep
|
||||
continue
|
||||
}
|
||||
|
||||
if err := os.Remove(filepath.Join(dir, file.Name())); err != nil {
|
||||
return xerrors.Errorf("rm %s: %w", file.Name(), err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func toReadableFile(r io.Reader, n int64) (*os.File, func() error, error) {
|
||||
f, ok := r.(*os.File)
|
||||
if ok {
|
||||
return f, func() error { return nil }, nil
|
||||
}
|
||||
|
||||
var w *os.File
|
||||
|
||||
f, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
var wait sync.Mutex
|
||||
var werr error
|
||||
|
||||
wait.Lock()
|
||||
go func() {
|
||||
defer wait.Unlock()
|
||||
|
||||
var copied int64
|
||||
copied, werr = io.CopyN(w, r, n)
|
||||
if werr != nil {
|
||||
log.Warnf("toReadableFile: copy error: %+v", werr)
|
||||
}
|
||||
|
||||
err := w.Close()
|
||||
if werr == nil && err != nil {
|
||||
werr = err
|
||||
log.Warnf("toReadableFile: close error: %+v", err)
|
||||
return
|
||||
}
|
||||
if copied != n {
|
||||
log.Warnf("copied different amount than expected: %d != %d", copied, n)
|
||||
werr = xerrors.Errorf("copied different amount than expected: %d != %d", copied, n)
|
||||
}
|
||||
}()
|
||||
|
||||
return f, func() error {
|
||||
wait.Lock()
|
||||
return werr
|
||||
}, nil
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package sectorbuilder
|
||||
|
||||
import (
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"golang.org/x/xerrors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
type dataType string
|
||||
|
||||
const (
|
||||
dataCache dataType = "cache"
|
||||
dataStaging dataType = "staging"
|
||||
dataSealed dataType = "sealed"
|
||||
dataUnsealed dataType = "unsealed"
|
||||
)
|
||||
|
||||
var overheadMul = map[dataType]uint64{ // * sectorSize
|
||||
dataCache: 11, // TODO: check if true for 32G sectors
|
||||
dataStaging: 1,
|
||||
dataSealed: 1,
|
||||
dataUnsealed: 1,
|
||||
}
|
||||
|
||||
type fs struct {
|
||||
path string
|
||||
|
||||
// in progress actions
|
||||
|
||||
reserved map[dataType]uint64
|
||||
|
||||
lk sync.Mutex
|
||||
}
|
||||
|
||||
func openFs(dir string) *fs {
|
||||
return &fs{
|
||||
path: dir,
|
||||
reserved: map[dataType]uint64{},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fs) init() error {
|
||||
for _, dir := range []string{f.path,
|
||||
f.pathFor(dataCache),
|
||||
f.pathFor(dataStaging),
|
||||
f.pathFor(dataSealed),
|
||||
f.pathFor(dataUnsealed)} {
|
||||
if err := os.Mkdir(dir, 0755); err != nil {
|
||||
if os.IsExist(err) {
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fs) pathFor(typ dataType) string {
|
||||
_, found := overheadMul[typ]
|
||||
if !found {
|
||||
panic("unknown data path requested")
|
||||
}
|
||||
|
||||
return filepath.Join(f.path, string(typ))
|
||||
}
|
||||
|
||||
func (f *fs) reservedBytes() int64 {
|
||||
var out int64
|
||||
for _, r := range f.reserved {
|
||||
out += int64(r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (f *fs) reserve(typ dataType, size uint64) error {
|
||||
f.lk.Lock()
|
||||
defer f.lk.Unlock()
|
||||
|
||||
var fsstat syscall.Statfs_t
|
||||
|
||||
if err := syscall.Statfs(f.pathFor(typ), &fsstat); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fsavail := int64(fsstat.Bavail) * int64(fsstat.Bsize)
|
||||
|
||||
avail := fsavail - f.reservedBytes()
|
||||
|
||||
need := overheadMul[typ] * size
|
||||
|
||||
if int64(need) > avail {
|
||||
return xerrors.Errorf("not enough space in '%s', need %s, available %s (fs: %s, reserved: %s)",
|
||||
f.path,
|
||||
types.NewInt(need).SizeStr(),
|
||||
types.NewInt(uint64(avail)).SizeStr(),
|
||||
types.NewInt(uint64(fsavail)).SizeStr(),
|
||||
types.NewInt(uint64(f.reservedBytes())).SizeStr())
|
||||
}
|
||||
|
||||
f.reserved[typ] += need
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fs) free(typ dataType, sectorSize uint64) {
|
||||
f.lk.Lock()
|
||||
defer f.lk.Unlock()
|
||||
|
||||
f.reserved[typ] -= overheadMul[typ] * sectorSize
|
||||
|
||||
return
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
package sectorbuilder
|
||||
|
||||
import (
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||
)
|
||||
|
||||
func TempSectorbuilderDir(dir string, sectorSize uint64, ds dtypes.MetadataDS) (*SectorBuilder, error) {
|
||||
addr, err := address.NewFromString("t3vfxagwiegrywptkbmyohqqbfzd7xzbryjydmxso4hfhgsnv6apddyihltsbiikjf3lm7x2myiaxhuc77capq")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sb, err := New(&Config{
|
||||
SectorSize: sectorSize,
|
||||
|
||||
Dir: dir,
|
||||
|
||||
WorkerThreads: 2,
|
||||
Miner: addr,
|
||||
}, ds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return sb, nil
|
||||
}
|
||||
@@ -1,174 +0,0 @@
|
||||
package sectorbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
type WorkerTaskType int
|
||||
|
||||
const (
|
||||
WorkerPreCommit WorkerTaskType = iota
|
||||
WorkerCommit
|
||||
)
|
||||
|
||||
type WorkerTask struct {
|
||||
Type WorkerTaskType
|
||||
TaskID uint64
|
||||
|
||||
SectorID uint64
|
||||
|
||||
// preCommit
|
||||
SealTicket SealTicket
|
||||
Pieces []PublicPieceInfo
|
||||
|
||||
// commit
|
||||
SealSeed SealSeed
|
||||
Rspco RawSealPreCommitOutput
|
||||
}
|
||||
|
||||
type workerCall struct {
|
||||
task WorkerTask
|
||||
ret chan SealRes
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) AddWorker(ctx context.Context, cfg WorkerCfg) (<-chan WorkerTask, error) {
|
||||
sb.remoteLk.Lock()
|
||||
defer sb.remoteLk.Unlock()
|
||||
|
||||
taskCh := make(chan WorkerTask)
|
||||
r := &remote{
|
||||
sealTasks: taskCh,
|
||||
busy: 0,
|
||||
}
|
||||
|
||||
sb.remoteCtr++
|
||||
sb.remotes[sb.remoteCtr] = r
|
||||
|
||||
go sb.remoteWorker(ctx, r, cfg)
|
||||
|
||||
return taskCh, nil
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) returnTask(task workerCall) {
|
||||
var ret chan workerCall
|
||||
switch task.task.Type {
|
||||
case WorkerPreCommit:
|
||||
ret = sb.precommitTasks
|
||||
case WorkerCommit:
|
||||
ret = sb.commitTasks
|
||||
default:
|
||||
log.Error("unknown task type", task.task.Type)
|
||||
}
|
||||
|
||||
go func() {
|
||||
select {
|
||||
case ret <- task:
|
||||
case <-sb.stopping:
|
||||
return
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) remoteWorker(ctx context.Context, r *remote, cfg WorkerCfg) {
|
||||
defer log.Warn("Remote worker disconnected")
|
||||
|
||||
defer func() {
|
||||
sb.remoteLk.Lock()
|
||||
defer sb.remoteLk.Unlock()
|
||||
|
||||
for i, vr := range sb.remotes {
|
||||
if vr == r {
|
||||
delete(sb.remotes, i)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
precommits := sb.precommitTasks
|
||||
if cfg.NoPreCommit {
|
||||
precommits = nil
|
||||
}
|
||||
commits := sb.commitTasks
|
||||
if cfg.NoCommit {
|
||||
commits = nil
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case task := <-commits:
|
||||
sb.doTask(ctx, r, task)
|
||||
case task := <-precommits:
|
||||
sb.doTask(ctx, r, task)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-sb.stopping:
|
||||
return
|
||||
}
|
||||
|
||||
r.lk.Lock()
|
||||
r.busy = 0
|
||||
r.lk.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) doTask(ctx context.Context, r *remote, task workerCall) {
|
||||
resCh := make(chan SealRes)
|
||||
|
||||
sb.remoteLk.Lock()
|
||||
sb.remoteResults[task.task.TaskID] = resCh
|
||||
sb.remoteLk.Unlock()
|
||||
|
||||
// send the task
|
||||
select {
|
||||
case r.sealTasks <- task.task:
|
||||
case <-ctx.Done():
|
||||
sb.returnTask(task)
|
||||
return
|
||||
}
|
||||
|
||||
r.lk.Lock()
|
||||
r.busy = task.task.TaskID
|
||||
r.lk.Unlock()
|
||||
|
||||
// wait for the result
|
||||
select {
|
||||
case res := <-resCh:
|
||||
|
||||
// send the result back to the caller
|
||||
select {
|
||||
case task.ret <- res:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-sb.stopping:
|
||||
return
|
||||
}
|
||||
|
||||
case <-ctx.Done():
|
||||
log.Warnf("context expired while waiting for task %d (sector %d): %s", task.task.TaskID, task.task.SectorID, ctx.Err())
|
||||
return
|
||||
case <-sb.stopping:
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) TaskDone(ctx context.Context, task uint64, res SealRes) error {
|
||||
sb.remoteLk.Lock()
|
||||
rres, ok := sb.remoteResults[task]
|
||||
if ok {
|
||||
delete(sb.remoteResults, task)
|
||||
}
|
||||
sb.remoteLk.Unlock()
|
||||
|
||||
if !ok {
|
||||
return xerrors.Errorf("task %d not found", task)
|
||||
}
|
||||
|
||||
select {
|
||||
case rres <- res:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
package sectorbuilder
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
sectorbuilder "github.com/filecoin-project/filecoin-ffi"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
type Fault struct {
|
||||
SectorID uint64
|
||||
|
||||
Err error
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) Scrub(sectorSet sectorbuilder.SortedPublicSectorInfo) []*Fault {
|
||||
var faults []*Fault
|
||||
|
||||
for _, sector := range sectorSet.Values() {
|
||||
err := sb.checkSector(sector.SectorID)
|
||||
if err != nil {
|
||||
faults = append(faults, &Fault{SectorID: sector.SectorID, Err: err})
|
||||
}
|
||||
}
|
||||
|
||||
return faults
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) checkSector(sectorID uint64) error {
|
||||
cache, err := sb.sectorCacheDir(sectorID)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("getting sector cache dir: %w", err)
|
||||
}
|
||||
|
||||
if err := assertFile(filepath.Join(cache, "p_aux"), 96, 96); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := assertFile(filepath.Join(cache, "sc-01-data-tree-r-last.dat"), (2*sb.ssize)-32, (2*sb.ssize)-32); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// TODO: better validate this
|
||||
if err := assertFile(filepath.Join(cache, "t_aux"), 100, 32000); err != nil { // TODO: what should this actually be?
|
||||
return err
|
||||
}
|
||||
|
||||
dent, err := ioutil.ReadDir(cache)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("reading cache dir %s", cache)
|
||||
}
|
||||
if len(dent) != 3 {
|
||||
return xerrors.Errorf("found %d files in %s, expected 3", len(dent), cache)
|
||||
}
|
||||
|
||||
sealed, err := sb.SealedSectorPath(sectorID)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("getting sealed sector path: %w", err)
|
||||
}
|
||||
|
||||
if err := assertFile(filepath.Join(sealed), sb.ssize, sb.ssize); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func assertFile(path string, minSz uint64, maxSz uint64) error {
|
||||
st, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("stat %s: %w", path, err)
|
||||
}
|
||||
|
||||
if st.IsDir() {
|
||||
return xerrors.Errorf("expected %s to be a regular file", path)
|
||||
}
|
||||
|
||||
if uint64(st.Size()) < minSz || uint64(st.Size()) > maxSz {
|
||||
return xerrors.Errorf("%s wasn't within size bounds, expected %d < f < %d, got %d", minSz, maxSz, st.Size())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,799 +0,0 @@
|
||||
package sectorbuilder
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
sectorbuilder "github.com/filecoin-project/filecoin-ffi"
|
||||
"github.com/ipfs/go-datastore"
|
||||
logging "github.com/ipfs/go-log"
|
||||
dcopy "github.com/otiai10/copy"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||
)
|
||||
|
||||
const PoStReservedWorkers = 1
|
||||
const PoRepProofPartitions = 10
|
||||
|
||||
var lastSectorIdKey = datastore.NewKey("/sectorbuilder/last")
|
||||
|
||||
var log = logging.Logger("sectorbuilder")
|
||||
|
||||
type SortedPublicSectorInfo = sectorbuilder.SortedPublicSectorInfo
|
||||
type SortedPrivateSectorInfo = sectorbuilder.SortedPrivateSectorInfo
|
||||
|
||||
type SealTicket = sectorbuilder.SealTicket
|
||||
|
||||
type SealSeed = sectorbuilder.SealSeed
|
||||
|
||||
type SealPreCommitOutput = sectorbuilder.SealPreCommitOutput
|
||||
|
||||
type SealCommitOutput = sectorbuilder.SealCommitOutput
|
||||
|
||||
type PublicPieceInfo = sectorbuilder.PublicPieceInfo
|
||||
|
||||
type RawSealPreCommitOutput sectorbuilder.RawSealPreCommitOutput
|
||||
|
||||
type EPostCandidate = sectorbuilder.Candidate
|
||||
|
||||
const CommLen = sectorbuilder.CommitmentBytesLen
|
||||
|
||||
type WorkerCfg struct {
|
||||
NoPreCommit bool
|
||||
NoCommit bool
|
||||
|
||||
// TODO: 'cost' info, probably in terms of sealing + transfer speed
|
||||
}
|
||||
|
||||
type SectorBuilder struct {
|
||||
ds dtypes.MetadataDS
|
||||
idLk sync.Mutex
|
||||
|
||||
ssize uint64
|
||||
lastID uint64
|
||||
|
||||
Miner address.Address
|
||||
|
||||
unsealLk sync.Mutex
|
||||
|
||||
noCommit bool
|
||||
noPreCommit bool
|
||||
rateLimit chan struct{}
|
||||
|
||||
precommitTasks chan workerCall
|
||||
commitTasks chan workerCall
|
||||
|
||||
taskCtr uint64
|
||||
remoteLk sync.Mutex
|
||||
remoteCtr int
|
||||
remotes map[int]*remote
|
||||
remoteResults map[uint64]chan<- SealRes
|
||||
|
||||
addPieceWait int32
|
||||
preCommitWait int32
|
||||
commitWait int32
|
||||
unsealWait int32
|
||||
|
||||
fsLk sync.Mutex
|
||||
filesystem *fs // TODO: multi-fs support
|
||||
|
||||
stopping chan struct{}
|
||||
}
|
||||
|
||||
type JsonRSPCO struct {
|
||||
CommD []byte
|
||||
CommR []byte
|
||||
}
|
||||
|
||||
func (rspco *RawSealPreCommitOutput) ToJson() JsonRSPCO {
|
||||
return JsonRSPCO{
|
||||
CommD: rspco.CommD[:],
|
||||
CommR: rspco.CommR[:],
|
||||
}
|
||||
}
|
||||
|
||||
func (rspco *JsonRSPCO) rspco() RawSealPreCommitOutput {
|
||||
var out RawSealPreCommitOutput
|
||||
copy(out.CommD[:], rspco.CommD)
|
||||
copy(out.CommR[:], rspco.CommR)
|
||||
return out
|
||||
}
|
||||
|
||||
type SealRes struct {
|
||||
Err string
|
||||
GoErr error `json:"-"`
|
||||
|
||||
Proof []byte
|
||||
Rspco JsonRSPCO
|
||||
}
|
||||
|
||||
type remote struct {
|
||||
lk sync.Mutex
|
||||
|
||||
sealTasks chan<- WorkerTask
|
||||
busy uint64 // only for metrics
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
SectorSize uint64
|
||||
Miner address.Address
|
||||
|
||||
WorkerThreads uint8
|
||||
FallbackLastID uint64
|
||||
NoCommit bool
|
||||
NoPreCommit bool
|
||||
|
||||
Dir string
|
||||
_ struct{} // guard against nameless init
|
||||
}
|
||||
|
||||
func New(cfg *Config, ds dtypes.MetadataDS) (*SectorBuilder, error) {
|
||||
if cfg.WorkerThreads < PoStReservedWorkers {
|
||||
return nil, xerrors.Errorf("minimum worker threads is %d, specified %d", PoStReservedWorkers, cfg.WorkerThreads)
|
||||
}
|
||||
|
||||
var lastUsedID uint64
|
||||
b, err := ds.Get(lastSectorIdKey)
|
||||
switch err {
|
||||
case nil:
|
||||
i, err := strconv.ParseInt(string(b), 10, 64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lastUsedID = uint64(i)
|
||||
case datastore.ErrNotFound:
|
||||
lastUsedID = cfg.FallbackLastID
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rlimit := cfg.WorkerThreads - PoStReservedWorkers
|
||||
|
||||
sealLocal := rlimit > 0
|
||||
|
||||
if rlimit == 0 {
|
||||
rlimit = 1
|
||||
}
|
||||
|
||||
sb := &SectorBuilder{
|
||||
ds: ds,
|
||||
|
||||
ssize: cfg.SectorSize,
|
||||
lastID: lastUsedID,
|
||||
|
||||
filesystem: openFs(cfg.Dir),
|
||||
|
||||
Miner: cfg.Miner,
|
||||
|
||||
noPreCommit: cfg.NoPreCommit || !sealLocal,
|
||||
noCommit: cfg.NoCommit || !sealLocal,
|
||||
rateLimit: make(chan struct{}, rlimit),
|
||||
|
||||
taskCtr: 1,
|
||||
precommitTasks: make(chan workerCall),
|
||||
commitTasks: make(chan workerCall),
|
||||
remoteResults: map[uint64]chan<- SealRes{},
|
||||
remotes: map[int]*remote{},
|
||||
|
||||
stopping: make(chan struct{}),
|
||||
}
|
||||
|
||||
if err := sb.filesystem.init(); err != nil {
|
||||
return nil, xerrors.Errorf("initializing sectorbuilder filesystem: %w", err)
|
||||
}
|
||||
|
||||
return sb, nil
|
||||
}
|
||||
|
||||
func NewStandalone(cfg *Config) (*SectorBuilder, error) {
|
||||
sb := &SectorBuilder{
|
||||
ds: nil,
|
||||
|
||||
ssize: cfg.SectorSize,
|
||||
|
||||
Miner: cfg.Miner,
|
||||
filesystem: openFs(cfg.Dir),
|
||||
|
||||
taskCtr: 1,
|
||||
remotes: map[int]*remote{},
|
||||
rateLimit: make(chan struct{}, cfg.WorkerThreads),
|
||||
stopping: make(chan struct{}),
|
||||
}
|
||||
|
||||
if err := sb.filesystem.init(); err != nil {
|
||||
return nil, xerrors.Errorf("initializing sectorbuilder filesystem: %w", err)
|
||||
}
|
||||
|
||||
return sb, nil
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) checkRateLimit() {
|
||||
if cap(sb.rateLimit) == len(sb.rateLimit) {
|
||||
log.Warn("rate-limiting local sectorbuilder call")
|
||||
}
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) RateLimit() func() {
|
||||
sb.checkRateLimit()
|
||||
|
||||
sb.rateLimit <- struct{}{}
|
||||
|
||||
return func() {
|
||||
<-sb.rateLimit
|
||||
}
|
||||
}
|
||||
|
||||
type WorkerStats struct {
|
||||
LocalFree int
|
||||
LocalReserved int
|
||||
LocalTotal int
|
||||
// todo: post in progress
|
||||
RemotesTotal int
|
||||
RemotesFree int
|
||||
|
||||
AddPieceWait int
|
||||
PreCommitWait int
|
||||
CommitWait int
|
||||
UnsealWait int
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) WorkerStats() WorkerStats {
|
||||
sb.remoteLk.Lock()
|
||||
defer sb.remoteLk.Unlock()
|
||||
|
||||
remoteFree := len(sb.remotes)
|
||||
for _, r := range sb.remotes {
|
||||
if r.busy > 0 {
|
||||
remoteFree--
|
||||
}
|
||||
}
|
||||
|
||||
return WorkerStats{
|
||||
LocalFree: cap(sb.rateLimit) - len(sb.rateLimit),
|
||||
LocalReserved: PoStReservedWorkers,
|
||||
LocalTotal: cap(sb.rateLimit) + PoStReservedWorkers,
|
||||
RemotesTotal: len(sb.remotes),
|
||||
RemotesFree: remoteFree,
|
||||
|
||||
AddPieceWait: int(atomic.LoadInt32(&sb.addPieceWait)),
|
||||
PreCommitWait: int(atomic.LoadInt32(&sb.preCommitWait)),
|
||||
CommitWait: int(atomic.LoadInt32(&sb.commitWait)),
|
||||
UnsealWait: int(atomic.LoadInt32(&sb.unsealWait)),
|
||||
}
|
||||
}
|
||||
|
||||
func addressToProverID(a address.Address) [32]byte {
|
||||
var proverId [32]byte
|
||||
copy(proverId[:], a.Payload())
|
||||
return proverId
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) AcquireSectorId() (uint64, error) {
|
||||
sb.idLk.Lock()
|
||||
defer sb.idLk.Unlock()
|
||||
|
||||
sb.lastID++
|
||||
id := sb.lastID
|
||||
|
||||
err := sb.ds.Put(lastSectorIdKey, []byte(fmt.Sprint(id)))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) AddPiece(pieceSize uint64, sectorId uint64, file io.Reader, existingPieceSizes []uint64) (PublicPieceInfo, error) {
|
||||
fs := sb.filesystem
|
||||
|
||||
if err := fs.reserve(dataStaging, sb.ssize); err != nil {
|
||||
return PublicPieceInfo{}, err
|
||||
}
|
||||
defer fs.free(dataStaging, sb.ssize)
|
||||
|
||||
atomic.AddInt32(&sb.addPieceWait, 1)
|
||||
ret := sb.RateLimit()
|
||||
atomic.AddInt32(&sb.addPieceWait, -1)
|
||||
defer ret()
|
||||
|
||||
f, werr, err := toReadableFile(file, int64(pieceSize))
|
||||
if err != nil {
|
||||
return PublicPieceInfo{}, err
|
||||
}
|
||||
|
||||
stagedFile, err := sb.stagedSectorFile(sectorId)
|
||||
if err != nil {
|
||||
return PublicPieceInfo{}, err
|
||||
}
|
||||
|
||||
_, _, commP, err := sectorbuilder.WriteWithAlignment(f, pieceSize, stagedFile, existingPieceSizes)
|
||||
if err != nil {
|
||||
return PublicPieceInfo{}, err
|
||||
}
|
||||
|
||||
if err := stagedFile.Close(); err != nil {
|
||||
return PublicPieceInfo{}, err
|
||||
}
|
||||
|
||||
if err := f.Close(); err != nil {
|
||||
return PublicPieceInfo{}, err
|
||||
}
|
||||
|
||||
return PublicPieceInfo{
|
||||
Size: pieceSize,
|
||||
CommP: commP,
|
||||
}, werr()
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) ReadPieceFromSealedSector(sectorID uint64, offset uint64, size uint64, ticket []byte, commD []byte) (io.ReadCloser, error) {
|
||||
fs := sb.filesystem
|
||||
|
||||
if err := fs.reserve(dataUnsealed, sb.ssize); err != nil { // TODO: this needs to get smarter when we start supporting partial unseals
|
||||
return nil, err
|
||||
}
|
||||
defer fs.free(dataUnsealed, sb.ssize)
|
||||
|
||||
atomic.AddInt32(&sb.unsealWait, 1)
|
||||
// TODO: Don't wait if cached
|
||||
ret := sb.RateLimit() // TODO: check perf, consider remote unseal worker
|
||||
defer ret()
|
||||
atomic.AddInt32(&sb.unsealWait, -1)
|
||||
|
||||
sb.unsealLk.Lock() // TODO: allow unsealing unrelated sectors in parallel
|
||||
defer sb.unsealLk.Unlock()
|
||||
|
||||
cacheDir, err := sb.sectorCacheDir(sectorID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sealedPath, err := sb.SealedSectorPath(sectorID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
unsealedPath := sb.unsealedSectorPath(sectorID)
|
||||
|
||||
// TODO: GC for those
|
||||
// (Probably configurable count of sectors to be kept unsealed, and just
|
||||
// remove last used one (or use whatever other cache policy makes sense))
|
||||
f, err := os.OpenFile(unsealedPath, os.O_RDONLY, 0644)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var commd [CommLen]byte
|
||||
copy(commd[:], commD)
|
||||
|
||||
var tkt [CommLen]byte
|
||||
copy(tkt[:], ticket)
|
||||
|
||||
err = sectorbuilder.Unseal(sb.ssize,
|
||||
PoRepProofPartitions,
|
||||
cacheDir,
|
||||
sealedPath,
|
||||
unsealedPath,
|
||||
sectorID,
|
||||
addressToProverID(sb.Miner),
|
||||
tkt,
|
||||
commd)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("unseal failed: %w", err)
|
||||
}
|
||||
|
||||
f, err = os.OpenFile(unsealedPath, os.O_RDONLY, 0644)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := f.Seek(int64(offset), io.SeekStart); err != nil {
|
||||
return nil, xerrors.Errorf("seek: %w", err)
|
||||
}
|
||||
|
||||
lr := io.LimitReader(f, int64(size))
|
||||
|
||||
return &struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
}{
|
||||
Reader: lr,
|
||||
Closer: f,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) sealPreCommitRemote(call workerCall) (RawSealPreCommitOutput, error) {
|
||||
atomic.AddInt32(&sb.preCommitWait, -1)
|
||||
|
||||
select {
|
||||
case ret := <-call.ret:
|
||||
var err error
|
||||
if ret.Err != "" {
|
||||
err = xerrors.New(ret.Err)
|
||||
}
|
||||
return ret.Rspco.rspco(), err
|
||||
case <-sb.stopping:
|
||||
return RawSealPreCommitOutput{}, xerrors.New("sectorbuilder stopped")
|
||||
}
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) SealPreCommit(sectorID uint64, ticket SealTicket, pieces []PublicPieceInfo) (RawSealPreCommitOutput, error) {
|
||||
fs := sb.filesystem
|
||||
|
||||
if err := fs.reserve(dataCache, sb.ssize); err != nil {
|
||||
return RawSealPreCommitOutput{}, err
|
||||
}
|
||||
defer fs.free(dataCache, sb.ssize)
|
||||
|
||||
if err := fs.reserve(dataSealed, sb.ssize); err != nil {
|
||||
return RawSealPreCommitOutput{}, err
|
||||
}
|
||||
defer fs.free(dataSealed, sb.ssize)
|
||||
|
||||
call := workerCall{
|
||||
task: WorkerTask{
|
||||
Type: WorkerPreCommit,
|
||||
TaskID: atomic.AddUint64(&sb.taskCtr, 1),
|
||||
SectorID: sectorID,
|
||||
SealTicket: ticket,
|
||||
Pieces: pieces,
|
||||
},
|
||||
ret: make(chan SealRes),
|
||||
}
|
||||
|
||||
atomic.AddInt32(&sb.preCommitWait, 1)
|
||||
|
||||
select { // prefer remote
|
||||
case sb.precommitTasks <- call:
|
||||
return sb.sealPreCommitRemote(call)
|
||||
default:
|
||||
}
|
||||
|
||||
sb.checkRateLimit()
|
||||
|
||||
rl := sb.rateLimit
|
||||
if sb.noPreCommit {
|
||||
rl = make(chan struct{})
|
||||
}
|
||||
|
||||
select { // use whichever is available
|
||||
case sb.precommitTasks <- call:
|
||||
return sb.sealPreCommitRemote(call)
|
||||
case rl <- struct{}{}:
|
||||
}
|
||||
|
||||
atomic.AddInt32(&sb.preCommitWait, -1)
|
||||
|
||||
// local
|
||||
|
||||
defer func() {
|
||||
<-sb.rateLimit
|
||||
}()
|
||||
|
||||
cacheDir, err := sb.sectorCacheDir(sectorID)
|
||||
if err != nil {
|
||||
return RawSealPreCommitOutput{}, xerrors.Errorf("getting cache dir: %w", err)
|
||||
}
|
||||
|
||||
sealedPath, err := sb.SealedSectorPath(sectorID)
|
||||
if err != nil {
|
||||
return RawSealPreCommitOutput{}, xerrors.Errorf("getting sealed sector path: %w", err)
|
||||
}
|
||||
|
||||
e, err := os.OpenFile(sealedPath, os.O_RDWR|os.O_CREATE, 0644)
|
||||
if err != nil {
|
||||
return RawSealPreCommitOutput{}, xerrors.Errorf("ensuring sealed file exists: %w", err)
|
||||
}
|
||||
if err := e.Close(); err != nil {
|
||||
return RawSealPreCommitOutput{}, err
|
||||
}
|
||||
|
||||
var sum uint64
|
||||
for _, piece := range pieces {
|
||||
sum += piece.Size
|
||||
}
|
||||
ussize := UserBytesForSectorSize(sb.ssize)
|
||||
if sum != ussize {
|
||||
return RawSealPreCommitOutput{}, xerrors.Errorf("aggregated piece sizes don't match sector size: %d != %d (%d)", sum, ussize, int64(ussize-sum))
|
||||
}
|
||||
|
||||
stagedPath := sb.StagedSectorPath(sectorID)
|
||||
|
||||
rspco, err := sectorbuilder.SealPreCommit(
|
||||
sb.ssize,
|
||||
PoRepProofPartitions,
|
||||
cacheDir,
|
||||
stagedPath,
|
||||
sealedPath,
|
||||
sectorID,
|
||||
addressToProverID(sb.Miner),
|
||||
ticket.TicketBytes,
|
||||
pieces,
|
||||
)
|
||||
if err != nil {
|
||||
return RawSealPreCommitOutput{}, xerrors.Errorf("presealing sector %d (%s): %w", sectorID, stagedPath, err)
|
||||
}
|
||||
|
||||
return RawSealPreCommitOutput(rspco), nil
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) sealCommitRemote(call workerCall) (proof []byte, err error) {
|
||||
atomic.AddInt32(&sb.commitWait, -1)
|
||||
|
||||
select {
|
||||
case ret := <-call.ret:
|
||||
if ret.Err != "" {
|
||||
err = xerrors.New(ret.Err)
|
||||
}
|
||||
return ret.Proof, err
|
||||
case <-sb.stopping:
|
||||
return nil, xerrors.New("sectorbuilder stopped")
|
||||
}
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) sealCommitLocal(sectorID uint64, ticket SealTicket, seed SealSeed, pieces []PublicPieceInfo, rspco RawSealPreCommitOutput) (proof []byte, err error) {
|
||||
atomic.AddInt32(&sb.commitWait, -1)
|
||||
|
||||
defer func() {
|
||||
<-sb.rateLimit
|
||||
}()
|
||||
|
||||
cacheDir, err := sb.sectorCacheDir(sectorID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
proof, err = sectorbuilder.SealCommit(
|
||||
sb.ssize,
|
||||
PoRepProofPartitions,
|
||||
cacheDir,
|
||||
sectorID,
|
||||
addressToProverID(sb.Miner),
|
||||
ticket.TicketBytes,
|
||||
seed.TicketBytes,
|
||||
pieces,
|
||||
sectorbuilder.RawSealPreCommitOutput(rspco),
|
||||
)
|
||||
if err != nil {
|
||||
log.Warn("StandaloneSealCommit error: ", err)
|
||||
log.Warnf("sid:%d tkt:%v seed:%v, ppi:%v rspco:%v", sectorID, ticket, seed, pieces, rspco)
|
||||
|
||||
return nil, xerrors.Errorf("StandaloneSealCommit: %w", err)
|
||||
}
|
||||
|
||||
return proof, nil
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) SealCommit(sectorID uint64, ticket SealTicket, seed SealSeed, pieces []PublicPieceInfo, rspco RawSealPreCommitOutput) (proof []byte, err error) {
|
||||
call := workerCall{
|
||||
task: WorkerTask{
|
||||
Type: WorkerCommit,
|
||||
TaskID: atomic.AddUint64(&sb.taskCtr, 1),
|
||||
SectorID: sectorID,
|
||||
SealTicket: ticket,
|
||||
Pieces: pieces,
|
||||
|
||||
SealSeed: seed,
|
||||
Rspco: rspco,
|
||||
},
|
||||
ret: make(chan SealRes),
|
||||
}
|
||||
|
||||
atomic.AddInt32(&sb.commitWait, 1)
|
||||
|
||||
select { // prefer remote
|
||||
case sb.commitTasks <- call:
|
||||
proof, err = sb.sealCommitRemote(call)
|
||||
default:
|
||||
sb.checkRateLimit()
|
||||
|
||||
rl := sb.rateLimit
|
||||
if sb.noCommit {
|
||||
rl = make(chan struct{})
|
||||
}
|
||||
|
||||
select { // use whichever is available
|
||||
case sb.commitTasks <- call:
|
||||
proof, err = sb.sealCommitRemote(call)
|
||||
case rl <- struct{}{}:
|
||||
proof, err = sb.sealCommitLocal(sectorID, ticket, seed, pieces, rspco)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("commit: %w", err)
|
||||
}
|
||||
|
||||
return proof, nil
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) ComputeElectionPoSt(sectorInfo SortedPublicSectorInfo, challengeSeed []byte, winners []EPostCandidate) ([]byte, error) {
|
||||
if len(challengeSeed) != CommLen {
|
||||
return nil, xerrors.Errorf("given challenge seed was the wrong length: %d != %d", len(challengeSeed), CommLen)
|
||||
}
|
||||
var cseed [CommLen]byte
|
||||
copy(cseed[:], challengeSeed)
|
||||
|
||||
privsects, err := sb.pubSectorToPriv(sectorInfo, nil) // TODO: faults
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
proverID := addressToProverID(sb.Miner)
|
||||
|
||||
return sectorbuilder.GeneratePoSt(sb.ssize, proverID, privsects, cseed, winners)
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) GenerateEPostCandidates(sectorInfo SortedPublicSectorInfo, challengeSeed [CommLen]byte, faults []uint64) ([]EPostCandidate, error) {
|
||||
privsectors, err := sb.pubSectorToPriv(sectorInfo, faults)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values())), len(faults))
|
||||
|
||||
proverID := addressToProverID(sb.Miner)
|
||||
return sectorbuilder.GenerateCandidates(sb.ssize, proverID, challengeSeed, challengeCount, privsectors)
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) pubSectorToPriv(sectorInfo SortedPublicSectorInfo, faults []uint64) (SortedPrivateSectorInfo, error) {
|
||||
fmap := map[uint64]struct{}{}
|
||||
for _, fault := range faults {
|
||||
fmap[fault] = struct{}{}
|
||||
}
|
||||
|
||||
var out []sectorbuilder.PrivateSectorInfo
|
||||
for _, s := range sectorInfo.Values() {
|
||||
if _, faulty := fmap[s.SectorID]; faulty {
|
||||
continue
|
||||
}
|
||||
|
||||
cachePath, err := sb.sectorCacheDir(s.SectorID)
|
||||
if err != nil {
|
||||
return SortedPrivateSectorInfo{}, xerrors.Errorf("getting cache path for sector %d: %w", s.SectorID, err)
|
||||
}
|
||||
|
||||
sealedPath, err := sb.SealedSectorPath(s.SectorID)
|
||||
if err != nil {
|
||||
return SortedPrivateSectorInfo{}, xerrors.Errorf("getting sealed path for sector %d: %w", s.SectorID, err)
|
||||
}
|
||||
|
||||
out = append(out, sectorbuilder.PrivateSectorInfo{
|
||||
SectorID: s.SectorID,
|
||||
CommR: s.CommR,
|
||||
CacheDirPath: cachePath,
|
||||
SealedSectorPath: sealedPath,
|
||||
})
|
||||
}
|
||||
return NewSortedPrivateSectorInfo(out), nil
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) GenerateFallbackPoSt(sectorInfo SortedPublicSectorInfo, challengeSeed [CommLen]byte, faults []uint64) ([]EPostCandidate, []byte, error) {
|
||||
privsectors, err := sb.pubSectorToPriv(sectorInfo, faults)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo.Values())), len(faults))
|
||||
|
||||
proverID := addressToProverID(sb.Miner)
|
||||
candidates, err := sectorbuilder.GenerateCandidates(sb.ssize, proverID, challengeSeed, challengeCount, privsectors)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
proof, err := sectorbuilder.GeneratePoSt(sb.ssize, proverID, privsectors, challengeSeed, candidates)
|
||||
return candidates, proof, err
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) Stop() {
|
||||
close(sb.stopping)
|
||||
}
|
||||
|
||||
func fallbackPostChallengeCount(sectors uint64, faults int) uint64 {
|
||||
challengeCount := types.ElectionPostChallengeCount(sectors, faults)
|
||||
if challengeCount > build.MaxFallbackPostChallengeCount {
|
||||
return build.MaxFallbackPostChallengeCount
|
||||
}
|
||||
return challengeCount
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) ImportFrom(osb *SectorBuilder, symlink bool) error {
|
||||
if err := migrate(osb.filesystem.pathFor(dataCache), sb.filesystem.pathFor(dataCache), symlink); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := migrate(osb.filesystem.pathFor(dataStaging), sb.filesystem.pathFor(dataStaging), symlink); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := migrate(osb.filesystem.pathFor(dataSealed), sb.filesystem.pathFor(dataSealed), symlink); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
val, err := osb.ds.Get(lastSectorIdKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := sb.ds.Put(lastSectorIdKey, val); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sb.lastID = osb.lastID
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sb *SectorBuilder) SetLastSectorID(id uint64) error {
|
||||
if err := sb.ds.Put(lastSectorIdKey, []byte(fmt.Sprint(id))); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sb.lastID = id
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrate(from, to string, symlink bool) error {
|
||||
st, err := os.Stat(from)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if st.IsDir() {
|
||||
return migrateDir(from, to, symlink)
|
||||
}
|
||||
return migrateFile(from, to, symlink)
|
||||
}
|
||||
|
||||
func migrateDir(from, to string, symlink bool) error {
|
||||
tost, err := os.Stat(to)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(to, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if !tost.IsDir() {
|
||||
return xerrors.Errorf("target %q already exists and is a file (expected directory)")
|
||||
}
|
||||
|
||||
dirents, err := ioutil.ReadDir(from)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, inf := range dirents {
|
||||
n := inf.Name()
|
||||
if inf.IsDir() {
|
||||
if err := migrate(filepath.Join(from, n), filepath.Join(to, n), symlink); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err := migrate(filepath.Join(from, n), filepath.Join(to, n), symlink); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateFile(from, to string, symlink bool) error {
|
||||
if symlink {
|
||||
return os.Symlink(from, to)
|
||||
}
|
||||
|
||||
return dcopy.Copy(from, to)
|
||||
}
|
||||
@@ -1,365 +0,0 @@
|
||||
package sectorbuilder_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ffi "github.com/filecoin-project/filecoin-ffi"
|
||||
paramfetch "github.com/filecoin-project/go-paramfetch"
|
||||
"github.com/ipfs/go-datastore"
|
||||
logging "github.com/ipfs/go-log"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/lib/sectorbuilder"
|
||||
)
|
||||
|
||||
func init() {
|
||||
logging.SetLogLevel("*", "INFO")
|
||||
}
|
||||
|
||||
const sectorSize = 1024
|
||||
|
||||
type seal struct {
|
||||
sid uint64
|
||||
|
||||
pco sectorbuilder.RawSealPreCommitOutput
|
||||
ppi sectorbuilder.PublicPieceInfo
|
||||
|
||||
ticket sectorbuilder.SealTicket
|
||||
}
|
||||
|
||||
func (s *seal) precommit(t *testing.T, sb *sectorbuilder.SectorBuilder, sid uint64, done func()) {
|
||||
dlen := sectorbuilder.UserBytesForSectorSize(sectorSize)
|
||||
|
||||
var err error
|
||||
r := io.LimitReader(rand.New(rand.NewSource(42+int64(sid))), int64(dlen))
|
||||
s.ppi, err = sb.AddPiece(dlen, sid, r, []uint64{})
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
|
||||
s.ticket = sectorbuilder.SealTicket{
|
||||
BlockHeight: 5,
|
||||
TicketBytes: [32]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2},
|
||||
}
|
||||
|
||||
s.pco, err = sb.SealPreCommit(sid, s.ticket, []sectorbuilder.PublicPieceInfo{s.ppi})
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
|
||||
done()
|
||||
}
|
||||
|
||||
func (s *seal) commit(t *testing.T, sb *sectorbuilder.SectorBuilder, done func()) {
|
||||
seed := sectorbuilder.SealSeed{
|
||||
BlockHeight: 15,
|
||||
TicketBytes: [32]byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9},
|
||||
}
|
||||
|
||||
proof, err := sb.SealCommit(s.sid, s.ticket, seed, []sectorbuilder.PublicPieceInfo{s.ppi}, s.pco)
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
|
||||
ok, err := sectorbuilder.VerifySeal(sectorSize, s.pco.CommR[:], s.pco.CommD[:], sb.Miner, s.ticket.TicketBytes[:], seed.TicketBytes[:], s.sid, proof)
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
|
||||
if !ok {
|
||||
t.Fatal("proof failed to validate")
|
||||
}
|
||||
|
||||
done()
|
||||
}
|
||||
|
||||
func post(t *testing.T, sb *sectorbuilder.SectorBuilder, seals ...seal) time.Time {
|
||||
cSeed := [32]byte{0, 9, 2, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9}
|
||||
|
||||
ppi := make([]ffi.PublicSectorInfo, len(seals))
|
||||
for i, s := range seals {
|
||||
ppi[i] = ffi.PublicSectorInfo{
|
||||
SectorID: s.sid,
|
||||
CommR: s.pco.CommR,
|
||||
}
|
||||
}
|
||||
|
||||
ssi := sectorbuilder.NewSortedPublicSectorInfo(ppi)
|
||||
|
||||
candndates, err := sb.GenerateEPostCandidates(ssi, cSeed, []uint64{})
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
|
||||
genCandidates := time.Now()
|
||||
|
||||
if len(candndates) != 1 {
|
||||
t.Fatal("expected 1 candidate")
|
||||
}
|
||||
|
||||
postProof, err := sb.ComputeElectionPoSt(ssi, cSeed[:], candndates)
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
|
||||
ok, err := sectorbuilder.VerifyElectionPost(context.TODO(), sb.SectorSize(), ssi, cSeed[:], postProof, candndates, sb.Miner)
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("bad post")
|
||||
}
|
||||
|
||||
return genCandidates
|
||||
}
|
||||
|
||||
func TestSealAndVerify(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping test in short mode")
|
||||
}
|
||||
if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
|
||||
t.Skip("this is slow")
|
||||
}
|
||||
_ = os.Setenv("RUST_LOG", "info")
|
||||
|
||||
build.SectorSizes = []uint64{sectorSize}
|
||||
|
||||
if err := paramfetch.GetParams(sectorSize); err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
|
||||
ds := datastore.NewMapDatastore()
|
||||
|
||||
dir, err := ioutil.TempDir("", "sbtest")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
cleanup := func() {
|
||||
if t.Failed() {
|
||||
fmt.Printf("not removing %s\n", dir)
|
||||
return
|
||||
}
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
si, err := sb.AcquireSectorId()
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
|
||||
s := seal{sid: si}
|
||||
|
||||
start := time.Now()
|
||||
|
||||
s.precommit(t, sb, 1, func() {})
|
||||
|
||||
precommit := time.Now()
|
||||
|
||||
s.commit(t, sb, func() {})
|
||||
|
||||
commit := time.Now()
|
||||
|
||||
genCandidiates := post(t, sb, s)
|
||||
|
||||
epost := time.Now()
|
||||
|
||||
// Restart sectorbuilder, re-run post
|
||||
sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
|
||||
post(t, sb, s)
|
||||
|
||||
fmt.Printf("PreCommit: %s\n", precommit.Sub(start).String())
|
||||
fmt.Printf("Commit: %s\n", commit.Sub(precommit).String())
|
||||
fmt.Printf("GenCandidates: %s\n", genCandidiates.Sub(commit).String())
|
||||
fmt.Printf("EPoSt: %s\n", epost.Sub(genCandidiates).String())
|
||||
}
|
||||
|
||||
func TestSealPoStNoCommit(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping test in short mode")
|
||||
}
|
||||
if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
|
||||
t.Skip("this is slow")
|
||||
}
|
||||
_ = os.Setenv("RUST_LOG", "info")
|
||||
|
||||
build.SectorSizes = []uint64{sectorSize}
|
||||
|
||||
if err := paramfetch.GetParams(sectorSize); err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
|
||||
ds := datastore.NewMapDatastore()
|
||||
|
||||
dir, err := ioutil.TempDir("", "sbtest")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
cleanup := func() {
|
||||
if t.Failed() {
|
||||
fmt.Printf("not removing %s\n", dir)
|
||||
return
|
||||
}
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
si, err := sb.AcquireSectorId()
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
|
||||
s := seal{sid: si}
|
||||
|
||||
start := time.Now()
|
||||
|
||||
s.precommit(t, sb, 1, func() {})
|
||||
|
||||
precommit := time.Now()
|
||||
|
||||
// Restart sectorbuilder, re-run post
|
||||
sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
|
||||
if err := sb.TrimCache(1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
genCandidiates := post(t, sb, s)
|
||||
|
||||
epost := time.Now()
|
||||
|
||||
fmt.Printf("PreCommit: %s\n", precommit.Sub(start).String())
|
||||
fmt.Printf("GenCandidates: %s\n", genCandidiates.Sub(precommit).String())
|
||||
fmt.Printf("EPoSt: %s\n", epost.Sub(genCandidiates).String())
|
||||
}
|
||||
|
||||
func TestSealAndVerify2(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping test in short mode")
|
||||
}
|
||||
if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
|
||||
t.Skip("this is slow")
|
||||
}
|
||||
_ = os.Setenv("RUST_LOG", "info")
|
||||
|
||||
build.SectorSizes = []uint64{sectorSize}
|
||||
|
||||
if err := paramfetch.GetParams(sectorSize); err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
|
||||
ds := datastore.NewMapDatastore()
|
||||
|
||||
dir, err := ioutil.TempDir("", "sbtest")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
cleanup := func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
defer cleanup()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
si1, err := sb.AcquireSectorId()
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
si2, err := sb.AcquireSectorId()
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
|
||||
s1 := seal{sid: si1}
|
||||
s2 := seal{sid: si2}
|
||||
|
||||
wg.Add(2)
|
||||
go s1.precommit(t, sb, 1, wg.Done)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
go s2.precommit(t, sb, 2, wg.Done)
|
||||
wg.Wait()
|
||||
|
||||
wg.Add(2)
|
||||
go s1.commit(t, sb, wg.Done)
|
||||
go s2.commit(t, sb, wg.Done)
|
||||
wg.Wait()
|
||||
|
||||
post(t, sb, s1, s2)
|
||||
}
|
||||
|
||||
func TestAcquireID(t *testing.T) {
|
||||
ds := datastore.NewMapDatastore()
|
||||
|
||||
dir, err := ioutil.TempDir("", "sbtest")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
|
||||
assertAcquire := func(expect uint64) {
|
||||
id, err := sb.AcquireSectorId()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, expect, id)
|
||||
}
|
||||
|
||||
assertAcquire(1)
|
||||
assertAcquire(2)
|
||||
assertAcquire(3)
|
||||
|
||||
sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
|
||||
assertAcquire(4)
|
||||
assertAcquire(5)
|
||||
assertAcquire(6)
|
||||
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package sectorbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
sectorbuilder "github.com/filecoin-project/filecoin-ffi"
|
||||
"go.opencensus.io/trace"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
)
|
||||
|
||||
func (sb *SectorBuilder) SectorSize() uint64 {
|
||||
return sb.ssize
|
||||
}
|
||||
|
||||
var UserBytesForSectorSize = sectorbuilder.GetMaxUserBytesPerStagedSector
|
||||
|
||||
func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address, ticket []byte, seed []byte, sectorID uint64, proof []byte) (bool, error) {
|
||||
var commRa, commDa, ticketa, seeda [32]byte
|
||||
copy(commRa[:], commR)
|
||||
copy(commDa[:], commD)
|
||||
copy(ticketa[:], ticket)
|
||||
copy(seeda[:], seed)
|
||||
proverIDa := addressToProverID(proverID)
|
||||
|
||||
return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, seeda, sectorID, proof)
|
||||
}
|
||||
|
||||
func NewSortedPrivateSectorInfo(sectors []sectorbuilder.PrivateSectorInfo) SortedPrivateSectorInfo {
|
||||
return sectorbuilder.NewSortedPrivateSectorInfo(sectors...)
|
||||
}
|
||||
|
||||
func NewSortedPublicSectorInfo(sectors []sectorbuilder.PublicSectorInfo) SortedPublicSectorInfo {
|
||||
return sectorbuilder.NewSortedPublicSectorInfo(sectors...)
|
||||
}
|
||||
|
||||
func VerifyElectionPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address) (bool, error) {
|
||||
challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values())), 0)
|
||||
return verifyPost(ctx, sectorSize, sectorInfo, challengeCount, challengeSeed, proof, candidates, proverID)
|
||||
}
|
||||
|
||||
func VerifyFallbackPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address, faults int) (bool, error) {
|
||||
challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo.Values())), faults)
|
||||
return verifyPost(ctx, sectorSize, sectorInfo, challengeCount, challengeSeed, proof, candidates, proverID)
|
||||
}
|
||||
|
||||
func verifyPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeCount uint64, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address) (bool, error) {
|
||||
var challengeSeeda [CommLen]byte
|
||||
copy(challengeSeeda[:], challengeSeed)
|
||||
|
||||
_, span := trace.StartSpan(ctx, "VerifyPoSt")
|
||||
defer span.End()
|
||||
prover := addressToProverID(proverID)
|
||||
return sectorbuilder.VerifyPoSt(sectorSize, sectorInfo, challengeSeeda, challengeCount, proof, candidates, prover)
|
||||
}
|
||||
|
||||
func GeneratePieceCommitment(piece io.Reader, pieceSize uint64) (commP [CommLen]byte, err error) {
|
||||
f, werr, err := toReadableFile(piece, int64(pieceSize))
|
||||
if err != nil {
|
||||
return [32]byte{}, err
|
||||
}
|
||||
|
||||
commP, err = sectorbuilder.GeneratePieceCommitmentFromFile(f, pieceSize)
|
||||
if err != nil {
|
||||
return [32]byte{}, err
|
||||
}
|
||||
|
||||
return commP, werr()
|
||||
}
|
||||
|
||||
func GenerateDataCommitment(ssize uint64, pieces []sectorbuilder.PublicPieceInfo) ([CommLen]byte, error) {
|
||||
return sectorbuilder.GenerateDataCommitment(ssize, pieces)
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
package statestore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"reflect"
|
||||
|
||||
cborutil "github.com/filecoin-project/go-cbor-util"
|
||||
"github.com/ipfs/go-datastore"
|
||||
cbg "github.com/whyrusleeping/cbor-gen"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
type StoredState struct {
|
||||
ds datastore.Datastore
|
||||
name datastore.Key
|
||||
}
|
||||
|
||||
func (st *StoredState) End() error {
|
||||
has, err := st.ds.Has(st.name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !has {
|
||||
return xerrors.Errorf("No state for %s", st.name)
|
||||
}
|
||||
if err := st.ds.Delete(st.name); err != nil {
|
||||
return xerrors.Errorf("removing state from datastore: %w", err)
|
||||
}
|
||||
st.name = datastore.Key{}
|
||||
st.ds = nil
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (st *StoredState) Get(out cbg.CBORUnmarshaler) error {
|
||||
val, err := st.ds.Get(st.name)
|
||||
if err != nil {
|
||||
if xerrors.Is(err, datastore.ErrNotFound) {
|
||||
return xerrors.Errorf("No state for %s: %w", st.name, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
return out.UnmarshalCBOR(bytes.NewReader(val))
|
||||
}
|
||||
|
||||
// mutator func(*T) error
|
||||
func (st *StoredState) Mutate(mutator interface{}) error {
|
||||
return st.mutate(cborMutator(mutator))
|
||||
}
|
||||
|
||||
func (st *StoredState) mutate(mutator func([]byte) ([]byte, error)) error {
|
||||
has, err := st.ds.Has(st.name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !has {
|
||||
return xerrors.Errorf("No state for %s", st.name)
|
||||
}
|
||||
|
||||
cur, err := st.ds.Get(st.name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mutated, err := mutator(cur)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return st.ds.Put(st.name, mutated)
|
||||
}
|
||||
|
||||
func cborMutator(mutator interface{}) func([]byte) ([]byte, error) {
|
||||
rmut := reflect.ValueOf(mutator)
|
||||
|
||||
return func(in []byte) ([]byte, error) {
|
||||
state := reflect.New(rmut.Type().In(0).Elem())
|
||||
|
||||
err := cborutil.ReadCborRPC(bytes.NewReader(in), state.Interface())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := rmut.Call([]reflect.Value{state})
|
||||
|
||||
if err := out[0].Interface(); err != nil {
|
||||
return nil, err.(error)
|
||||
}
|
||||
|
||||
return cborutil.Dump(state.Interface())
|
||||
}
|
||||
}
|
||||
@@ -1,96 +0,0 @@
|
||||
package statestore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"github.com/filecoin-project/go-cbor-util"
|
||||
"github.com/ipfs/go-datastore"
|
||||
"github.com/ipfs/go-datastore/query"
|
||||
"go.uber.org/multierr"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
type StateStore struct {
|
||||
ds datastore.Datastore
|
||||
}
|
||||
|
||||
func New(ds datastore.Datastore) *StateStore {
|
||||
return &StateStore{ds: ds}
|
||||
}
|
||||
|
||||
func ToKey(k interface{}) datastore.Key {
|
||||
switch t := k.(type) {
|
||||
case uint64:
|
||||
return datastore.NewKey(fmt.Sprint(t))
|
||||
case fmt.Stringer:
|
||||
return datastore.NewKey(t.String())
|
||||
default:
|
||||
panic("unexpected key type")
|
||||
}
|
||||
}
|
||||
|
||||
func (st *StateStore) Begin(i interface{}, state interface{}) error {
|
||||
k := ToKey(i)
|
||||
has, err := st.ds.Has(k)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if has {
|
||||
return xerrors.Errorf("already tracking state for %v", i)
|
||||
}
|
||||
|
||||
b, err := cborutil.Dump(state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return st.ds.Put(k, b)
|
||||
}
|
||||
|
||||
func (st *StateStore) Get(i interface{}) *StoredState {
|
||||
return &StoredState{
|
||||
ds: st.ds,
|
||||
name: ToKey(i),
|
||||
}
|
||||
}
|
||||
|
||||
func (st *StateStore) Has(i interface{}) (bool, error) {
|
||||
return st.ds.Has(ToKey(i))
|
||||
}
|
||||
|
||||
// out: *[]T
|
||||
func (st *StateStore) List(out interface{}) error {
|
||||
res, err := st.ds.Query(query.Query{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer res.Close()
|
||||
|
||||
outT := reflect.TypeOf(out).Elem().Elem()
|
||||
rout := reflect.ValueOf(out)
|
||||
|
||||
var errs error
|
||||
|
||||
for {
|
||||
res, ok := res.NextSync()
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
if res.Error != nil {
|
||||
return res.Error
|
||||
}
|
||||
|
||||
elem := reflect.New(outT)
|
||||
err := cborutil.ReadCborRPC(bytes.NewReader(res.Value), elem.Interface())
|
||||
if err != nil {
|
||||
errs = multierr.Append(errs, xerrors.Errorf("decoding state for key '%s': %w", res.Key, err))
|
||||
continue
|
||||
}
|
||||
|
||||
rout.Elem().Set(reflect.Append(rout.Elem(), elem.Elem()))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package statestore
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/ipfs/go-datastore"
|
||||
|
||||
"github.com/filecoin-project/go-cbor-util"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
)
|
||||
|
||||
func TestList(t *testing.T) {
|
||||
ds := datastore.NewMapDatastore()
|
||||
|
||||
e, err := cborutil.Dump(types.NewInt(7))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := ds.Put(datastore.NewKey("/2"), e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
st := &StateStore{ds: ds}
|
||||
|
||||
var out []types.BigInt
|
||||
if err := st.List(&out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(out) != 1 {
|
||||
t.Fatal("wrong len")
|
||||
}
|
||||
|
||||
if out[0].Int64() != 7 {
|
||||
t.Fatal("wrong data")
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
logging "github.com/ipfs/go-log"
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
)
|
||||
|
||||
var log = logging.Logger("tarutil")
|
||||
|
||||
Reference in New Issue
Block a user