cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Fix tests after specs-storage changes

Browse Source
This commit is contained in:
Łukasz Magiera 2020-03-18 02:08:11 +01:00
parent d8ce5078d9
commit 5f2caffb51
19 changed files with 90 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
api/apistruct/struct.go
View File

@ -204,11 +204,11 @@ type WorkerStruct struct {
TaskTypes func(context.Context) (map[sealmgr.TaskType]struct{}, error) `perm:"admin"`
Paths func(context.Context) ([]stores.StoragePath, error) `perm:"admin"`
SealPreCommit1 func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) `perm:"admin"`
SealPreCommit2 func(context.Context, abi.SectorID, storage.PreCommit1Out) (cids storage.SectorCids, err error) `perm:"admin"`
SealPreCommit1 func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) `perm:"admin"`
SealPreCommit2 func(context.Context, abi.SectorID, storage.PreCommit1Out) (cids storage.SectorCids, err error) `perm:"admin"`
SealCommit1 func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) `perm:"admin"`
SealCommit2 func(context.Context, abi.SectorID, storage.Commit1Out) (storage.Proof, error) `perm:"admin"`
FinalizeSector func(context.Context, abi.SectorID) error `perm:"admin"`
SealCommit2 func(context.Context, abi.SectorID, storage.Commit1Out) (storage.Proof, error) `perm:"admin"`
FinalizeSector func(context.Context, abi.SectorID) error `perm:"admin"`
}
}

40
cmd/lotus-bench/main.go
View File

@ -168,7 +168,6 @@ func main() {
}
cfg := &sectorbuilder.Config{
Miner: maddr,
SealProofType: spt,
PoStProofType: ppt,
}
@ -186,8 +185,7 @@ func main() {
}
sbfs := &fs.Basic{
Miner: maddr,
Root: sbdir,
Root: sbdir,
}
sb, err := sectorbuilder.New(sbfs, cfg)
@ -206,12 +204,17 @@ func main() {
var sealedSectors []abi.SectorInfo
numSectors := abi.SectorNumber(1)
for i := abi.SectorNumber(1); i <= numSectors && robench == ""; i++ {
sid := abi.SectorID{
Miner: mid,
Number: i,
}
start := time.Now()
log.Info("Writing piece into sector...")
r := rand.New(rand.NewSource(100 + int64(i)))
pi, err := sb.AddPiece(context.TODO(), i, nil, abi.PaddedPieceSize(sectorSize).Unpadded(), r)
pi, err := sb.AddPiece(context.TODO(), sid, nil, abi.PaddedPieceSize(sectorSize).Unpadded(), r)
if err != nil {
return err
}
@ -223,7 +226,7 @@ func main() {
log.Info("Running replication(1)...")
pieces := []abi.PieceInfo{pi}
pc1o, err := sb.SealPreCommit1(context.TODO(), i, ticket, pieces)
pc1o, err := sb.SealPreCommit1(context.TODO(), sid, ticket, pieces)
if err != nil {
return xerrors.Errorf("commit: %w", err)
}
@ -231,7 +234,7 @@ func main() {
precommit1 := time.Now()
log.Info("Running replication(2)...")
commR, commD, err := sb.SealPreCommit2(context.TODO(), i, pc1o)
cids, err := sb.SealPreCommit2(context.TODO(), sid, pc1o)
if err != nil {
return xerrors.Errorf("commit: %w", err)
}
@ -241,7 +244,7 @@ func main() {
sealedSectors = append(sealedSectors, abi.SectorInfo{
RegisteredProof: spt,
SectorNumber: i,
SealedCID: commR,
SealedCID: cids.Sealed,
})
seed := lapi.SealSeed{
@ -250,7 +253,7 @@ func main() {
}
log.Info("Generating PoRep for sector (1)")
c1o, err := sb.SealCommit1(context.TODO(), i, ticket, seed.Value, pieces, commR, commD)
c1o, err := sb.SealCommit1(context.TODO(), sid, ticket, seed.Value, pieces, cids)
if err != nil {
return err
}
@ -278,7 +281,7 @@ func main() {
var proof storage.Proof
if !c.Bool("skip-commit2") {
proof, err = sb.SealCommit2(context.TODO(), i, c1o)
proof, err = sb.SealCommit2(context.TODO(), sid, c1o)
if err != nil {
return err
}
@ -291,7 +294,7 @@ func main() {
svi := abi.SealVerifyInfo{
SectorID: abi.SectorID{Miner: mid, Number: i},
OnChain: abi.OnChainSealVerifyInfo{
SealedCID: commR,
SealedCID: cids.Sealed,
InteractiveEpoch: seed.Epoch,
RegisteredProof: spt,
Proof: proof,
@ -301,7 +304,7 @@ func main() {
},
Randomness: ticket,
InteractiveRandomness: seed.Value,
UnsealedCID: commD,
UnsealedCID: cids.Unsealed,
}
ok, err := sectorbuilder.ProofVerifier.VerifySeal(svi)
@ -318,7 +321,7 @@ func main() {
if !c.Bool("skip-unseal") {
log.Info("Unsealing sector")
// TODO: RM unsealed sector first
rc, err := sb.ReadPieceFromSealedSector(context.TODO(), 1, 0, abi.UnpaddedPieceSize(sectorSize), ticket, commD)
rc, err := sb.ReadPieceFromSealedSector(context.TODO(), abi.SectorID{Miner: mid, Number: 1}, 0, abi.UnpaddedPieceSize(sectorSize), ticket, cids.Unsealed)
if err != nil {
return err
}
@ -380,7 +383,7 @@ func main() {
if !c.Bool("skip-commit2") {
log.Info("generating election post candidates")
fcandidates, err := sb.GenerateEPostCandidates(sealedSectors, challenge[:], []abi.SectorNumber{})
fcandidates, err := sb.GenerateEPostCandidates(context.TODO(), mid, sealedSectors, challenge[:], []abi.SectorNumber{})
if err != nil {
return err
}
@ -394,7 +397,7 @@ func main() {
gencandidates := time.Now()
log.Info("computing election post snark (cold)")
proof1, err := sb.ComputeElectionPoSt(sealedSectors, challenge[:], candidates[:1])
proof1, err := sb.ComputeElectionPoSt(context.TODO(), mid, sealedSectors, challenge[:], candidates[:1])
if err != nil {
return err
}
@ -402,7 +405,7 @@ func main() {
epost1 := time.Now()
log.Info("computing election post snark (hot)")
proof2, err := sb.ComputeElectionPoSt(sealedSectors, challenge[:], candidates[:1])
proof2, err := sb.ComputeElectionPoSt(context.TODO(), mid, sealedSectors, challenge[:], candidates[:1])
if err != nil {
return err
}
@ -528,6 +531,10 @@ var proveCmd = &cli.Command{
if err != nil {
return err
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return err
}
ppt, spt, err := lapi.ProofTypeFromSectorSize(abi.SectorSize(c2in.SectorSize))
if err != nil {
@ -535,7 +542,6 @@ var proveCmd = &cli.Command{
}
cfg := &sectorbuilder.Config{
Miner: maddr,
SealProofType: spt,
PoStProofType: ppt,
}
@ -547,7 +553,7 @@ var proveCmd = &cli.Command{
start := time.Now()
proof, err := sb.SealCommit2(context.TODO(), abi.SectorNumber(c2in.SectorNum), c2in.Phase1Out)
proof, err := sb.SealCommit2(context.TODO(), abi.SectorID{Miner: abi.ActorID(mid), Number: abi.SectorNumber(c2in.SectorNum)}, c2in.Phase1Out)
if err != nil {
return err
}

2
cmd/lotus-seed/seed/seed.go
View File

@ -59,7 +59,7 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum
next := offset
sbfs := &fs.Basic{
Root: sbroot,
Root: sbroot,
}
sb, err := sectorbuilder.New(sbfs, cfg)

4
cmd/lotus-storage-miner/init.go
View File

@ -400,11 +400,11 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode,
smgr, err := advmgr.New(lr, stores.NewIndex(), &sectorbuilder.Config{
SealProofType: spt,
PoStProofType: ppt,
}, nil)
}, nil, stores.NewIndex())
if err != nil {
return err
}
epp := storage.NewElectionPoStProver(smgr, abi.ActorID(mid))
epp := storage.NewElectionPoStProver(smgr, dtypes.MinerID(mid))
m := miner.NewMiner(api, epp)
{

2
go.mod
View File

@ -22,7 +22,7 @@ require (
github.com/filecoin-project/go-fil-markets v0.0.0-20200304003055-d449a980d4bd
github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6
github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200317165603-bd9e7cb04d81
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200317221918-42574fc2aab9
github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9
github.com/filecoin-project/go-statestore v0.1.0
github.com/filecoin-project/specs-actors v0.0.0-20200311215506-e95895452888

2
go.sum
View File

@ -128,6 +128,8 @@ github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200314022627-38af9db49ba
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200314022627-38af9db49ba2/go.mod h1:NcE+iL0bbYnamGmYQgCPVGbSaf8VF2/CLra/61B3I3I=
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200317165603-bd9e7cb04d81 h1:W5yekTpVTUiB86rSDiZo6rTI3lrLKrsrdY0tx/IqgJA=
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200317165603-bd9e7cb04d81/go.mod h1:3c3MEU9GHLlau37+MmefFNunTo9sVEKfjaJuHBgksdY=
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200317221918-42574fc2aab9 h1:ROfxm5X9dMATYk6MvNe8WBcL2yaelpgGEvve5CnZ1+g=
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200317221918-42574fc2aab9/go.mod h1:3c3MEU9GHLlau37+MmefFNunTo9sVEKfjaJuHBgksdY=
github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 h1:k9qVR9ItcziSB2rxtlkN/MDWNlbsI6yzec+zjUatLW0=
github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ=

2
markets/retrievaladapter/provider.go
View File

@ -45,7 +45,7 @@ func (rpn *retrievalProviderNode) UnsealSector(ctx context.Context, sectorID uin
}
sid := abi.SectorID{
Miner: abi.ActorID(mid),
Miner: abi.ActorID(mid),
Number: abi.SectorNumber(sectorID),
}
return rpn.sealer.ReadPieceFromSealedSector(ctx, sid, sectorbuilder.UnpaddedByteIndex(offset), abi.UnpaddedPieceSize(length), si.Ticket.Value, *si.CommD)

3
node/builder.go
View File

@ -260,6 +260,9 @@ func Online() Option {
// Storage miner
ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner },
Override(new(*stores.Index), stores.NewIndex()),
Override(new(stores.SectorIndex), From(new(*stores.Index))),
Override(new(dtypes.MinerID), modules.MinerID),
Override(new(dtypes.MinerAddress), modules.MinerAddress),
Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig),
Override(new(stores.LocalStorage), From(new(repo.LockedRepo))),
Override(new(sealing.SectorIDCounter), modules.SectorIDCounter),

9
node/modules/dtypes/miner.go Normal file
View File

@ -0,0 +1,9 @@
package dtypes
import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/specs-actors/actors/abi"
)
type MinerAddress address.Address
type MinerID abi.ActorID

21
node/modules/storageminer.go
View File

@ -73,13 +73,18 @@ func GetParams(sbc *sectorbuilder.Config) error {
return nil
}
func SectorBuilderConfig(ds dtypes.MetadataDS, fnapi lapi.FullNode) (*sectorbuilder.Config, error) {
minerAddr, err := minerAddrFromDS(ds)
if err != nil {
return nil, err
}
func MinerAddress(ds dtypes.MetadataDS) (dtypes.MinerAddress, error) {
ma, err := minerAddrFromDS(ds)
return dtypes.MinerAddress(ma), err
}
ssize, err := fnapi.StateMinerSectorSize(context.TODO(), minerAddr, types.EmptyTSK)
func MinerID(ma dtypes.MinerAddress) (dtypes.MinerID, error) {
id, err := address.IDFromAddress(address.Address(ma))
return dtypes.MinerID(id), err
}
func SectorBuilderConfig(maddr dtypes.MinerAddress, fnapi lapi.FullNode) (*sectorbuilder.Config, error) {
ssize, err := fnapi.StateMinerSectorSize(context.TODO(), address.Address(maddr), types.EmptyTSK)
if err != nil {
return nil, err
}
@ -111,7 +116,7 @@ func SectorIDCounter(ds dtypes.MetadataDS) sealing.SectorIDCounter {
return &sidsc{sc}
}
func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h host.Host, ds dtypes.MetadataDS, sealer sealmgr.Manager, tktFn sealing.TicketFn) (*storage.Miner, error) {
func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h host.Host, ds dtypes.MetadataDS, sealer sealmgr.Manager, sc sealing.SectorIDCounter, tktFn sealing.TicketFn) (*storage.Miner, error) {
maddr, err := minerAddrFromDS(ds)
if err != nil {
return nil, err
@ -131,7 +136,7 @@ func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h
fps := storage.NewFPoStScheduler(api, sealer, maddr, worker, ppt)
sm, err := storage.NewMiner(api, maddr, worker, h, ds, sealer, tktFn)
sm, err := storage.NewMiner(api, maddr, worker, h, ds, sealer, sc, tktFn)
if err != nil {
return nil, err
}

5
node/node_test.go
View File

@ -80,6 +80,7 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
for i := 0; i < nPreseal; i++ {
nic.Next()
}
nic.Next()
err = lr.Close()
require.NoError(t, err)
@ -256,7 +257,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn, node.Options())
if err := storers[i].StorageAddLocal(ctx, presealDirs[i]); err != nil {
t.Fatal(err)
t.Fatalf("%+v", err)
}
/*
sma := storers[i].StorageMiner.(*impl.StorageMinerAPI)
@ -386,7 +387,7 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t
storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn, node.Options(
node.Override(new(sealmgr.Manager), func() (sealmgr.Manager, error) {
return sealmgr.NewSimpleManager(storedcounter.New(datastore.NewMapDatastore(), datastore.NewKey("/potato")), genMiner, sbmock.NewMockSectorBuilder(5, build.SectorSizes[0]))
return sealmgr.NewSimpleManager(genMiner, sbmock.NewMockSectorBuilder(5, build.SectorSizes[0]))
}),
node.Unset(new(*advmgr.Manager)),
))

13
storage/miner.go
View File

@ -3,6 +3,7 @@ package storage
import (
"context"
"errors"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"time"
"github.com/filecoin-project/go-address"
@ -36,6 +37,7 @@ type Miner struct {
sealer sealmgr.Manager
ds datastore.Batching
tktFn sealing.TicketFn
sc sealing.SectorIDCounter
maddr address.Address
worker address.Address
@ -72,13 +74,14 @@ type storageMinerApi interface {
WalletHas(context.Context, address.Address) (bool, error)
}
func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, ds datastore.Batching, sealer sealmgr.Manager, tktFn sealing.TicketFn) (*Miner, error) {
func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, ds datastore.Batching, sealer sealmgr.Manager, sc sealing.SectorIDCounter, tktFn sealing.TicketFn) (*Miner, error) {
m := &Miner{
api: api,
h: h,
sealer: sealer,
ds: ds,
tktFn: tktFn,
sc: sc,
maddr: maddr,
worker: worker,
@ -93,7 +96,7 @@ func (m *Miner) Run(ctx context.Context) error {
}
evts := events.NewEvents(ctx, m.api)
m.sealing = sealing.New(m.api, evts, m.maddr, m.worker, m.ds, m.sealer, m.tktFn)
m.sealing = sealing.New(m.api, evts, m.maddr, m.worker, m.ds, m.sealer, m.sc, m.tktFn)
go m.sealing.Run(ctx)
@ -121,11 +124,11 @@ func (m *Miner) runPreflightChecks(ctx context.Context) error {
type SectorBuilderEpp struct {
prover storage.Prover
miner abi.ActorID
miner abi.ActorID
}
func NewElectionPoStProver(sb storage.Prover, miner abi.ActorID) *SectorBuilderEpp {
return &SectorBuilderEpp{sb, miner}
func NewElectionPoStProver(sb storage.Prover, miner dtypes.MinerID) *SectorBuilderEpp {
return &SectorBuilderEpp{sb, abi.ActorID(miner)}
}
var _ gen.ElectionPoStProver = (*SectorBuilderEpp)(nil)

16
storage/sbmock/sbmock.go
View File

@ -304,25 +304,27 @@ func (sb *SBMock) ReadPieceFromSealedSector(ctx context.Context, sectorID abi.Se
return ioutil.NopCloser(io.LimitReader(bytes.NewReader(sb.sectors[sectorID].pieces[0].Bytes()[offset:]), int64(size))), nil
}
func (sb *SBMock) StageFakeData(mid abi.ActorID) (abi.SectorNumber, []abi.PieceInfo, error) {
func (sb *SBMock) StageFakeData(mid abi.ActorID) (abi.SectorID, []abi.PieceInfo, error) {
usize := abi.PaddedPieceSize(sb.sectorSize).Unpadded()
sid, err := sb.AcquireSectorNumber()
if err != nil {
return 0, nil, err
return abi.SectorID{}, nil, err
}
buf := make([]byte, usize)
rand.Read(buf)
pi, err := sb.AddPiece(context.TODO(), abi.SectorID{
id := abi.SectorID{
Miner: mid,
Number: sid,
}, nil, usize, bytes.NewReader(buf))
if err != nil {
return 0, nil, err
}
return sid, []abi.PieceInfo{pi}, nil
pi, err := sb.AddPiece(context.TODO(), id, nil, usize, bytes.NewReader(buf))
if err != nil {
return abi.SectorID{}, nil, err
}
return id, []abi.PieceInfo{pi}, nil
}
func (sb *SBMock) FinalizeSector(context.Context, abi.SectorID) error {

2
storage/sbmock/sbmock_test.go
View File

@ -11,7 +11,7 @@ import (
func TestOpFinish(t *testing.T) {
sb := NewMockSectorBuilder(1, 2048)
sid, pieces, err := sb.StageFakeData()
sid, pieces, err := sb.StageFakeData(123)
if err != nil {
t.Fatal(err)
}

5
storage/sealing/sealing.go
View File

@ -75,7 +75,7 @@ type Sealing struct {
sc SectorIDCounter
}
func New(api sealingApi, events *events.Events, maddr address.Address, worker address.Address, ds datastore.Batching, sealer sealmgr.Manager, tktFn TicketFn) *Sealing {
func New(api sealingApi, events *events.Events, maddr address.Address, worker address.Address, ds datastore.Batching, sealer sealmgr.Manager, sc SectorIDCounter, tktFn TicketFn) *Sealing {
s := &Sealing{
api: api,
events: events,
@ -84,6 +84,7 @@ func New(api sealingApi, events *events.Events, maddr address.Address, worker ad
worker: worker,
sealer: sealer,
tktFn: tktFn,
sc: sc,
}
s.sectors = statemachine.New(namespace.Wrap(ds, datastore.NewKey(SectorStorePrefix)), s, SectorInfo{})
@ -163,7 +164,7 @@ func (m *Sealing) minerSector(num abi.SectorNumber) abi.SectorID {
return abi.SectorID{
Number: num,
Miner: abi.ActorID(mid),
Miner: abi.ActorID(mid),
}
}

1
storage/sealmgr/advmgr/local.go
View File

@ -70,7 +70,6 @@ func (l *LocalWorker) sb() (sectorbuilder.Basic, error) {
return sectorbuilder.New(&localWorkerPathProvider{w: l}, l.scfg)
}
func (l *LocalWorker) NewSector(ctx context.Context, sector abi.SectorID) error {
sb, err := l.sb()
if err != nil {

4
storage/sealmgr/advmgr/manager.go
View File

@ -42,7 +42,7 @@ type Manager struct {
storage2.Prover
}
func New(ls stores.LocalStorage, si *stores.Index, cfg *sectorbuilder.Config, urls URLs) (*Manager, error) {
func New(ls stores.LocalStorage, si *stores.Index, cfg *sectorbuilder.Config, urls URLs, sindex stores.SectorIndex) (*Manager, error) {
stor, err := stores.NewLocal(ls)
if err != nil {
return nil, err
@ -59,7 +59,7 @@ func New(ls stores.LocalStorage, si *stores.Index, cfg *sectorbuilder.Config, ur
m := &Manager{
workers: []Worker{
&LocalWorker{scfg: cfg, storage: stor},
&LocalWorker{scfg: cfg, localStore: stor, storage: stor, sindex: sindex},
},
scfg: cfg,

2
storage/sealmgr/advmgr/roprov.go
View File

@ -11,7 +11,7 @@ import (
)
type readonlyProvider struct {
stor *stores.Local
stor *stores.Local
}
func (l *readonlyProvider) AcquireSector(ctx context.Context, id abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) {

4
storage/sealmgr/simple.go
View File

@ -31,6 +31,10 @@ type sszgetter interface {
SectorSize() abi.SectorSize
}
func (s *LocalWorker) SectorSize() abi.SectorSize {
return s.Basic.(sszgetter).SectorSize()
}
func (s *Simple) SectorSize() abi.SectorSize {
return s.worker.(sszgetter).SectorSize()
}