lotus/extern/sector-storage/mock/mock.go

431 lines
10 KiB
Go
Raw Normal View History

2020-03-23 11:40:02 +00:00
package mock
import (
"bytes"
"context"
"crypto/sha256"
2020-03-23 11:40:02 +00:00
"fmt"
"io"
"math/rand"
"sync"
commcid "github.com/filecoin-project/go-fil-commcid"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-storage/storage"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
2020-03-23 11:40:02 +00:00
)
var log = logging.Logger("sbmock")
type SectorMgr struct {
sectors map[abi.SectorID]*sectorState
2020-05-26 14:39:25 +00:00
pieces map[cid.Cid][]byte
2020-03-23 11:40:02 +00:00
sectorSize abi.SectorSize
nextSectorID abi.SectorNumber
2020-06-15 12:32:17 +00:00
proofType abi.RegisteredSealProof
2020-03-23 11:40:02 +00:00
lk sync.Mutex
}
type mockVerif struct{}
2020-07-23 17:00:20 +00:00
func NewMockSectorMgr(ssize abi.SectorSize, genesisSectors []abi.SectorID) *SectorMgr {
2020-04-10 21:01:35 +00:00
rt, err := ffiwrapper.SealProofTypeFromSectorSize(ssize)
2020-03-23 11:40:02 +00:00
if err != nil {
panic(err)
}
2020-07-23 17:00:20 +00:00
sectors := make(map[abi.SectorID]*sectorState)
for _, sid := range genesisSectors {
sectors[sid] = &sectorState{
failed: false,
state: stateCommit,
}
}
2020-03-23 11:40:02 +00:00
return &SectorMgr{
2020-07-23 17:00:20 +00:00
sectors: sectors,
2020-05-26 14:39:25 +00:00
pieces: map[cid.Cid][]byte{},
2020-03-23 11:40:02 +00:00
sectorSize: ssize,
nextSectorID: 5,
proofType: rt,
}
}
const (
statePacking = iota
statePreCommit
2020-04-01 23:18:20 +00:00
stateCommit // nolint
2020-03-23 11:40:02 +00:00
)
type sectorState struct {
pieces []cid.Cid
failed bool
state int
lk sync.Mutex
}
2020-04-10 19:12:23 +00:00
func (mgr *SectorMgr) NewSector(ctx context.Context, sector abi.SectorID) error {
2020-03-23 11:40:02 +00:00
return nil
}
2020-08-16 10:40:35 +00:00
func (mgr *SectorMgr) AddPiece(ctx context.Context, sectorID abi.SectorID, existingPieces []abi.UnpaddedPieceSize, size abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) {
log.Warn("Add piece: ", sectorID, size, mgr.proofType)
2020-03-23 11:40:02 +00:00
2020-05-26 14:39:25 +00:00
var b bytes.Buffer
tr := io.TeeReader(r, &b)
c, err := ffiwrapper.GeneratePieceCIDFromFile(mgr.proofType, tr, size)
2020-03-23 11:40:02 +00:00
if err != nil {
return abi.PieceInfo{}, xerrors.Errorf("failed to generate piece cid: %w", err)
}
log.Warn("Generated Piece CID: ", c)
mgr.lk.Lock()
2020-05-26 14:39:25 +00:00
mgr.pieces[c] = b.Bytes()
2020-08-16 10:40:35 +00:00
ss, ok := mgr.sectors[sectorID]
if !ok {
ss = &sectorState{
state: statePacking,
}
2020-08-16 10:40:35 +00:00
mgr.sectors[sectorID] = ss
}
mgr.lk.Unlock()
ss.lk.Lock()
2020-03-23 11:40:02 +00:00
ss.pieces = append(ss.pieces, c)
ss.lk.Unlock()
2020-03-23 11:40:02 +00:00
return abi.PieceInfo{
2020-03-23 11:40:02 +00:00
Size: size.Padded(),
PieceCID: c,
}, nil
}
2020-04-10 19:12:23 +00:00
func (mgr *SectorMgr) SectorSize() abi.SectorSize {
return mgr.sectorSize
2020-03-23 11:40:02 +00:00
}
2020-04-10 19:12:23 +00:00
func (mgr *SectorMgr) AcquireSectorNumber() (abi.SectorNumber, error) {
mgr.lk.Lock()
defer mgr.lk.Unlock()
id := mgr.nextSectorID
mgr.nextSectorID++
2020-03-23 11:40:02 +00:00
return id, nil
}
2020-04-10 19:12:23 +00:00
func (mgr *SectorMgr) SealPreCommit1(ctx context.Context, sid abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) {
mgr.lk.Lock()
ss, ok := mgr.sectors[sid]
mgr.lk.Unlock()
2020-03-23 11:40:02 +00:00
if !ok {
2020-03-26 19:34:38 +00:00
return nil, xerrors.Errorf("no sector with id %d in storage", sid)
2020-03-23 11:40:02 +00:00
}
ss.lk.Lock()
defer ss.lk.Unlock()
2020-04-10 19:12:23 +00:00
ussize := abi.PaddedPieceSize(mgr.sectorSize).Unpadded()
2020-03-23 11:40:02 +00:00
// TODO: verify pieces in sinfo.pieces match passed in pieces
var sum abi.UnpaddedPieceSize
for _, p := range pieces {
sum += p.Size.Unpadded()
}
if sum != ussize {
return nil, xerrors.Errorf("aggregated piece sizes don't match up: %d != %d", sum, ussize)
}
if ss.state != statePacking {
return nil, xerrors.Errorf("cannot call pre-seal on sector not in 'packing' state")
}
opFinishWait(ctx)
ss.state = statePreCommit
pis := make([]abi.PieceInfo, len(ss.pieces))
for i, piece := range ss.pieces {
pis[i] = abi.PieceInfo{
Size: pieces[i].Size,
PieceCID: piece,
}
}
2020-04-10 19:12:23 +00:00
commd, err := MockVerifier.GenerateDataCommitment(mgr.proofType, pis)
2020-03-23 11:40:02 +00:00
if err != nil {
return nil, err
}
_, _, cc, err := commcid.CIDToCommitment(commd)
2020-03-23 11:40:02 +00:00
if err != nil {
panic(err)
}
cc[0] ^= 'd'
return cc, nil
}
2020-04-10 19:12:23 +00:00
func (mgr *SectorMgr) SealPreCommit2(ctx context.Context, sid abi.SectorID, phase1Out storage.PreCommit1Out) (cids storage.SectorCids, err error) {
2020-03-23 11:40:02 +00:00
db := []byte(string(phase1Out))
db[0] ^= 'd'
d, _ := commcid.DataCommitmentV1ToCID(db)
2020-03-23 11:40:02 +00:00
commr := make([]byte, 32)
for i := range db {
commr[32-(i+1)] = db[i]
}
commR, _ := commcid.ReplicaCommitmentV1ToCID(commr)
2020-03-23 11:40:02 +00:00
return storage.SectorCids{
Unsealed: d,
Sealed: commR,
}, nil
}
2020-04-10 19:12:23 +00:00
func (mgr *SectorMgr) SealCommit1(ctx context.Context, sid abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (output storage.Commit1Out, err error) {
mgr.lk.Lock()
ss, ok := mgr.sectors[sid]
mgr.lk.Unlock()
2020-03-23 11:40:02 +00:00
if !ok {
return nil, xerrors.Errorf("no such sector %d", sid)
}
ss.lk.Lock()
defer ss.lk.Unlock()
if ss.failed {
return nil, xerrors.Errorf("[mock] cannot commit failed sector %d", sid)
}
if ss.state != statePreCommit {
return nil, xerrors.Errorf("cannot commit sector that has not been precommitted")
}
opFinishWait(ctx)
var out [32]byte
for i := range out {
out[i] = cids.Unsealed.Bytes()[i] + cids.Sealed.Bytes()[31-i] - ticket[i]*seed[i] ^ byte(sid.Number&0xff)
}
return out[:], nil
}
2020-04-10 19:12:23 +00:00
func (mgr *SectorMgr) SealCommit2(ctx context.Context, sid abi.SectorID, phase1Out storage.Commit1Out) (proof storage.Proof, err error) {
2020-03-23 11:40:02 +00:00
var out [32]byte
for i := range out {
out[i] = phase1Out[i] ^ byte(sid.Number&0xff)
}
return out[:], nil
}
// Test Instrumentation Methods
func (mgr *SectorMgr) MarkFailed(sid abi.SectorID, failed bool) error {
2020-04-10 19:12:23 +00:00
mgr.lk.Lock()
defer mgr.lk.Unlock()
ss, ok := mgr.sectors[sid]
2020-03-23 11:40:02 +00:00
if !ok {
2020-03-26 19:34:38 +00:00
return fmt.Errorf("no such sector in storage")
2020-03-23 11:40:02 +00:00
}
ss.failed = failed
2020-03-23 11:40:02 +00:00
return nil
}
func opFinishWait(ctx context.Context) {
val, ok := ctx.Value("opfinish").(chan struct{})
if !ok {
return
}
<-val
}
func AddOpFinish(ctx context.Context) (context.Context, func()) {
done := make(chan struct{})
2020-08-16 10:40:35 +00:00
return context.WithValue(ctx, "opfinish", done), func() { // nolint
2020-03-23 11:40:02 +00:00
close(done)
}
}
2020-04-10 19:12:23 +00:00
func (mgr *SectorMgr) GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []abi.SectorInfo, randomness abi.PoStRandomness) ([]abi.PoStProof, error) {
2020-07-23 17:00:20 +00:00
return generateFakePoSt(sectorInfo, abi.RegisteredSealProof.RegisteredWinningPoStProof, randomness), nil
2020-03-23 11:40:02 +00:00
}
2020-06-08 18:30:48 +00:00
func (mgr *SectorMgr) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []abi.SectorInfo, randomness abi.PoStRandomness) ([]abi.PoStProof, []abi.SectorID, error) {
2020-07-23 17:00:20 +00:00
si := make([]abi.SectorInfo, 0, len(sectorInfo))
var skipped []abi.SectorID
for _, info := range sectorInfo {
sid := abi.SectorID{
Miner: minerID,
Number: info.SectorNumber,
}
_, found := mgr.sectors[sid]
if found && !mgr.sectors[sid].failed {
si = append(si, info)
} else {
skipped = append(skipped, sid)
}
}
return generateFakePoSt(si, abi.RegisteredSealProof.RegisteredWindowPoStProof, randomness), skipped, nil
}
func generateFakePoStProof(sectorInfo []abi.SectorInfo, randomness abi.PoStRandomness) []byte {
hasher := sha256.New()
_, _ = hasher.Write(randomness)
2020-07-23 17:00:20 +00:00
for _, info := range sectorInfo {
err := info.MarshalCBOR(hasher)
if err != nil {
panic(err)
}
}
return hasher.Sum(nil)
2020-07-23 17:00:20 +00:00
}
2020-07-23 17:00:20 +00:00
func generateFakePoSt(sectorInfo []abi.SectorInfo, rpt func(abi.RegisteredSealProof) (abi.RegisteredPoStProof, error), randomness abi.PoStRandomness) []abi.PoStProof {
wp, err := rpt(sectorInfo[0].SealProof)
2020-06-15 12:32:17 +00:00
if err != nil {
panic(err)
}
return []abi.PoStProof{
{
2020-06-15 12:32:17 +00:00
PoStProof: wp,
ProofBytes: generateFakePoStProof(sectorInfo, randomness),
},
}
2020-03-23 11:40:02 +00:00
}
func (mgr *SectorMgr) ReadPiece(ctx context.Context, w io.Writer, sectorID abi.SectorID, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, c cid.Cid) error {
2020-05-26 14:39:25 +00:00
if len(mgr.sectors[sectorID].pieces) > 1 || offset != 0 {
2020-03-23 11:40:02 +00:00
panic("implme")
}
2020-05-26 08:19:42 +00:00
2020-05-26 14:39:25 +00:00
_, err := io.CopyN(w, bytes.NewReader(mgr.pieces[mgr.sectors[sectorID].pieces[0]]), int64(size))
2020-05-26 08:19:42 +00:00
return err
2020-03-23 11:40:02 +00:00
}
2020-04-10 19:12:23 +00:00
func (mgr *SectorMgr) StageFakeData(mid abi.ActorID) (abi.SectorID, []abi.PieceInfo, error) {
usize := abi.PaddedPieceSize(mgr.sectorSize).Unpadded()
sid, err := mgr.AcquireSectorNumber()
2020-03-23 11:40:02 +00:00
if err != nil {
return abi.SectorID{}, nil, err
}
buf := make([]byte, usize)
2020-08-16 10:40:35 +00:00
_, _ = rand.Read(buf) // nolint:gosec
2020-03-23 11:40:02 +00:00
id := abi.SectorID{
Miner: mid,
Number: sid,
}
2020-04-10 19:12:23 +00:00
pi, err := mgr.AddPiece(context.TODO(), id, nil, usize, bytes.NewReader(buf))
2020-03-23 11:40:02 +00:00
if err != nil {
return abi.SectorID{}, nil, err
}
return id, []abi.PieceInfo{pi}, nil
}
func (mgr *SectorMgr) FinalizeSector(context.Context, abi.SectorID, []storage.Range) error {
return nil
}
func (mgr *SectorMgr) ReleaseUnsealed(ctx context.Context, sector abi.SectorID, safeToFree []storage.Range) error {
return nil
}
func (mgr *SectorMgr) Remove(ctx context.Context, sector abi.SectorID) error {
mgr.lk.Lock()
defer mgr.lk.Unlock()
if _, has := mgr.sectors[sector]; !has {
return xerrors.Errorf("sector not found")
}
delete(mgr.sectors, sector)
2020-03-23 11:40:02 +00:00
return nil
}
2020-07-23 17:00:20 +00:00
func (mgr *SectorMgr) CheckProvable(ctx context.Context, spt abi.RegisteredSealProof, ids []abi.SectorID) ([]abi.SectorID, error) {
var bad []abi.SectorID
for _, sid := range ids {
_, found := mgr.sectors[sid]
if !found || mgr.sectors[sid].failed {
bad = append(bad, sid)
}
}
return bad, nil
2020-05-16 21:03:29 +00:00
}
2020-03-23 11:40:02 +00:00
func (m mockVerif) VerifySeal(svi abi.SealVerifyInfo) (bool, error) {
2020-05-22 01:19:46 +00:00
if len(svi.Proof) != 32 { // Real ones are longer, but this should be fine
2020-03-23 11:40:02 +00:00
return false, nil
}
2020-05-22 01:19:46 +00:00
for i, b := range svi.Proof {
if b != svi.UnsealedCID.Bytes()[i]+svi.SealedCID.Bytes()[31-i]-svi.InteractiveRandomness[i]*svi.Randomness[i] {
2020-03-23 11:40:02 +00:00
return false, nil
}
}
return true, nil
}
2020-04-10 19:12:23 +00:00
func (m mockVerif) VerifyWinningPoSt(ctx context.Context, info abi.WinningPoStVerifyInfo) (bool, error) {
return true, nil
2020-04-10 19:12:23 +00:00
}
func (m mockVerif) VerifyWindowPoSt(ctx context.Context, info abi.WindowPoStVerifyInfo) (bool, error) {
2020-07-23 17:00:20 +00:00
if len(info.Proofs) != 1 {
return false, xerrors.Errorf("expected 1 proof entry")
}
proof := info.Proofs[0]
expected := generateFakePoStProof(info.ChallengedSectors, info.Randomness)
if !bytes.Equal(proof.ProofBytes, expected) {
return false, xerrors.Errorf("bad proof")
2020-07-23 17:00:20 +00:00
}
return true, nil
2020-04-10 19:12:23 +00:00
}
2020-06-15 12:32:17 +00:00
func (m mockVerif) GenerateDataCommitment(pt abi.RegisteredSealProof, pieces []abi.PieceInfo) (cid.Cid, error) {
2020-03-26 02:50:56 +00:00
return ffiwrapper.GenerateUnsealedCID(pt, pieces)
2020-03-23 11:40:02 +00:00
}
2020-06-15 12:32:17 +00:00
func (m mockVerif) GenerateWinningPoStSectorChallenge(ctx context.Context, proofType abi.RegisteredPoStProof, minerID abi.ActorID, randomness abi.PoStRandomness, eligibleSectorCount uint64) ([]uint64, error) {
2020-04-17 22:52:42 +00:00
return []uint64{0}, nil
}
2020-03-23 11:40:02 +00:00
var MockVerifier = mockVerif{}
var _ storage.Sealer = &SectorMgr{}
2020-03-26 02:50:56 +00:00
var _ ffiwrapper.Verifier = MockVerifier