cerc-io/lotus
15
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

fix TestAPIDeal tests

Browse Source
This commit is contained in:
Anton Evangelatov 2021-06-04 15:41:38 +02:00
parent 0a79f49dc5
commit 88756f3ebf
8 changed files with 31 additions and 314 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
dev.gen
View File

Binary file not shown.

12
extern/sector-storage/mock/mock.go vendored
View File

@ -6,6 +6,7 @@ import (
"crypto/sha256"
"fmt"
"io"
"io/ioutil"
"math/rand"
"sync"
@ -72,6 +73,10 @@ func (mgr *SectorMgr) NewSector(ctx context.Context, sector storage.SectorRef) e
return nil
}
func (mgr *SectorMgr) SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error {
panic("SectorMgr: unsealing piece: implement me")
}
func (mgr *SectorMgr) AddPiece(ctx context.Context, sectorID storage.SectorRef, existingPieces []abi.UnpaddedPieceSize, size abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) {
log.Warn("Add piece: ", sectorID, size, sectorID.ProofType)
@ -372,13 +377,14 @@ func generateFakePoSt(sectorInfo []proof2.SectorInfo, rpt func(abi.RegisteredSea
}
}
func (mgr *SectorMgr) ReadPiece(ctx context.Context, w io.Writer, sectorID storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, c cid.Cid) error {
func (mgr *SectorMgr) ReadPiece(ctx context.Context, sectorID storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, c cid.Cid) (io.ReadCloser, bool, error) {
if offset != 0 {
panic("implme")
}
_, err := io.CopyN(w, bytes.NewReader(mgr.pieces[mgr.sectors[sectorID.ID].pieces[0]]), int64(size))
return err
reader := bytes.NewReader(mgr.pieces[mgr.sectors[sectorID.ID].pieces[0]])
return ioutil.NopCloser(reader), true, nil
}
func (mgr *SectorMgr) StageFakeData(mid abi.ActorID, spt abi.RegisteredSealProof) (storage.SectorRef, []abi.PieceInfo, error) {

14
extern/sector-storage/piece_provider.go vendored
View File

@ -20,21 +20,25 @@ type Unsealer interface {
SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error
}
type PieceProvider struct {
type PieceProvider interface {
ReadPiece(context.Context, storage.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, bool, error)
}
type pieceProvider struct {
storage *stores.Remote
index stores.SectorIndex
uns Unsealer
}
func NewPieceProvider(storage *stores.Remote, index stores.SectorIndex, uns Unsealer) *PieceProvider {
return &PieceProvider{
func NewPieceProvider(storage *stores.Remote, index stores.SectorIndex, uns Unsealer) PieceProvider {
return &pieceProvider{
storage: storage,
index: index,
uns: uns,
}
}
func (p *PieceProvider) tryReadUnsealedPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (io.ReadCloser, context.CancelFunc, error) {
func (p *pieceProvider) tryReadUnsealedPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (io.ReadCloser, context.CancelFunc, error) {
// acquire a lock purely for reading unsealed sectors
ctx, cancel := context.WithCancel(ctx)
if err := p.index.StorageLock(ctx, sector.ID, storiface.FTUnsealed, storiface.FTNone); err != nil {
@ -54,7 +58,7 @@ func (p *PieceProvider) tryReadUnsealedPiece(ctx context.Context, sector storage
return r, cancel, nil
}
func (p *PieceProvider) ReadPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) (io.ReadCloser, bool, error) {
func (p *pieceProvider) ReadPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) (io.ReadCloser, bool, error) {
if err := offset.Valid(); err != nil {
return nil, false, xerrors.Errorf("offset is not valid: %w", err)
}

2
extern/sector-storage/stores/remote.go vendored
View File

@ -357,7 +357,7 @@ func (r *Remote) readRemote(ctx context.Context, url string, spt abi.RegisteredS
return resp.Body, nil
}
// Reated gets a reader for unsealed file range. Can return nil in case the requested range isn't allocated in the file
// Reader gets a reader for unsealed file range. Can return nil in case the requested range isn't allocated in the file
func (r *Remote) Reader(ctx context.Context, s storage.SectorRef, offset, size abi.PaddedPieceSize, ft storiface.SectorFileType) (io.ReadCloser, error) {
if ft != storiface.FTUnsealed {
return nil, xerrors.Errorf("reader only supports unsealed files")

300
localnet.json
View File

@ -1,300 +0,0 @@
{
"NetworkVersion": 12,
"Accounts": [
{
"Type": "account",
"Balance": "50000000000000000000000000",
"Meta": {
"Owner": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq"
}
}
],
"Miners": [
{
"ID": "t01000",
"Owner": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq",
"Worker": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq",
"PeerId": "12D3KooWPteu8RJqJzoscKmU1nyd9kxuWUqEoBZNni3hBEWN1e9J",
"MarketBalance": "0",
"PowerBalance": "0",
"SectorSize": 2048,
"Sectors": [
{
"CommR": {
"/": "bagboea4b5abcabff7i7vgvlgnf2366jre2rvyxc5qhj6cyzxzimus6dedvrqxxi7"
},
"CommD": {
"/": "baga6ea4seaqabbd2j6tsgzgdi76f6vhzacjbw26pithbdqtvw2mahoslwywswni"
},
"SectorID": 0,
"Deal": {
"PieceCID": {
"/": "baga6ea4seaqabbd2j6tsgzgdi76f6vhzacjbw26pithbdqtvw2mahoslwywswni"
},
"PieceSize": 2048,
"VerifiedDeal": false,
"Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq",
"Provider": "t01000",
"Label": "0",
"StartEpoch": 0,
"EndEpoch": 9001,
"StoragePricePerEpoch": "0",
"ProviderCollateral": "0",
"ClientCollateral": "0"
},
"ProofType": 5
},
{
"CommR": {
"/": "bagboea4b5abcavd5dbzijtfwkbpbbqmr5kdhgb2tewdy6n63egem2evlr7hk4zzi"
},
"CommD": {
"/": "baga6ea4seaqbolro7w44yc545wvbbexjw3b5ohiuoutousvgw2pwlojnzz3ggma"
},
"SectorID": 1,
"Deal": {
"PieceCID": {
"/": "baga6ea4seaqbolro7w44yc545wvbbexjw3b5ohiuoutousvgw2pwlojnzz3ggma"
},
"PieceSize": 2048,
"VerifiedDeal": false,
"Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq",
"Provider": "t01000",
"Label": "1",
"StartEpoch": 0,
"EndEpoch": 9001,
"StoragePricePerEpoch": "0",
"ProviderCollateral": "0",
"ClientCollateral": "0"
},
"ProofType": 5
},
{
"CommR": {
"/": "bagboea4b5abcbrnifjhj3vauuh2rnoejhl7q7uayo5ijf4iodwb3jiblzurficab"
},
"CommD": {
"/": "baga6ea4seaqcrjzlcq234z2xrl3mefikifxlb4dqmzt3uaj3mv5mqh22govmgfa"
},
"SectorID": 2,
"Deal": {
"PieceCID": {
"/": "baga6ea4seaqcrjzlcq234z2xrl3mefikifxlb4dqmzt3uaj3mv5mqh22govmgfa"
},
"PieceSize": 2048,
"VerifiedDeal": false,
"Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq",
"Provider": "t01000",
"Label": "2",
"StartEpoch": 0,
"EndEpoch": 9001,
"StoragePricePerEpoch": "0",
"ProviderCollateral": "0",
"ClientCollateral": "0"
},
"ProofType": 5
},
{
"CommR": {
"/": "bagboea4b5abcayxkrdo2tkfczg6jkrfvgp6pwituchpn4qxtgn47ejmvwici6wyf"
},
"CommD": {
"/": "baga6ea4seaqbop36vw2aa3tpetkmdvlztopr664rfnaqqvgvstiatheixvnc6nq"
},
"SectorID": 3,
"Deal": {
"PieceCID": {
"/": "baga6ea4seaqbop36vw2aa3tpetkmdvlztopr664rfnaqqvgvstiatheixvnc6nq"
},
"PieceSize": 2048,
"VerifiedDeal": false,
"Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq",
"Provider": "t01000",
"Label": "3",
"StartEpoch": 0,
"EndEpoch": 9001,
"StoragePricePerEpoch": "0",
"ProviderCollateral": "0",
"ClientCollateral": "0"
},
"ProofType": 5
},
{
"CommR": {
"/": "bagboea4b5abcbjid7nmwjl57jos4p7ojliwxq4qobstf65tynz3sn5lfy44jzhds"
},
"CommD": {
"/": "baga6ea4seaqntlbccxmhxic2zpac5dkbsby5doafv42aitlzk3c5qdhixtbtaba"
},
"SectorID": 4,
"Deal": {
"PieceCID": {
"/": "baga6ea4seaqntlbccxmhxic2zpac5dkbsby5doafv42aitlzk3c5qdhixtbtaba"
},
"PieceSize": 2048,
"VerifiedDeal": false,
"Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq",
"Provider": "t01000",
"Label": "4",
"StartEpoch": 0,
"EndEpoch": 9001,
"StoragePricePerEpoch": "0",
"ProviderCollateral": "0",
"ClientCollateral": "0"
},
"ProofType": 5
},
{
"CommR": {
"/": "bagboea4b5abcbstygxcgobkm7qx7ya5xwpcf5axfb4ousjkcgwzfqeztczfavhsn"
},
"CommD": {
"/": "baga6ea4seaqnsf5hd64532vliew3h3cef7nr6o66yd7v5s4ch5yx3bymlhypyoa"
},
"SectorID": 5,
"Deal": {
"PieceCID": {
"/": "baga6ea4seaqnsf5hd64532vliew3h3cef7nr6o66yd7v5s4ch5yx3bymlhypyoa"
},
"PieceSize": 2048,
"VerifiedDeal": false,
"Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq",
"Provider": "t01000",
"Label": "5",
"StartEpoch": 0,
"EndEpoch": 9001,
"StoragePricePerEpoch": "0",
"ProviderCollateral": "0",
"ClientCollateral": "0"
},
"ProofType": 5
},
{
"CommR": {
"/": "bagboea4b5abcbfyeiqwyooe74zmcnkk3akcjn4ounuy4m6sovn5nqei54aqcefsv"
},
"CommD": {
"/": "baga6ea4seaqfkrsl36uvnj3xn3264rkvt2lgrb3pe2pbt6ka2plzemdhqudvmoq"
},
"SectorID": 6,
"Deal": {
"PieceCID": {
"/": "baga6ea4seaqfkrsl36uvnj3xn3264rkvt2lgrb3pe2pbt6ka2plzemdhqudvmoq"
},
"PieceSize": 2048,
"VerifiedDeal": false,
"Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq",
"Provider": "t01000",
"Label": "6",
"StartEpoch": 0,
"EndEpoch": 9001,
"StoragePricePerEpoch": "0",
"ProviderCollateral": "0",
"ClientCollateral": "0"
},
"ProofType": 5
},
{
"CommR": {
"/": "bagboea4b5abcbidmr6ca4eb45n7bdqbrzq4jyxmpazfg3liqld45mpf42b2iofdf"
},
"CommD": {
"/": "baga6ea4seaqaeahwyiswdcd4mhy7ltj4u7gx22tbearu7savnwqkb7xxqas64pa"
},
"SectorID": 7,
"Deal": {
"PieceCID": {
"/": "baga6ea4seaqaeahwyiswdcd4mhy7ltj4u7gx22tbearu7savnwqkb7xxqas64pa"
},
"PieceSize": 2048,
"VerifiedDeal": false,
"Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq",
"Provider": "t01000",
"Label": "7",
"StartEpoch": 0,
"EndEpoch": 9001,
"StoragePricePerEpoch": "0",
"ProviderCollateral": "0",
"ClientCollateral": "0"
},
"ProofType": 5
},
{
"CommR": {
"/": "bagboea4b5abcbrap6kd5jbpp7rikxvpuqc7gp2tmj7jycnwhhtwzzffyaryoxdrv"
},
"CommD": {
"/": "baga6ea4seaqoc53weeh6snwltlbutthfa6tqpbfmzsqxvhkfsq4v47jacqzv6pi"
},
"SectorID": 8,
"Deal": {
"PieceCID": {
"/": "baga6ea4seaqoc53weeh6snwltlbutthfa6tqpbfmzsqxvhkfsq4v47jacqzv6pi"
},
"PieceSize": 2048,
"VerifiedDeal": false,
"Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq",
"Provider": "t01000",
"Label": "8",
"StartEpoch": 0,
"EndEpoch": 9001,
"StoragePricePerEpoch": "0",
"ProviderCollateral": "0",
"ClientCollateral": "0"
},
"ProofType": 5
},
{
"CommR": {
"/": "bagboea4b5abcb2sl6swcwv4qchtolwj5o6onqanbslcjdcylww3hwfw5hxzmdurq"
},
"CommD": {
"/": "baga6ea4seaqb256sqbv6syryaqlxrapawsfs2dmbhhfzjms7l4vndyih55d6kji"
},
"SectorID": 9,
"Deal": {
"PieceCID": {
"/": "baga6ea4seaqb256sqbv6syryaqlxrapawsfs2dmbhhfzjms7l4vndyih55d6kji"
},
"PieceSize": 2048,
"VerifiedDeal": false,
"Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq",
"Provider": "t01000",
"Label": "9",
"StartEpoch": 0,
"EndEpoch": 9001,
"StoragePricePerEpoch": "0",
"ProviderCollateral": "0",
"ClientCollateral": "0"
},
"ProofType": 5
}
]
}
],
"NetworkName": "localnet-9899f5bc-a891-44ff-883d-176654eab965",
"VerifregRootKey": {
"Type": "multisig",
"Balance": "0",
"Meta": {
"Signers": [
"t1ceb34gnsc6qk5dt6n7xg6ycwzasjhbxm3iylkiy"
],
"Threshold": 1,
"VestingDuration": 0,
"VestingStart": 0
}
},
"RemainderAccount": {
"Type": "multisig",
"Balance": "0",
"Meta": {
"Signers": [
"t1ceb34gnsc6qk5dt6n7xg6ycwzasjhbxm3iylkiy"
],
"Threshold": 1,
"VestingDuration": 0,
"VestingStart": 0
}
}
}

4
markets/retrievaladapter/provider.go
View File

@ -29,13 +29,13 @@ var log = logging.Logger("retrievaladapter")
type retrievalProviderNode struct {
maddr address.Address
secb sectorblocks.SectorBuilder
pp *sectorstorage.PieceProvider
pp sectorstorage.PieceProvider
full v1api.FullNode
}
// NewRetrievalProviderNode returns a new node adapter for a retrieval provider that talks to the
// Lotus Node
func NewRetrievalProviderNode(maddr dtypes.MinerAddress, secb sectorblocks.SectorBuilder, pp *sectorstorage.PieceProvider, full v1api.FullNode) retrievalmarket.RetrievalProviderNode {
func NewRetrievalProviderNode(maddr dtypes.MinerAddress, secb sectorblocks.SectorBuilder, pp sectorstorage.PieceProvider, full v1api.FullNode) retrievalmarket.RetrievalProviderNode {
return &retrievalProviderNode{address.Address(maddr), secb, pp, full}
}

2
node/builder_miner.go
View File

@ -123,7 +123,7 @@ func ConfigStorageMiner(c interface{}) Option {
Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks),
// Markets (retrieval deps)
Override(new(*sectorstorage.PieceProvider), sectorstorage.NewPieceProvider),
Override(new(sectorstorage.PieceProvider), sectorstorage.NewPieceProvider),
// Markets (retrieval)

11
node/test/builder.go
View File

@ -521,6 +521,7 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []tes
Number: sector.SectorID,
}
}
mgr := mock.NewMockSectorMgr(sectors)
opts := def.Opts
if opts == nil {
@ -528,10 +529,16 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []tes
}
storers[i] = CreateTestStorageNode(ctx, t, genms[i].Worker, maddrs[i], pidKeys[i], f, mn, node.Options(
node.Override(new(sectorstorage.SectorManager), func() (sectorstorage.SectorManager, error) {
return mock.NewMockSectorMgr(sectors), nil
return mgr, nil
}),
node.Override(new(sectorstorage.PieceProvider), func() (sectorstorage.PieceProvider, error) {
return mgr, nil
}),
node.Override(new(sectorstorage.Unsealer), func() (sectorstorage.Unsealer, error) {
return mgr, nil
}),
node.Override(new(ffiwrapper.Verifier), mock.MockVerifier),
node.Unset(new(*sectorstorage.Manager)),
//node.Unset(new(*sectorstorage.Manager)),
opts,
))