diff --git a/dev.gen b/dev.gen deleted file mode 100644 index 44bbf6e36..000000000 Binary files a/dev.gen and /dev/null differ diff --git a/extern/sector-storage/mock/mock.go b/extern/sector-storage/mock/mock.go index ae7d54985..56b8f0627 100644 --- a/extern/sector-storage/mock/mock.go +++ b/extern/sector-storage/mock/mock.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "fmt" "io" + "io/ioutil" "math/rand" "sync" @@ -72,6 +73,10 @@ func (mgr *SectorMgr) NewSector(ctx context.Context, sector storage.SectorRef) e return nil } +func (mgr *SectorMgr) SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error { + panic("SectorMgr: unsealing piece: implement me") +} + func (mgr *SectorMgr) AddPiece(ctx context.Context, sectorID storage.SectorRef, existingPieces []abi.UnpaddedPieceSize, size abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) { log.Warn("Add piece: ", sectorID, size, sectorID.ProofType) @@ -372,13 +377,14 @@ func generateFakePoSt(sectorInfo []proof2.SectorInfo, rpt func(abi.RegisteredSea } } -func (mgr *SectorMgr) ReadPiece(ctx context.Context, w io.Writer, sectorID storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, c cid.Cid) error { +func (mgr *SectorMgr) ReadPiece(ctx context.Context, sectorID storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, c cid.Cid) (io.ReadCloser, bool, error) { if offset != 0 { panic("implme") } - _, err := io.CopyN(w, bytes.NewReader(mgr.pieces[mgr.sectors[sectorID.ID].pieces[0]]), int64(size)) - return err + reader := bytes.NewReader(mgr.pieces[mgr.sectors[sectorID.ID].pieces[0]]) + + return ioutil.NopCloser(reader), true, nil } func (mgr *SectorMgr) StageFakeData(mid abi.ActorID, spt abi.RegisteredSealProof) (storage.SectorRef, []abi.PieceInfo, error) { diff --git a/extern/sector-storage/piece_provider.go b/extern/sector-storage/piece_provider.go index 99677121d..20d19d527 100644 --- a/extern/sector-storage/piece_provider.go +++ b/extern/sector-storage/piece_provider.go @@ -20,21 +20,25 @@ type Unsealer interface { SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error } -type PieceProvider struct { +type PieceProvider interface { + ReadPiece(context.Context, storage.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, bool, error) +} + +type pieceProvider struct { storage *stores.Remote index stores.SectorIndex uns Unsealer } -func NewPieceProvider(storage *stores.Remote, index stores.SectorIndex, uns Unsealer) *PieceProvider { - return &PieceProvider{ +func NewPieceProvider(storage *stores.Remote, index stores.SectorIndex, uns Unsealer) PieceProvider { + return &pieceProvider{ storage: storage, index: index, uns: uns, } } -func (p *PieceProvider) tryReadUnsealedPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (io.ReadCloser, context.CancelFunc, error) { +func (p *pieceProvider) tryReadUnsealedPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (io.ReadCloser, context.CancelFunc, error) { // acquire a lock purely for reading unsealed sectors ctx, cancel := context.WithCancel(ctx) if err := p.index.StorageLock(ctx, sector.ID, storiface.FTUnsealed, storiface.FTNone); err != nil { @@ -54,7 +58,7 @@ func (p *PieceProvider) tryReadUnsealedPiece(ctx context.Context, sector storage return r, cancel, nil } -func (p *PieceProvider) ReadPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) (io.ReadCloser, bool, error) { +func (p *pieceProvider) ReadPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) (io.ReadCloser, bool, error) { if err := offset.Valid(); err != nil { return nil, false, xerrors.Errorf("offset is not valid: %w", err) } diff --git a/extern/sector-storage/stores/remote.go b/extern/sector-storage/stores/remote.go index bb4741ab6..8f5043c73 100644 --- a/extern/sector-storage/stores/remote.go +++ b/extern/sector-storage/stores/remote.go @@ -357,7 +357,7 @@ func (r *Remote) readRemote(ctx context.Context, url string, spt abi.RegisteredS return resp.Body, nil } -// Reated gets a reader for unsealed file range. Can return nil in case the requested range isn't allocated in the file +// Reader gets a reader for unsealed file range. Can return nil in case the requested range isn't allocated in the file func (r *Remote) Reader(ctx context.Context, s storage.SectorRef, offset, size abi.PaddedPieceSize, ft storiface.SectorFileType) (io.ReadCloser, error) { if ft != storiface.FTUnsealed { return nil, xerrors.Errorf("reader only supports unsealed files") diff --git a/localnet.json b/localnet.json deleted file mode 100644 index 6f4a5fbab..000000000 --- a/localnet.json +++ /dev/null @@ -1,300 +0,0 @@ -{ - "NetworkVersion": 12, - "Accounts": [ - { - "Type": "account", - "Balance": "50000000000000000000000000", - "Meta": { - "Owner": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq" - } - } - ], - "Miners": [ - { - "ID": "t01000", - "Owner": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq", - "Worker": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq", - "PeerId": "12D3KooWPteu8RJqJzoscKmU1nyd9kxuWUqEoBZNni3hBEWN1e9J", - "MarketBalance": "0", - "PowerBalance": "0", - "SectorSize": 2048, - "Sectors": [ - { - "CommR": { - "/": "bagboea4b5abcabff7i7vgvlgnf2366jre2rvyxc5qhj6cyzxzimus6dedvrqxxi7" - }, - "CommD": { - "/": "baga6ea4seaqabbd2j6tsgzgdi76f6vhzacjbw26pithbdqtvw2mahoslwywswni" - }, - "SectorID": 0, - "Deal": { - "PieceCID": { - "/": "baga6ea4seaqabbd2j6tsgzgdi76f6vhzacjbw26pithbdqtvw2mahoslwywswni" - }, - "PieceSize": 2048, - "VerifiedDeal": false, - "Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq", - "Provider": "t01000", - "Label": "0", - "StartEpoch": 0, - "EndEpoch": 9001, - "StoragePricePerEpoch": "0", - "ProviderCollateral": "0", - "ClientCollateral": "0" - }, - "ProofType": 5 - }, - { - "CommR": { - "/": "bagboea4b5abcavd5dbzijtfwkbpbbqmr5kdhgb2tewdy6n63egem2evlr7hk4zzi" - }, - "CommD": { - "/": "baga6ea4seaqbolro7w44yc545wvbbexjw3b5ohiuoutousvgw2pwlojnzz3ggma" - }, - "SectorID": 1, - "Deal": { - "PieceCID": { - "/": "baga6ea4seaqbolro7w44yc545wvbbexjw3b5ohiuoutousvgw2pwlojnzz3ggma" - }, - "PieceSize": 2048, - "VerifiedDeal": false, - "Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq", - "Provider": "t01000", - "Label": "1", - "StartEpoch": 0, - "EndEpoch": 9001, - "StoragePricePerEpoch": "0", - "ProviderCollateral": "0", - "ClientCollateral": "0" - }, - "ProofType": 5 - }, - { - "CommR": { - "/": "bagboea4b5abcbrnifjhj3vauuh2rnoejhl7q7uayo5ijf4iodwb3jiblzurficab" - }, - "CommD": { - "/": "baga6ea4seaqcrjzlcq234z2xrl3mefikifxlb4dqmzt3uaj3mv5mqh22govmgfa" - }, - "SectorID": 2, - "Deal": { - "PieceCID": { - "/": "baga6ea4seaqcrjzlcq234z2xrl3mefikifxlb4dqmzt3uaj3mv5mqh22govmgfa" - }, - "PieceSize": 2048, - "VerifiedDeal": false, - "Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq", - "Provider": "t01000", - "Label": "2", - "StartEpoch": 0, - "EndEpoch": 9001, - "StoragePricePerEpoch": "0", - "ProviderCollateral": "0", - "ClientCollateral": "0" - }, - "ProofType": 5 - }, - { - "CommR": { - "/": "bagboea4b5abcayxkrdo2tkfczg6jkrfvgp6pwituchpn4qxtgn47ejmvwici6wyf" - }, - "CommD": { - "/": "baga6ea4seaqbop36vw2aa3tpetkmdvlztopr664rfnaqqvgvstiatheixvnc6nq" - }, - "SectorID": 3, - "Deal": { - "PieceCID": { - "/": "baga6ea4seaqbop36vw2aa3tpetkmdvlztopr664rfnaqqvgvstiatheixvnc6nq" - }, - "PieceSize": 2048, - "VerifiedDeal": false, - "Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq", - "Provider": "t01000", - "Label": "3", - "StartEpoch": 0, - "EndEpoch": 9001, - "StoragePricePerEpoch": "0", - "ProviderCollateral": "0", - "ClientCollateral": "0" - }, - "ProofType": 5 - }, - { - "CommR": { - "/": "bagboea4b5abcbjid7nmwjl57jos4p7ojliwxq4qobstf65tynz3sn5lfy44jzhds" - }, - "CommD": { - "/": "baga6ea4seaqntlbccxmhxic2zpac5dkbsby5doafv42aitlzk3c5qdhixtbtaba" - }, - "SectorID": 4, - "Deal": { - "PieceCID": { - "/": "baga6ea4seaqntlbccxmhxic2zpac5dkbsby5doafv42aitlzk3c5qdhixtbtaba" - }, - "PieceSize": 2048, - "VerifiedDeal": false, - "Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq", - "Provider": "t01000", - "Label": "4", - "StartEpoch": 0, - "EndEpoch": 9001, - "StoragePricePerEpoch": "0", - "ProviderCollateral": "0", - "ClientCollateral": "0" - }, - "ProofType": 5 - }, - { - "CommR": { - "/": "bagboea4b5abcbstygxcgobkm7qx7ya5xwpcf5axfb4ousjkcgwzfqeztczfavhsn" - }, - "CommD": { - "/": "baga6ea4seaqnsf5hd64532vliew3h3cef7nr6o66yd7v5s4ch5yx3bymlhypyoa" - }, - "SectorID": 5, - "Deal": { - "PieceCID": { - "/": "baga6ea4seaqnsf5hd64532vliew3h3cef7nr6o66yd7v5s4ch5yx3bymlhypyoa" - }, - "PieceSize": 2048, - "VerifiedDeal": false, - "Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq", - "Provider": "t01000", - "Label": "5", - "StartEpoch": 0, - "EndEpoch": 9001, - "StoragePricePerEpoch": "0", - "ProviderCollateral": "0", - "ClientCollateral": "0" - }, - "ProofType": 5 - }, - { - "CommR": { - "/": "bagboea4b5abcbfyeiqwyooe74zmcnkk3akcjn4ounuy4m6sovn5nqei54aqcefsv" - }, - "CommD": { - "/": "baga6ea4seaqfkrsl36uvnj3xn3264rkvt2lgrb3pe2pbt6ka2plzemdhqudvmoq" - }, - "SectorID": 6, - "Deal": { - "PieceCID": { - "/": "baga6ea4seaqfkrsl36uvnj3xn3264rkvt2lgrb3pe2pbt6ka2plzemdhqudvmoq" - }, - "PieceSize": 2048, - "VerifiedDeal": false, - "Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq", - "Provider": "t01000", - "Label": "6", - "StartEpoch": 0, - "EndEpoch": 9001, - "StoragePricePerEpoch": "0", - "ProviderCollateral": "0", - "ClientCollateral": "0" - }, - "ProofType": 5 - }, - { - "CommR": { - "/": "bagboea4b5abcbidmr6ca4eb45n7bdqbrzq4jyxmpazfg3liqld45mpf42b2iofdf" - }, - "CommD": { - "/": "baga6ea4seaqaeahwyiswdcd4mhy7ltj4u7gx22tbearu7savnwqkb7xxqas64pa" - }, - "SectorID": 7, - "Deal": { - "PieceCID": { - "/": "baga6ea4seaqaeahwyiswdcd4mhy7ltj4u7gx22tbearu7savnwqkb7xxqas64pa" - }, - "PieceSize": 2048, - "VerifiedDeal": false, - "Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq", - "Provider": "t01000", - "Label": "7", - "StartEpoch": 0, - "EndEpoch": 9001, - "StoragePricePerEpoch": "0", - "ProviderCollateral": "0", - "ClientCollateral": "0" - }, - "ProofType": 5 - }, - { - "CommR": { - "/": "bagboea4b5abcbrap6kd5jbpp7rikxvpuqc7gp2tmj7jycnwhhtwzzffyaryoxdrv" - }, - "CommD": { - "/": "baga6ea4seaqoc53weeh6snwltlbutthfa6tqpbfmzsqxvhkfsq4v47jacqzv6pi" - }, - "SectorID": 8, - "Deal": { - "PieceCID": { - "/": "baga6ea4seaqoc53weeh6snwltlbutthfa6tqpbfmzsqxvhkfsq4v47jacqzv6pi" - }, - "PieceSize": 2048, - "VerifiedDeal": false, - "Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq", - "Provider": "t01000", - "Label": "8", - "StartEpoch": 0, - "EndEpoch": 9001, - "StoragePricePerEpoch": "0", - "ProviderCollateral": "0", - "ClientCollateral": "0" - }, - "ProofType": 5 - }, - { - "CommR": { - "/": "bagboea4b5abcb2sl6swcwv4qchtolwj5o6onqanbslcjdcylww3hwfw5hxzmdurq" - }, - "CommD": { - "/": "baga6ea4seaqb256sqbv6syryaqlxrapawsfs2dmbhhfzjms7l4vndyih55d6kji" - }, - "SectorID": 9, - "Deal": { - "PieceCID": { - "/": "baga6ea4seaqb256sqbv6syryaqlxrapawsfs2dmbhhfzjms7l4vndyih55d6kji" - }, - "PieceSize": 2048, - "VerifiedDeal": false, - "Client": "t3rnqpbzyhqnvkscl4qimgpqqukynzdbqyuvlc24kdam6i6oeuzhhwed5l2ahvqkdiqacc34wi4oh2grcnshuq", - "Provider": "t01000", - "Label": "9", - "StartEpoch": 0, - "EndEpoch": 9001, - "StoragePricePerEpoch": "0", - "ProviderCollateral": "0", - "ClientCollateral": "0" - }, - "ProofType": 5 - } - ] - } - ], - "NetworkName": "localnet-9899f5bc-a891-44ff-883d-176654eab965", - "VerifregRootKey": { - "Type": "multisig", - "Balance": "0", - "Meta": { - "Signers": [ - "t1ceb34gnsc6qk5dt6n7xg6ycwzasjhbxm3iylkiy" - ], - "Threshold": 1, - "VestingDuration": 0, - "VestingStart": 0 - } - }, - "RemainderAccount": { - "Type": "multisig", - "Balance": "0", - "Meta": { - "Signers": [ - "t1ceb34gnsc6qk5dt6n7xg6ycwzasjhbxm3iylkiy" - ], - "Threshold": 1, - "VestingDuration": 0, - "VestingStart": 0 - } - } -} \ No newline at end of file diff --git a/markets/retrievaladapter/provider.go b/markets/retrievaladapter/provider.go index 2e255c07c..887c6ebff 100644 --- a/markets/retrievaladapter/provider.go +++ b/markets/retrievaladapter/provider.go @@ -29,13 +29,13 @@ var log = logging.Logger("retrievaladapter") type retrievalProviderNode struct { maddr address.Address secb sectorblocks.SectorBuilder - pp *sectorstorage.PieceProvider + pp sectorstorage.PieceProvider full v1api.FullNode } // NewRetrievalProviderNode returns a new node adapter for a retrieval provider that talks to the // Lotus Node -func NewRetrievalProviderNode(maddr dtypes.MinerAddress, secb sectorblocks.SectorBuilder, pp *sectorstorage.PieceProvider, full v1api.FullNode) retrievalmarket.RetrievalProviderNode { +func NewRetrievalProviderNode(maddr dtypes.MinerAddress, secb sectorblocks.SectorBuilder, pp sectorstorage.PieceProvider, full v1api.FullNode) retrievalmarket.RetrievalProviderNode { return &retrievalProviderNode{address.Address(maddr), secb, pp, full} } diff --git a/node/builder_miner.go b/node/builder_miner.go index 8478288cb..bcca25915 100644 --- a/node/builder_miner.go +++ b/node/builder_miner.go @@ -123,7 +123,7 @@ func ConfigStorageMiner(c interface{}) Option { Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), // Markets (retrieval deps) - Override(new(*sectorstorage.PieceProvider), sectorstorage.NewPieceProvider), + Override(new(sectorstorage.PieceProvider), sectorstorage.NewPieceProvider), // Markets (retrieval) diff --git a/node/test/builder.go b/node/test/builder.go index bd180ee41..6f3dc7cfc 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -521,6 +521,7 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []tes Number: sector.SectorID, } } + mgr := mock.NewMockSectorMgr(sectors) opts := def.Opts if opts == nil { @@ -528,10 +529,16 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []tes } storers[i] = CreateTestStorageNode(ctx, t, genms[i].Worker, maddrs[i], pidKeys[i], f, mn, node.Options( node.Override(new(sectorstorage.SectorManager), func() (sectorstorage.SectorManager, error) { - return mock.NewMockSectorMgr(sectors), nil + return mgr, nil + }), + node.Override(new(sectorstorage.PieceProvider), func() (sectorstorage.PieceProvider, error) { + return mgr, nil + }), + node.Override(new(sectorstorage.Unsealer), func() (sectorstorage.Unsealer, error) { + return mgr, nil }), node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), - node.Unset(new(*sectorstorage.Manager)), + //node.Unset(new(*sectorstorage.Manager)), opts, ))