lotus/itests/deals_anycid_test.go

163 lines
5.0 KiB
Go
Raw Normal View History

package itests
import (
2021-09-17 07:27:53 +00:00
"bufio"
"context"
"os"
"testing"
"time"
chore: migrate to boxo This migrates everything except the `go-car` librairy: https://github.com/ipfs/boxo/issues/218#issuecomment-1529922103 I didn't migrated everything in the previous release because all the boxo code wasn't compatible with the go-ipld-prime one due to a an in flight (/ aftermath) revert of github.com/ipfs/go-block-format. go-block-format has been unmigrated since slight bellow absolutely everything depends on it that would have required everything to be moved on boxo or everything to optin into using boxo which were all deal breakers for different groups. This worked fine because lotus's codebase could live hapely on the first multirepo setup however boost is now trying to use boxo's code with lotus's (still on multirepo) setup: https://filecoinproject.slack.com/archives/C03AQ3QAUG1/p1685022344779649 The alternative would be for boost to write shim types which just forward calls and return with the different interface definitions. Btw why is that an issue in the first place is because unlike what go's duck typing model suggest interfaces are not transparent https://github.com/golang/go/issues/58112, interfaces are strongly typed but they have implicit narrowing. The issue is if you return an interface from an interface Go does not have a function definition to insert the implicit conversion thus instead the type checker complains you are not returning the right type. Stubbing types were reverted https://github.com/ipfs/boxo/issues/218#issuecomment-1478650351 Last time I only migrated `go-bitswap` to `boxo/bitswap` because of the security issues and because we never had the interface return an interface problem (we had concrete wrappers where the implicit conversion took place).
2023-05-25 14:31:53 +00:00
dag "github.com/ipfs/boxo/ipld/merkledag"
"github.com/ipfs/go-cid"
ipldcbor "github.com/ipfs/go-ipld-cbor"
format "github.com/ipfs/go-ipld-format"
"github.com/ipld/go-car"
2021-09-17 13:32:29 +00:00
"github.com/ipld/go-car/v2/blockstore"
2022-06-14 15:00:51 +00:00
selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/go-fil-markets/storagemarket"
2022-06-14 15:00:51 +00:00
"github.com/filecoin-project/go-state-types/abi"
2021-09-17 13:32:29 +00:00
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/itests/kit"
2021-09-17 13:32:29 +00:00
"github.com/filecoin-project/lotus/node"
"github.com/filecoin-project/lotus/node/modules"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
func TestDealRetrieveByAnyCid(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode")
}
ctx := context.Background()
kit.QuietMiningLogs()
// For these tests where the block time is artificially short, just use
// a deal start epoch that is guaranteed to be far enough in the future
// so that the deal starts sealing in time
startEpoch := abi.ChainEpoch(2 << 12)
// Override the dependency injection for the blockstore accessor, so that
// we can get a reference to the blockstore containing our deal later in
// the test
var bsa storagemarket.BlockstoreAccessor
bsaFn := func(importmgr dtypes.ClientImportMgr) storagemarket.BlockstoreAccessor {
bsa = modules.StorageBlockstoreAccessor(importmgr)
return bsa
}
bsaOpt := kit.ConstructorOpts(node.Override(new(storagemarket.BlockstoreAccessor), bsaFn))
2021-09-17 13:32:29 +00:00
// Allow 8MB sectors
eightMBSectorsOpt := kit.SectorSize(8 << 20)
// Create a client, and a miner with its own full node
_, client, miner, ens := kit.EnsembleTwoOne(t, kit.MockProofs(), bsaOpt, eightMBSectorsOpt)
ens.InterconnectAll().BeginMining(250 * time.Millisecond)
dh := kit.NewDealHarness(t, client, miner, miner)
2021-09-17 13:32:29 +00:00
// Generate a DAG with multiple levels, so that we can test the case where
// the client requests a CID for a block which is not the root block but
// does have a subtree below it in the DAG
dagOpts := kit.GeneratedDAGOpts{
2021-09-17 13:32:29 +00:00
// Max size of a block
ChunkSize: 1024,
2021-09-17 13:32:29 +00:00
// Max links from a block to other blocks
Maxlinks: 10,
}
2021-09-17 07:27:53 +00:00
carv1FilePath, _ := kit.CreateRandomCARv1(t, 5, 100*1024, dagOpts)
res, err := client.ClientImport(ctx, api.FileRef{Path: carv1FilePath, IsCAR: true})
require.NoError(t, err)
// Get the blockstore for the file
bs, err := bsa.Get(res.Root)
require.NoError(t, err)
// Get all CIDs from the file
2021-11-17 11:25:25 +00:00
sc := car.NewSelectiveCar(ctx, bs, []car.Dag{{Root: res.Root, Selector: selectorparse.CommonSelector_ExploreAllRecursively}})
prepared, err := sc.Prepare()
require.NoError(t, err)
chore: migrate to boxo This migrates everything except the `go-car` librairy: https://github.com/ipfs/boxo/issues/218#issuecomment-1529922103 I didn't migrated everything in the previous release because all the boxo code wasn't compatible with the go-ipld-prime one due to a an in flight (/ aftermath) revert of github.com/ipfs/go-block-format. go-block-format has been unmigrated since slight bellow absolutely everything depends on it that would have required everything to be moved on boxo or everything to optin into using boxo which were all deal breakers for different groups. This worked fine because lotus's codebase could live hapely on the first multirepo setup however boost is now trying to use boxo's code with lotus's (still on multirepo) setup: https://filecoinproject.slack.com/archives/C03AQ3QAUG1/p1685022344779649 The alternative would be for boost to write shim types which just forward calls and return with the different interface definitions. Btw why is that an issue in the first place is because unlike what go's duck typing model suggest interfaces are not transparent https://github.com/golang/go/issues/58112, interfaces are strongly typed but they have implicit narrowing. The issue is if you return an interface from an interface Go does not have a function definition to insert the implicit conversion thus instead the type checker complains you are not returning the right type. Stubbing types were reverted https://github.com/ipfs/boxo/issues/218#issuecomment-1478650351 Last time I only migrated `go-bitswap` to `boxo/bitswap` because of the security issues and because we never had the interface return an interface problem (we had concrete wrappers where the implicit conversion took place).
2023-05-25 14:31:53 +00:00
reg := format.Registry{}
reg.Register(cid.DagProtobuf, dag.DecodeProtobufBlock)
reg.Register(cid.DagCBOR, ipldcbor.DecodeBlock)
reg.Register(cid.Raw, dag.DecodeRawBlock)
cids := prepared.Cids()
for i, c := range cids {
2022-01-12 11:53:15 +00:00
blk, err := bs.Get(ctx, c)
require.NoError(t, err)
chore: migrate to boxo This migrates everything except the `go-car` librairy: https://github.com/ipfs/boxo/issues/218#issuecomment-1529922103 I didn't migrated everything in the previous release because all the boxo code wasn't compatible with the go-ipld-prime one due to a an in flight (/ aftermath) revert of github.com/ipfs/go-block-format. go-block-format has been unmigrated since slight bellow absolutely everything depends on it that would have required everything to be moved on boxo or everything to optin into using boxo which were all deal breakers for different groups. This worked fine because lotus's codebase could live hapely on the first multirepo setup however boost is now trying to use boxo's code with lotus's (still on multirepo) setup: https://filecoinproject.slack.com/archives/C03AQ3QAUG1/p1685022344779649 The alternative would be for boost to write shim types which just forward calls and return with the different interface definitions. Btw why is that an issue in the first place is because unlike what go's duck typing model suggest interfaces are not transparent https://github.com/golang/go/issues/58112, interfaces are strongly typed but they have implicit narrowing. The issue is if you return an interface from an interface Go does not have a function definition to insert the implicit conversion thus instead the type checker complains you are not returning the right type. Stubbing types were reverted https://github.com/ipfs/boxo/issues/218#issuecomment-1478650351 Last time I only migrated `go-bitswap` to `boxo/bitswap` because of the security issues and because we never had the interface return an interface problem (we had concrete wrappers where the implicit conversion took place).
2023-05-25 14:31:53 +00:00
nd, err := reg.Decode(blk)
require.NoError(t, err)
2021-09-17 13:32:29 +00:00
t.Log(i, c, len(nd.Links()))
}
// Create a storage deal
dp := dh.DefaultStartDealParams()
dp.Data.Root = res.Root
dp.DealStartEpoch = startEpoch
dp.EpochPrice = abi.NewTokenAmount(62500000) // minimum asking price
dealCid := dh.StartDeal(ctx, dp)
// Wait for the deal to be sealed
dh.WaitDealSealed(ctx, dealCid, false, false, nil)
ask, err := miner.MarketGetRetrievalAsk(ctx)
require.NoError(t, err)
ask.PricePerByte = abi.NewTokenAmount(0)
ask.UnsealPrice = abi.NewTokenAmount(0)
err = miner.MarketSetRetrievalAsk(ctx, ask)
require.NoError(t, err)
// Fetch the deal data
info, err := client.ClientGetDealInfo(ctx, *dealCid)
require.NoError(t, err)
2021-09-17 13:32:29 +00:00
// Make retrievals against CIDs at different levels in the DAG
2021-09-17 07:27:53 +00:00
cidIndices := []int{1, 11, 27, 32, 47}
for _, val := range cidIndices {
2021-09-17 13:32:29 +00:00
t.Logf("performing retrieval for cid at index %d", val)
2021-09-17 08:52:31 +00:00
2021-09-17 07:27:53 +00:00
targetCid := cids[val]
offer, err := client.ClientMinerQueryOffer(ctx, miner.ActorAddr, targetCid, &info.PieceCID)
require.NoError(t, err)
require.Empty(t, offer.Err)
2021-09-17 08:52:31 +00:00
// retrieve in a CAR file and ensure roots match
2022-01-12 12:55:45 +00:00
outputCar := dh.PerformRetrieval(ctx, dealCid, targetCid, true, offer)
2021-09-17 08:52:31 +00:00
_, err = os.Stat(outputCar)
2021-09-17 07:27:53 +00:00
require.NoError(t, err)
2021-09-17 08:52:31 +00:00
f, err := os.Open(outputCar)
2021-09-17 07:27:53 +00:00
require.NoError(t, err)
2022-01-12 12:55:45 +00:00
ch, err := car.ReadHeader(bufio.NewReader(f))
require.NoError(t, err)
2021-09-17 07:27:53 +00:00
require.EqualValues(t, ch.Roots[0], targetCid)
2021-09-17 08:52:31 +00:00
require.NoError(t, f.Close())
// create CAR from original file starting at targetCid and ensure it matches the retrieved CAR file.
tmp, err := os.CreateTemp(t.TempDir(), "randcarv1")
require.NoError(t, err)
rd, err := blockstore.OpenReadOnly(carv1FilePath, blockstore.UseWholeCIDs(true))
require.NoError(t, err)
err = car.NewSelectiveCar(
ctx,
rd,
[]car.Dag{{
Root: targetCid,
2021-11-17 11:25:25 +00:00
Selector: selectorparse.CommonSelector_ExploreAllRecursively,
2021-09-17 08:52:31 +00:00
}},
).Write(tmp)
2021-09-20 06:04:21 +00:00
require.NoError(t, err)
2021-09-17 08:52:31 +00:00
require.NoError(t, tmp.Close())
require.NoError(t, rd.Close())
2021-09-17 13:32:29 +00:00
kit.AssertFilesEqual(t, tmp.Name(), outputCar)
t.Log("car files match")
2021-09-17 07:27:53 +00:00
}
}