cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

refactor dagstore + carv2 integration (+).

Browse Source 
- Integrate with config.toml.
- Export DAGStore to DI context in preparation for JSON-RPC
  and cli commands.
- Renames.
This commit is contained in:
Raúl Kripalani
2021-08-03 23:09:28 +01:00
parent 691da14996
commit c06c8541f9
17 changed files with 335 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node/impl/client/client.go
+33 -34
View File
@@ -10,8 +10,7 @@ import (
"sort"
"time"
"github.com/filecoin-project/go-fil-markets/filestorecaradapter"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/ipld/go-car"
carv2 "github.com/ipld/go-car/v2"
"github.com/ipld/go-car/v2/blockstore"
@@ -26,7 +25,7 @@ import (
files "github.com/ipfs/go-ipfs-files"
"github.com/ipfs/go-merkledag"
unixfile "github.com/ipfs/go-unixfs/file"
"github.com/ipld/go-car"
basicnode "github.com/ipld/go-ipld-prime/node/basic"
"github.com/ipld/go-ipld-prime/traversal/selector"
"github.com/ipld/go-ipld-prime/traversal/selector/builder"
@@ -47,6 +46,7 @@ import (
"github.com/filecoin-project/go-fil-markets/shared"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-fil-markets/storagemarket/network"
"github.com/filecoin-project/go-fil-markets/stores"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/specs-actors/v3/actors/builtin/market"
@@ -54,6 +54,7 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/markets/utils"
@@ -188,17 +189,17 @@ func (a *API) dealStarter(ctx context.Context, params *api.StartDealParams, isSt
providerInfo := utils.NewStorageProviderInfo(params.Miner, mi.Worker, mi.SectorSize, *mi.PeerId, mi.Multiaddrs)
result, err := a.SMDealClient.ProposeStorageDeal(ctx, storagemarket.ProposeStorageDealParams{
Addr: params.Wallet,
Info: &providerInfo,
Data: params.Data,
StartEpoch: dealStart,
EndEpoch: calcDealExpiration(params.MinBlocksDuration, md, dealStart),
Price: params.EpochPrice,
Collateral: params.ProviderCollateral,
Rt: st,
FastRetrieval: params.FastRetrieval,
VerifiedDeal: params.VerifiedDeal,
FilestoreCARv2FilePath: CARV2FilePath,
Addr: params.Wallet,
Info: &providerInfo,
Data: params.Data,
StartEpoch: dealStart,
EndEpoch: calcDealExpiration(params.MinBlocksDuration, md, dealStart),
Price: params.EpochPrice,
Collateral: params.ProviderCollateral,
Rt: st,
FastRetrieval: params.FastRetrieval,
VerifiedDeal: params.VerifiedDeal,
IndexedCAR: CARV2FilePath,
})
if err != nil {
@@ -519,7 +520,7 @@ func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (res *api.Impor
}
}()
root, err = a.importNormalFileToFilestoreCARv2(ctx, id, ref.Path, carFile)
root, err = a.doImport(ctx, ref.Path, carFile)
if err != nil {
return nil, xerrors.Errorf("failed to import normal file to CARv2: %w", err)
}
@@ -1031,21 +1032,21 @@ func (w *lenWriter) Write(p []byte) (n int, err error) {
}
func (a *API) ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, error) {
fc, err := a.imgr().FilestoreCARV2FilePathFor(root)
path, err := a.imgr().FilestoreCARV2FilePathFor(root)
if err != nil {
return api.DataSize{}, xerrors.Errorf("failed to find CARv2 file for root: %w", err)
}
if len(fc) == 0 {
if len(path) == 0 {
return api.DataSize{}, xerrors.New("no CARv2 file for root")
}
rdOnly, err := filestorecaradapter.NewReadOnlyFileStore(fc)
fs, err := stores.ReadOnlyFilestore(path)
if err != nil {
return api.DataSize{}, xerrors.Errorf("failed to open read only filestore: %w", err)
return api.DataSize{}, xerrors.Errorf("failed to open filestore from carv2 in path %s: %w", path, err)
}
defer rdOnly.Close() //nolint:errcheck
defer fs.Close() //nolint:errcheck
dag := merkledag.NewDAGService(blockservice.New(rdOnly, offline.Exchange(rdOnly)))
dag := merkledag.NewDAGService(blockservice.New(fs, offline.Exchange(fs)))
w := lenWriter(0)
err = car.WriteCar(ctx, dag, []cid.Cid{root}, &w)
@@ -1062,21 +1063,21 @@ func (a *API) ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, e
}
func (a *API) ClientDealPieceCID(ctx context.Context, root cid.Cid) (api.DataCIDSize, error) {
fc, err := a.imgr().FilestoreCARV2FilePathFor(root)
path, err := a.imgr().FilestoreCARV2FilePathFor(root)
if err != nil {
return api.DataCIDSize{}, xerrors.Errorf("failed to find CARv2 file for root: %w", err)
}
if len(fc) == 0 {
if len(path) == 0 {
return api.DataCIDSize{}, xerrors.New("no CARv2 file for root")
}
rdOnly, err := filestorecaradapter.NewReadOnlyFileStore(fc)
fs, err := stores.ReadOnlyFilestore(path)
if err != nil {
return api.DataCIDSize{}, xerrors.Errorf("failed to open read only blockstore: %w", err)
return api.DataCIDSize{}, xerrors.Errorf("failed to open filestore from carv2 in path %s: %w", path, err)
}
defer rdOnly.Close() //nolint:errcheck
defer fs.Close() //nolint:errcheck
dag := merkledag.NewDAGService(blockservice.New(rdOnly, offline.Exchange(rdOnly)))
dag := merkledag.NewDAGService(blockservice.New(fs, offline.Exchange(fs)))
w := &writer.Writer{}
bw := bufio.NewWriterSize(w, int(writer.CommPBuf))
@@ -1095,22 +1096,20 @@ func (a *API) ClientDealPieceCID(ctx context.Context, root cid.Cid) (api.DataCID
func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath string) error {
id := importmgr.ImportID(rand.Uint64())
tmpCARv2File, err := a.imgr().NewTempFile(id)
tmp, err := a.imgr().NewTempFile(id)
if err != nil {
return xerrors.Errorf("failed to create temp file: %w", err)
}
defer os.Remove(tmpCARv2File) //nolint:errcheck
defer os.Remove(tmp) //nolint:errcheck
root, err := a.importNormalFileToFilestoreCARv2(ctx, id, ref.Path, tmpCARv2File)
root, err := a.doImport(ctx, ref.Path, tmp)
if err != nil {
return xerrors.Errorf("failed to import normal file to CARv2")
}
// generate a deterministic CARv1 payload from the UnixFS DAG by doing an IPLD
// traversal over the Unixfs DAG in the CARv2 file using the "all selector" i.e the entire DAG selector.
fs, err := filestorecaradapter.NewReadOnlyFileStore(tmpCARv2File)
fs, err := stores.ReadOnlyFilestore(tmp)
if err != nil {
return xerrors.Errorf("failed to open read only CARv2 blockstore: %w", err)
return xerrors.Errorf("failed to open filestore from carv2 in path %s: %w", tmp, err)
}
defer fs.Close() //nolint:errcheck
node/impl/client/import.go
+61 -40
View File
@@ -2,18 +2,14 @@ package client
import (
"context"
"io/ioutil"
"os"
"github.com/filecoin-project/go-fil-markets/filestorecaradapter"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/go-fil-markets/stores"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-cidutil"
"github.com/ipfs/go-datastore"
ds_sync "github.com/ipfs/go-datastore/sync"
"github.com/ipfs/go-filestore"
chunker "github.com/ipfs/go-ipfs-chunker"
offline "github.com/ipfs/go-ipfs-exchange-offline"
files2 "github.com/ipfs/go-ipfs-files"
@@ -21,56 +17,81 @@ import (
"github.com/ipfs/go-merkledag"
"github.com/ipfs/go-unixfs/importer/balanced"
ihelper "github.com/ipfs/go-unixfs/importer/helpers"
"github.com/ipld/go-car/v2"
"github.com/ipld/go-car/v2/blockstore"
"golang.org/x/xerrors"
)
// importNormalFileToFilestoreCARv2 transforms the client's "normal file" to a Unixfs IPLD DAG and writes out the DAG to a CARv2 file
// that can be used to back a filestore.
func (a *API) importNormalFileToFilestoreCARv2(ctx context.Context, importID importmgr.ImportID, inputFilePath string, outputCARv2Path string) (c cid.Cid, finalErr error) {
// TODO: We've currently put in a hack to create the Unixfs DAG as a CARv2 without using Badger.
// We first create the Unixfs DAG using a filestore to get the root of the Unixfs DAG.
// We can't create the UnixfsDAG right away using a CARv2 read-write blockstore as the blockstore
// needs the root of the DAG during instantiation to write out a valid CARv2 file.
// doImport takes a standard file (src), forms a UnixFS DAG, and writes a
// CARv2 file with positional mapping (backed by the go-filestore library).
func (a *API) doImport(ctx context.Context, src string, dst string) (cid.Cid, error) {
// This method uses a two-phase approach with a staging CAR blockstore and
// a final CAR blockstore.
//
// In the second pass, we create a CARv2 file with the root present using the root node we get in the above step.
// This hack should be fixed when CARv2 allows specifying the root AFTER finishing the CARv2 streaming write.
fm := filestore.NewFileManager(ds_sync.MutexWrap(datastore.NewMapDatastore()), "/")
fm.AllowFiles = true
fstore := filestore.NewFilestore(bstore.NewMemorySync(), fm)
// This is necessary because of https://github.com/ipld/go-car/issues/196
//
// TODO: do we need to chunk twice? Isn't the first output already in the
// right order? Can't we just copy the CAR file and replace the header?
f, err := ioutil.TempFile("", "")
if err != nil {
return cid.Undef, xerrors.Errorf("failed to create temp file: %w", err)
}
_ = f.Close() // close; we only want the path.
tmp := f.Name()
defer os.Remove(tmp)
// Step 1. Compute the UnixFS DAG and write it to a CARv2 file to get
// the root CID of the DAG.
fstore, err := stores.ReadWriteFilestore(tmp)
if err != nil {
return cid.Undef, xerrors.Errorf("failed to create temporary filestore: %w", err)
}
bsvc := blockservice.New(fstore, offline.Exchange(fstore))
defer bsvc.Close() //nolint:errcheck
dags := merkledag.NewDAGService(bsvc)
// ---- First Pass --- Write out the UnixFS DAG to a rootless CARv2 file by instantiating a read-write CARv2 blockstore without the root.
root, err := importNormalFileToUnixfsDAG(ctx, inputFilePath, merkledag.NewDAGService(bsvc))
root, err := buildUnixFS(ctx, src, dags)
if err != nil {
return cid.Undef, xerrors.Errorf("failed to import file to store: %w", err)
_ = fstore.Close()
return cid.Undef, xerrors.Errorf("failed to import file to store to compute root: %w", err)
}
//------ Second Pass --- Now that we have the root of the Unixfs DAG -> write out the Unixfs DAG to a CARv2 file with the root present by using a
// filestore backed by a read-write CARv2 blockstore.
fsb, err := filestorecaradapter.NewReadWriteFileStore(outputCARv2Path, []cid.Cid{root})
if err != nil {
return cid.Undef, xerrors.Errorf("failed to create a CARv2 read-write blockstore: %w", err)
}
defer fsb.Close() //nolint:errcheck
bsvc = blockservice.New(fsb, offline.Exchange(fsb))
root2, err := importNormalFileToUnixfsDAG(ctx, inputFilePath, merkledag.NewDAGService(bsvc))
if err != nil {
return cid.Undef, xerrors.Errorf("failed to create Unixfs DAG with CARv2 blockstore: %w", err)
if err := fstore.Close(); err != nil {
return cid.Undef, xerrors.Errorf("failed to finalize car filestore: %w", err)
}
if root != root2 {
// Step 2. We now have the root of the UnixFS DAG, and we can write the
// final CAR for real under `dst`.
bs, err := blockstore.OpenReadWrite(dst, []cid.Cid{root},
car.ZeroLengthSectionAsEOF(true),
blockstore.UseWholeCIDs(true))
if err != nil {
return cid.Undef, xerrors.Errorf("failed to create a carv2 read/write blockstore: %w", err)
}
bsvc = blockservice.New(bs, offline.Exchange(bs))
dags = merkledag.NewDAGService(bsvc)
finalRoot, err := buildUnixFS(ctx, src, dags)
if err != nil {
_ = bs.Close()
return cid.Undef, xerrors.Errorf("failed to create UnixFS DAG with carv2 blockstore: %w", err)
}
if err := bs.Finalize(); err != nil {
return cid.Undef, xerrors.Errorf("failed to finalize car blockstore: %w", err)
}
if root != finalRoot {
return cid.Undef, xerrors.New("roots do not match")
}
return root, nil
}
// importNormalFileToUnixfsDAG transforms a client's normal file to a UnixfsDAG and imports the DAG to the given DAG service.
func importNormalFileToUnixfsDAG(ctx context.Context, inputFilePath string, dag ipld.DAGService) (cid.Cid, error) {
f, err := os.Open(inputFilePath)
// buildUnixFS builds a UnixFS DAG out of the supplied ordinary file,
// and imports the DAG into the supplied service.
func buildUnixFS(ctx context.Context, src string, dag ipld.DAGService) (cid.Cid, error) {
f, err := os.Open(src)
if err != nil {
return cid.Undef, xerrors.Errorf("failed to open input file: %w", err)
}
@@ -81,7 +102,7 @@ func importNormalFileToUnixfsDAG(ctx context.Context, inputFilePath string, dag
return cid.Undef, xerrors.Errorf("failed to stat file :%w", err)
}
file, err := files2.NewReaderPathFile(inputFilePath, f, stat)
file, err := files2.NewReaderPathFile(src, f, stat)
if err != nil {
return cid.Undef, xerrors.Errorf("failed to create reader path file: %w", err)
}
node/impl/client/import_test.go
+44 -33
View File
@@ -4,12 +4,11 @@ import (
"context"
"io"
"io/ioutil"
"math/rand"
"os"
"strings"
"testing"
"github.com/filecoin-project/go-fil-markets/filestorecaradapter"
"github.com/filecoin-project/go-fil-markets/stores"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
@@ -22,39 +21,50 @@ import (
"github.com/stretchr/testify/require"
)
func TestImportNormalFileToUnixfsDAG(t *testing.T) {
// This test uses a full "dense" CARv2, and not a filestore (positional mapping).
func TestRoundtripUnixFS_Dense(t *testing.T) {
ctx := context.Background()
inputPath, inputContents := genNormalInputFile(t)
inputPath, inputContents := genInputFile(t)
defer os.Remove(inputPath) //nolint:errcheck
carv2File := genTmpFile(t)
carv2File := newTmpFile(t)
defer os.Remove(carv2File) //nolint:errcheck
// import a normal file to a Unixfs DAG using a CARv2 read-write blockstore and flush it out to a CARv2 file.
tempCARv2Store, err := blockstore.OpenReadWrite(carv2File, []cid.Cid{}, carv2.ZeroLengthSectionAsEOF(true), blockstore.UseWholeCIDs(true))
// import a file to a Unixfs DAG using a CARv2 read/write blockstore.
path, err := blockstore.OpenReadWrite(carv2File, nil,
carv2.ZeroLengthSectionAsEOF(true),
blockstore.UseWholeCIDs(true))
require.NoError(t, err)
bsvc := blockservice.New(tempCARv2Store, offline.Exchange(tempCARv2Store))
root, err := importNormalFileToUnixfsDAG(ctx, inputPath, merkledag.NewDAGService(bsvc))
bsvc := blockservice.New(path, offline.Exchange(path))
dags := merkledag.NewDAGService(bsvc)
root, err := buildUnixFS(ctx, inputPath, dags)
require.NoError(t, err)
require.NotEqual(t, cid.Undef, root)
require.NoError(t, tempCARv2Store.Finalize())
require.NoError(t, path.Finalize())
// convert the CARv2 file to a normal file again and ensure the contents match.
readOnly, err := blockstore.OpenReadOnly(carv2File, carv2.ZeroLengthSectionAsEOF(true), blockstore.UseWholeCIDs(true))
// reconstruct the file.
readOnly, err := blockstore.OpenReadOnly(carv2File,
carv2.ZeroLengthSectionAsEOF(true),
blockstore.UseWholeCIDs(true))
require.NoError(t, err)
defer readOnly.Close() //nolint:errcheck
dag := merkledag.NewDAGService(blockservice.New(readOnly, offline.Exchange(readOnly)))
nd, err := dag.Get(ctx, root)
require.NoError(t, err)
file, err := unixfile.NewUnixfsFile(ctx, dag, nd)
dags = merkledag.NewDAGService(blockservice.New(readOnly, offline.Exchange(readOnly)))
nd, err := dags.Get(ctx, root)
require.NoError(t, err)
tmpOutput := genTmpFile(t)
file, err := unixfile.NewUnixfsFile(ctx, dags, nd)
require.NoError(t, err)
tmpOutput := newTmpFile(t)
defer os.Remove(tmpOutput) //nolint:errcheck
require.NoError(t, files.WriteTo(file, tmpOutput))
// ensure contents of the initial input file and the output file are identical.
fo, err := os.Open(tmpOutput)
require.NoError(t, err)
bz2, err := ioutil.ReadAll(fo)
@@ -63,35 +73,36 @@ func TestImportNormalFileToUnixfsDAG(t *testing.T) {
require.Equal(t, inputContents, bz2)
}
func TestImportNormalFileToCARv2(t *testing.T) {
func TestRoundtripUnixFS_Filestore(t *testing.T) {
ctx := context.Background()
a := &API{
Imports: &importmgr.Mgr{},
}
importID := importmgr.ImportID(rand.Uint64())
inputFilePath, inputContents := genNormalInputFile(t)
inputFilePath, inputContents := genInputFile(t)
defer os.Remove(inputFilePath) //nolint:errcheck
outputCARv2 := genTmpFile(t)
defer os.Remove(outputCARv2) //nolint:errcheck
path := newTmpFile(t)
defer os.Remove(path) //nolint:errcheck
root, err := a.importNormalFileToFilestoreCARv2(ctx, importID, inputFilePath, outputCARv2)
root, err := a.doImport(ctx, inputFilePath, path)
require.NoError(t, err)
require.NotEqual(t, cid.Undef, root)
// convert the CARv2 to a normal file again and ensure the contents match
readOnly, err := filestorecaradapter.NewReadOnlyFileStore(outputCARv2)
fs, err := stores.ReadOnlyFilestore(path)
require.NoError(t, err)
defer readOnly.Close() //nolint:errcheck
dag := merkledag.NewDAGService(blockservice.New(readOnly, offline.Exchange(readOnly)))
defer fs.Close() //nolint:errcheck
nd, err := dag.Get(ctx, root)
require.NoError(t, err)
file, err := unixfile.NewUnixfsFile(ctx, dag, nd)
dags := merkledag.NewDAGService(blockservice.New(fs, offline.Exchange(fs)))
nd, err := dags.Get(ctx, root)
require.NoError(t, err)
tmpOutput := genTmpFile(t)
file, err := unixfile.NewUnixfsFile(ctx, dags, nd)
require.NoError(t, err)
tmpOutput := newTmpFile(t)
defer os.Remove(tmpOutput) //nolint:errcheck
require.NoError(t, files.WriteTo(file, tmpOutput))
@@ -104,14 +115,14 @@ func TestImportNormalFileToCARv2(t *testing.T) {
require.Equal(t, inputContents, bz2)
}
func genTmpFile(t *testing.T) string {
func newTmpFile(t *testing.T) string {
f, err := os.CreateTemp("", "")
require.NoError(t, err)
require.NoError(t, f.Close())
return f.Name()
}
func genNormalInputFile(t *testing.T) (filepath string, contents []byte) {
func genInputFile(t *testing.T) (filepath string, contents []byte) {
s := strings.Repeat("abcde", 100)
tmp, err := os.CreateTemp("", "")
require.NoError(t, err)