cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

add tests and merge master

Browse Source
This commit is contained in:
aarshkshah1992
2021-07-15 17:50:35 +05:30
parent 33c347956c f07a370711
commit 35b39063ac
26 changed files with 2717 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node/impl/client/client.go
+32 -15
View File
@@ -473,25 +473,41 @@ func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (res *api.Impor
return nil, err
}
carV2File, err := a.imgr().NewTempFile(id)
if err != nil {
return nil, xerrors.Errorf("failed to create temp CARv2 file: %w", err)
}
// make sure to remove the CARv2 file if anything goes wrong from here on.
defer func() {
if finalErr != nil {
_ = os.Remove(carV2File)
}
}()
var root cid.Cid
var carFile string
if ref.IsCAR {
root, err = transformCarToCARv2(ref.Path, carV2File)
// if user has given us a CAR file -> just ensure it's either a v1 or a v2, has one root and save it as it is as markets can do deal making for both.
f, err := os.Open(ref.Path)
if err != nil {
return nil, xerrors.Errorf("failed to import CAR file: %w", err)
return nil, xerrors.Errorf("failed to open CAR file: %w", err)
}
defer f.Close() //nolint:errcheck
hd, _, err := car.ReadHeader(bufio.NewReader(f))
if err != nil {
return nil, xerrors.Errorf("failed to read CAR header: %w", err)
}
if len(hd.Roots) != 1 {
return nil, xerrors.New("car file can have one and only one header")
}
if hd.Version != 1 && hd.Version != 2 {
return nil, xerrors.Errorf("car version must be 1 or 2, is %d", hd.Version)
}
carFile = ref.Path
root = hd.Roots[0]
} else {
root, err = a.importNormalFileToFilestoreCARv2(ctx, id, ref.Path, carV2File)
carFile, err = a.imgr().NewTempFile(id)
if err != nil {
return nil, xerrors.Errorf("failed to create temp CARv2 file: %w", err)
}
// make sure to remove the CARv2 file if anything goes wrong from here on.
defer func() {
if finalErr != nil {
_ = os.Remove(carFile)
}
}()
root, err = a.importNormalFileToFilestoreCARv2(ctx, id, ref.Path, carFile)
if err != nil {
return nil, xerrors.Errorf("failed to import normal file to CARv2: %w", err)
}
@@ -503,7 +519,7 @@ func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (res *api.Impor
if err := a.imgr().AddLabel(id, importmgr.LFileName, ref.Path); err != nil {
return nil, err
}
if err := a.imgr().AddLabel(id, importmgr.LFileStoreCARv2FilePath, carV2File); err != nil {
if err := a.imgr().AddLabel(id, importmgr.LFileStoreCARv2FilePath, carFile); err != nil {
return nil, err
}
if err := a.imgr().AddLabel(id, importmgr.LRootCid, root.String()); err != nil {
@@ -854,6 +870,7 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
return
}
finish(files.WriteTo(file, ref.Path))
return
}
node/impl/client/import.go
-53
View File
@@ -1,9 +1,7 @@
package client
import (
"bufio"
"context"
"io"
"os"
"github.com/filecoin-project/go-fil-markets/filestorecaradapter"
@@ -23,8 +21,6 @@ import (
"github.com/ipfs/go-merkledag"
"github.com/ipfs/go-unixfs/importer/balanced"
ihelper "github.com/ipfs/go-unixfs/importer/helpers"
"github.com/ipld/go-car"
carv2 "github.com/ipld/go-car/v2"
"golang.org/x/xerrors"
)
@@ -124,52 +120,3 @@ func importNormalFileToUnixfsDAG(ctx context.Context, inputFilePath string, dag
return nd.Cid(), nil
}
// transformCarToCARv2 transforms a client's CAR file to a CARv2 file.
func transformCarToCARv2(inputCARPath string, outputCARv2Path string) (root cid.Cid, err error) {
inputF, err := os.Open(inputCARPath)
if err != nil {
return cid.Undef, xerrors.Errorf("failed to open input CAR: %w", err)
}
defer inputF.Close() //nolint:errcheck
// read the CAR header to determine the DAG root and the CAR version.
hd, _, err := car.ReadHeader(bufio.NewReader(inputF))
if err != nil {
return cid.Undef, xerrors.Errorf("failed to read CAR header: %w", err)
}
if len(hd.Roots) != 1 {
return cid.Undef, xerrors.New("cannot import CAR with more than one root")
}
// we read the file to read the header -> seek to the start again to be able to read again.
if _, err := inputF.Seek(0, io.SeekStart); err != nil {
return cid.Undef, xerrors.Errorf("failed to seek to start of input CAR: %w", err)
}
switch hd.Version {
case 2:
// This is a CARv2, we can import it as it is by simply copying it.
outF, err := os.Open(outputCARv2Path)
if err != nil {
return cid.Undef, xerrors.Errorf("failed to open output CARv2 file: %w", err)
}
defer outF.Close() //nolint:errcheck
_, err = io.Copy(outF, inputF)
if err != nil {
return cid.Undef, xerrors.Errorf("failed to copy CARv2 file: %w", err)
}
case 1:
// This is a CARv1, let's transform it to a CARv2.
if err := carv2.WrapV1File(inputCARPath, outputCARv2Path); err != nil {
return cid.Undef, xerrors.Errorf("failed to transform CARv1 to CARv2: %w", err)
}
default:
return cid.Undef, xerrors.Errorf("unrecognized CAR version %d", hd.Version)
}
return hd.Roots[0], nil
}
node/impl/client/import_test.go
-95
View File
@@ -1,7 +1,6 @@
package client
import (
"bufio"
"context"
"io"
"io/ioutil"
@@ -11,7 +10,6 @@ import (
"testing"
"github.com/filecoin-project/go-fil-markets/filestorecaradapter"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
@@ -19,12 +17,7 @@ import (
files "github.com/ipfs/go-ipfs-files"
"github.com/ipfs/go-merkledag"
unixfile "github.com/ipfs/go-unixfs/file"
"github.com/ipld/go-car"
carv2 "github.com/ipld/go-car/v2"
"github.com/ipld/go-car/v2/blockstore"
basicnode "github.com/ipld/go-ipld-prime/node/basic"
"github.com/ipld/go-ipld-prime/traversal/selector"
"github.com/ipld/go-ipld-prime/traversal/selector/builder"
"github.com/stretchr/testify/require"
)
@@ -110,94 +103,6 @@ func TestImportNormalFileToCARv2(t *testing.T) {
require.Equal(t, inputContents, bz2)
}
func TestTransformCarv1ToCARv2(t *testing.T) {
inputFilePath, _ := genNormalInputFile(t)
defer os.Remove(inputFilePath) //nolint:errcheck
carv1FilePath := genCARv1(t, inputFilePath)
defer os.Remove(carv1FilePath) //nolint:errcheck
outputCARv2 := genTmpFile(t)
defer os.Remove(outputCARv2) //nolint:errcheck
root, err := transformCarToCARv2(carv1FilePath, outputCARv2)
require.NoError(t, err)
require.NotEqual(t, cid.Undef, root)
// assert what we got back is a valid CARv2 and that the CARv1 payload is exactly what we gave it
f2, err := os.Open(outputCARv2)
require.NoError(t, err)
hd, _, err := car.ReadHeader(bufio.NewReader(f2))
require.NoError(t, err)
require.EqualValues(t, 2, hd.Version)
require.NoError(t, f2.Close())
v2r, err := carv2.NewReaderMmap(outputCARv2)
require.NoError(t, err)
bzout, err := ioutil.ReadAll(v2r.CarV1Reader())
require.NoError(t, err)
require.NotNil(t, bzout)
require.NoError(t, v2r.Close())
fi, err := os.Open(carv1FilePath)
require.NoError(t, err)
bzin, err := ioutil.ReadAll(fi)
require.NoError(t, err)
require.NoError(t, fi.Close())
require.NotNil(t, bzin)
require.Equal(t, bzin, bzout)
}
func TestLoadCARv2ToBlockstore(t *testing.T) {
inputFilePath, _ := genNormalInputFile(t)
defer os.Remove(inputFilePath) //nolint:errcheck
carv1FilePath := genCARv1(t, inputFilePath)
defer os.Remove(carv1FilePath) //nolint:errcheck
outputCARv2 := genTmpFile(t)
defer os.Remove(outputCARv2) //nolint:errcheck
root, err := transformCarToCARv2(carv1FilePath, outputCARv2)
require.NoError(t, err)
require.NotEqual(t, cid.Undef, root)
bs := bstore.NewMemorySync()
carv2, err := carv2.NewReaderMmap(outputCARv2)
require.NoError(t, err)
defer carv2.Close() //nolint:errcheck
header, err := car.LoadCar(bs, carv2.CarV1Reader())
require.NoError(t, err)
require.EqualValues(t, root, header.Roots[0])
require.EqualValues(t, 1, header.Version)
}
func genCARv1(t *testing.T, normalFilePath string) string {
ctx := context.Background()
bs := bstore.NewMemorySync()
root, err := importNormalFileToUnixfsDAG(ctx, normalFilePath, merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))))
require.NoError(t, err)
ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any)
allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(),
ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node()
sc := car.NewSelectiveCar(ctx, bs, []car.Dag{{Root: root, Selector: allSelector}})
f, err := os.CreateTemp("", "")
require.NoError(t, err)
require.NoError(t, sc.Write(f))
_, err = f.Seek(0, io.SeekStart)
require.NoError(t, err)
hd, _, err := car.ReadHeader(bufio.NewReader(f))
require.NoError(t, err)
require.EqualValues(t, 1, hd.Version)
require.NoError(t, f.Close())
return f.Name()
}
func genTmpFile(t *testing.T) string {
f, err := os.CreateTemp("", "")
require.NoError(t, err)
node/impl/full/chain.go
+27
View File
@@ -228,6 +228,33 @@ func (a *ChainAPI) ChainGetParentReceipts(ctx context.Context, bcid cid.Cid) ([]
return out, nil
}
func (a *ChainAPI) ChainGetMessagesInTipset(ctx context.Context, tsk types.TipSetKey) ([]api.Message, error) {
ts, err := a.Chain.GetTipSetFromKey(tsk)
if err != nil {
return nil, err
}
// genesis block has no parent messages...
if ts.Height() == 0 {
return nil, nil
}
cm, err := a.Chain.MessagesForTipset(ts)
if err != nil {
return nil, err
}
var out []api.Message
for _, m := range cm {
out = append(out, api.Message{
Cid: m.Cid(),
Message: m.VMMessage(),
})
}
return out, nil
}
func (m *ChainModule) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) {
ts, err := m.Chain.GetTipSetFromKey(tsk)
if err != nil {