replace badger in storage client
This commit is contained in:
parent
872843e491
commit
46b78406b1
5
go.mod
5
go.mod
@ -26,7 +26,6 @@ require (
|
|||||||
github.com/elastic/gosigar v0.12.0
|
github.com/elastic/gosigar v0.12.0
|
||||||
github.com/etclabscore/go-openrpc-reflect v0.0.36
|
github.com/etclabscore/go-openrpc-reflect v0.0.36
|
||||||
github.com/fatih/color v1.9.0
|
github.com/fatih/color v1.9.0
|
||||||
github.com/filecoin-project/dagstore v0.0.0-20210708130647-e413e3ad83df
|
|
||||||
github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200910194244-f640612a1a1f
|
github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200910194244-f640612a1a1f
|
||||||
github.com/filecoin-project/go-address v0.0.5
|
github.com/filecoin-project/go-address v0.0.5
|
||||||
github.com/filecoin-project/go-bitfield v0.2.4
|
github.com/filecoin-project/go-bitfield v0.2.4
|
||||||
@ -35,7 +34,7 @@ require (
|
|||||||
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
|
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
|
||||||
github.com/filecoin-project/go-data-transfer v1.7.0
|
github.com/filecoin-project/go-data-transfer v1.7.0
|
||||||
github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a
|
github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a
|
||||||
github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210708144500-db723d7ea4ff
|
github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210715083804-52c35483392f
|
||||||
github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec
|
github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec
|
||||||
github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20
|
github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20
|
||||||
github.com/filecoin-project/go-paramfetch v0.0.2-0.20210614165157-25a6c7769498
|
github.com/filecoin-project/go-paramfetch v0.0.2-0.20210614165157-25a6c7769498
|
||||||
@ -96,7 +95,7 @@ require (
|
|||||||
github.com/ipfs/go-unixfs v0.2.6
|
github.com/ipfs/go-unixfs v0.2.6
|
||||||
github.com/ipfs/interface-go-ipfs-core v0.2.3
|
github.com/ipfs/interface-go-ipfs-core v0.2.3
|
||||||
github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d
|
github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d
|
||||||
github.com/ipld/go-car/v2 v2.0.0-20210708104948-d79de78d9567
|
github.com/ipld/go-car/v2 v2.0.0-20210715074855-81a9ae6dff48
|
||||||
github.com/ipld/go-ipld-prime v0.5.1-0.20201021195245-109253e8a018
|
github.com/ipld/go-ipld-prime v0.5.1-0.20201021195245-109253e8a018
|
||||||
github.com/kelseyhightower/envconfig v1.4.0
|
github.com/kelseyhightower/envconfig v1.4.0
|
||||||
github.com/lib/pq v1.7.0
|
github.com/lib/pq v1.7.0
|
||||||
|
13
go.sum
13
go.sum
@ -256,8 +256,8 @@ github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGj
|
|||||||
github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
|
github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
|
||||||
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
|
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
|
||||||
github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E=
|
github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E=
|
||||||
github.com/filecoin-project/dagstore v0.0.0-20210708130647-e413e3ad83df h1:S35PjZ9zJ/N/Oy6UshqCu8aRZujrKOwmkCtAgQaHBCU=
|
github.com/filecoin-project/dagstore v0.1.0 h1:lENA+8LlO2TtGBTP2MzZGF3kmjmzE9hB7hZ+bDGsnPY=
|
||||||
github.com/filecoin-project/dagstore v0.0.0-20210708130647-e413e3ad83df/go.mod h1:Qpv2Ka8Wg0iktm7cfcejJPG5hSjsKrhTy6LveOxXYYs=
|
github.com/filecoin-project/dagstore v0.1.0/go.mod h1:cqqORk5fbkKVwwZkFk3D7XfeLpsTbWkX5Uj1GrsBOmM=
|
||||||
github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8=
|
github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8=
|
||||||
github.com/filecoin-project/go-address v0.0.5 h1:SSaFT/5aLfPXycUlFyemoHYhRgdyXClXCyDdNJKPlDM=
|
github.com/filecoin-project/go-address v0.0.5 h1:SSaFT/5aLfPXycUlFyemoHYhRgdyXClXCyDdNJKPlDM=
|
||||||
github.com/filecoin-project/go-address v0.0.5/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8=
|
github.com/filecoin-project/go-address v0.0.5/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8=
|
||||||
@ -287,8 +287,8 @@ github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go
|
|||||||
github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a h1:hyJ+pUm/4U4RdEZBlg6k8Ma4rDiuvqyGpoICXAxwsTg=
|
github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a h1:hyJ+pUm/4U4RdEZBlg6k8Ma4rDiuvqyGpoICXAxwsTg=
|
||||||
github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
|
github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
|
||||||
github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c=
|
github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c=
|
||||||
github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210708144500-db723d7ea4ff h1:lRNlyyDMYbrkBBCq4BGtLmDpTllwuDCtbH+aaS/i9vs=
|
github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210715083804-52c35483392f h1:7DtJkZZwGZ6Cd8cNjemaFRyZ6MdnchR74kq0db8f1JE=
|
||||||
github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210708144500-db723d7ea4ff/go.mod h1:llRlmiWK9yCWVf0pKS13XF3oNUUzkmbX5ie0a72vKzw=
|
github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210715083804-52c35483392f/go.mod h1:+PSCPfW6bC+iAavOuVfoz3vQ7uhQJTlg+KKO+UuVxjw=
|
||||||
github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM=
|
github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM=
|
||||||
github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24=
|
github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24=
|
||||||
github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM=
|
github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM=
|
||||||
@ -750,8 +750,9 @@ github.com/ipld/go-car v0.1.0/go.mod h1:RCWzaUh2i4mOEkB3W45Vc+9jnS/M6Qay5ooytiBH
|
|||||||
github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4/go.mod h1:xrMEcuSq+D1vEwl+YAXsg/JfA98XGpXDwnkIL4Aimqw=
|
github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4/go.mod h1:xrMEcuSq+D1vEwl+YAXsg/JfA98XGpXDwnkIL4Aimqw=
|
||||||
github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d h1:iphSzTuPqyDgH7WUVZsdqUnQNzYgIblsVr1zhVNA33U=
|
github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d h1:iphSzTuPqyDgH7WUVZsdqUnQNzYgIblsVr1zhVNA33U=
|
||||||
github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d/go.mod h1:2Gys8L8MJ6zkh1gktTSXreY63t4UbyvNp5JaudTyxHQ=
|
github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d/go.mod h1:2Gys8L8MJ6zkh1gktTSXreY63t4UbyvNp5JaudTyxHQ=
|
||||||
github.com/ipld/go-car/v2 v2.0.0-20210708104948-d79de78d9567 h1:81/TGRHgkIbF7xSNF95jmR8nkILx6DBn/69ujuquO8Q=
|
|
||||||
github.com/ipld/go-car/v2 v2.0.0-20210708104948-d79de78d9567/go.mod h1:Ueq4zx/SNx7yHwmfr9xKlKpXxRCMM6wyqC8B0rv9oig=
|
github.com/ipld/go-car/v2 v2.0.0-20210708104948-d79de78d9567/go.mod h1:Ueq4zx/SNx7yHwmfr9xKlKpXxRCMM6wyqC8B0rv9oig=
|
||||||
|
github.com/ipld/go-car/v2 v2.0.0-20210715074855-81a9ae6dff48 h1:0Md1uGa6PLCPCwC4BiAgwWjYVCV5pQ+fgHETixmI52g=
|
||||||
|
github.com/ipld/go-car/v2 v2.0.0-20210715074855-81a9ae6dff48/go.mod h1:0nAH3QhJOua+Dz6SkD6zOYtoZMNCJSDHY4IrbYe3AQs=
|
||||||
github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w=
|
github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w=
|
||||||
github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e/go.mod h1:uVIwe/u0H4VdKv3kaN1ck7uCb6yD9cFLS9/ELyXbsw8=
|
github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e/go.mod h1:uVIwe/u0H4VdKv3kaN1ck7uCb6yD9cFLS9/ELyXbsw8=
|
||||||
github.com/ipld/go-ipld-prime v0.5.1-0.20200828233916-988837377a7f/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM=
|
github.com/ipld/go-ipld-prime v0.5.1-0.20200828233916-988837377a7f/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM=
|
||||||
@ -1280,6 +1281,8 @@ github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/g
|
|||||||
github.com/multiformats/go-multibase v0.0.2/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
|
github.com/multiformats/go-multibase v0.0.2/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
|
||||||
github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk=
|
github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk=
|
||||||
github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
|
github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
|
||||||
|
github.com/multiformats/go-multicodec v0.2.1-0.20210713081508-b421db6850ae h1:wfljHPpiR0UDOjeqld9ds0Zxl3Nt/j+0wnvyBc01JgY=
|
||||||
|
github.com/multiformats/go-multicodec v0.2.1-0.20210713081508-b421db6850ae/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ=
|
||||||
github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I=
|
github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I=
|
||||||
github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
|
github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
|
||||||
github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
|
github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
|
||||||
|
@ -7,8 +7,6 @@ import (
|
|||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"golang.org/x/xerrors"
|
"golang.org/x/xerrors"
|
||||||
|
|
||||||
"github.com/filecoin-project/dagstore"
|
|
||||||
"github.com/filecoin-project/dagstore/mount"
|
|
||||||
mktdagstore "github.com/filecoin-project/go-fil-markets/dagstore"
|
mktdagstore "github.com/filecoin-project/go-fil-markets/dagstore"
|
||||||
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
|
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
|
||||||
rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network"
|
rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network"
|
||||||
@ -149,8 +147,6 @@ func ConfigStorageMiner(c interface{}) Option {
|
|||||||
Override(new(dtypes.RetrievalPricingFunc), modules.RetrievalPricingFunc(cfg.Dealmaking)),
|
Override(new(dtypes.RetrievalPricingFunc), modules.RetrievalPricingFunc(cfg.Dealmaking)),
|
||||||
|
|
||||||
// DAG Store
|
// DAG Store
|
||||||
Override(new(*mount.Registry), modules.DAGStoreRegistry),
|
|
||||||
Override(new(*dagstore.DAGStore), modules.DAGStore),
|
|
||||||
Override(new(mktdagstore.DagStoreWrapper), modules.DAGStoreWrapper),
|
Override(new(mktdagstore.DagStoreWrapper), modules.DAGStoreWrapper),
|
||||||
|
|
||||||
// Markets (retrieval)
|
// Markets (retrieval)
|
||||||
|
@ -10,6 +10,7 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/filecoin-project/go-fil-markets/filestorecaradapter"
|
||||||
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
|
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
|
||||||
carv2 "github.com/ipld/go-car/v2"
|
carv2 "github.com/ipld/go-car/v2"
|
||||||
"github.com/ipld/go-car/v2/blockstore"
|
"github.com/ipld/go-car/v2/blockstore"
|
||||||
@ -124,14 +125,14 @@ func (a *API) dealStarter(ctx context.Context, params *api.StartDealParams, isSt
|
|||||||
return nil, xerrors.New("stateless storage deals can only be initiated with storage price of 0")
|
return nil, xerrors.New("stateless storage deals can only be initiated with storage price of 0")
|
||||||
}
|
}
|
||||||
} else if params.Data.TransferType == storagemarket.TTGraphsync {
|
} else if params.Data.TransferType == storagemarket.TTGraphsync {
|
||||||
c, err := a.imgr().CARV2FilePathFor(params.Data.Root)
|
fc, err := a.imgr().FilestoreCARV2FilePathFor(params.Data.Root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, xerrors.Errorf("failed to find CARv2 file path: %w", err)
|
return nil, xerrors.Errorf("failed to find CARv2 file path: %w", err)
|
||||||
}
|
}
|
||||||
if c == "" {
|
if fc == "" {
|
||||||
return nil, xerrors.New("no CARv2 file path for deal")
|
return nil, xerrors.New("no CARv2 file path for deal")
|
||||||
}
|
}
|
||||||
CARV2FilePath = c
|
CARV2FilePath = fc
|
||||||
}
|
}
|
||||||
|
|
||||||
walletKey, err := a.StateAccountKey(ctx, params.Wallet, types.EmptyTSK)
|
walletKey, err := a.StateAccountKey(ctx, params.Wallet, types.EmptyTSK)
|
||||||
@ -197,7 +198,7 @@ func (a *API) dealStarter(ctx context.Context, params *api.StartDealParams, isSt
|
|||||||
Rt: st,
|
Rt: st,
|
||||||
FastRetrieval: params.FastRetrieval,
|
FastRetrieval: params.FastRetrieval,
|
||||||
VerifiedDeal: params.VerifiedDeal,
|
VerifiedDeal: params.VerifiedDeal,
|
||||||
CARV2FilePath: CARV2FilePath,
|
FilestoreCARv2FilePath: CARV2FilePath,
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -398,11 +399,11 @@ func (a *API) newDealInfoWithTransfer(transferCh *api.DataTransferChannel, v sto
|
|||||||
|
|
||||||
func (a *API) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) {
|
func (a *API) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) {
|
||||||
// TODO: check if we have the ENTIRE dag
|
// TODO: check if we have the ENTIRE dag
|
||||||
carv2Path, err := a.imgr().CARV2FilePathFor(root)
|
fc, err := a.imgr().FilestoreCARV2FilePathFor(root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
if len(carv2Path) == 0 {
|
if len(fc) == 0 {
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
return true, nil
|
return true, nil
|
||||||
@ -490,7 +491,7 @@ func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (res *api.Impor
|
|||||||
return nil, xerrors.Errorf("failed to import CAR file: %w", err)
|
return nil, xerrors.Errorf("failed to import CAR file: %w", err)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
root, err = a.importNormalFileToCARv2(ctx, id, ref.Path, carV2File)
|
root, err = a.importNormalFileToFilestoreCARv2(ctx, id, ref.Path, carV2File)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, xerrors.Errorf("failed to import normal file to CARv2: %w", err)
|
return nil, xerrors.Errorf("failed to import normal file to CARv2: %w", err)
|
||||||
}
|
}
|
||||||
@ -502,7 +503,7 @@ func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (res *api.Impor
|
|||||||
if err := a.imgr().AddLabel(id, importmgr.LFileName, ref.Path); err != nil {
|
if err := a.imgr().AddLabel(id, importmgr.LFileName, ref.Path); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := a.imgr().AddLabel(id, importmgr.LCARv2FilePath, carV2File); err != nil {
|
if err := a.imgr().AddLabel(id, importmgr.LFileStoreCARv2FilePath, carV2File); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := a.imgr().AddLabel(id, importmgr.LRootCid, root.String()); err != nil {
|
if err := a.imgr().AddLabel(id, importmgr.LRootCid, root.String()); err != nil {
|
||||||
@ -522,7 +523,7 @@ func (a *API) ClientRemoveImport(ctx context.Context, importID importmgr.ImportI
|
|||||||
}
|
}
|
||||||
|
|
||||||
// remove the CARv2 file if we've created one.
|
// remove the CARv2 file if we've created one.
|
||||||
if path := info.Labels[importmgr.LCARv2FilePath]; path != "" {
|
if path := info.Labels[importmgr.LFileStoreCARv2FilePath]; path != "" {
|
||||||
_ = os.Remove(path)
|
_ = os.Remove(path)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -579,7 +580,7 @@ func (a *API) ClientListImports(ctx context.Context) ([]api.Import, error) {
|
|||||||
Key: id,
|
Key: id,
|
||||||
Source: info.Labels[importmgr.LSource],
|
Source: info.Labels[importmgr.LSource],
|
||||||
FilePath: info.Labels[importmgr.LFileName],
|
FilePath: info.Labels[importmgr.LFileName],
|
||||||
CARv2FilePath: info.Labels[importmgr.LCARv2FilePath],
|
CARv2FilePath: info.Labels[importmgr.LFileStoreCARv2FilePath],
|
||||||
}
|
}
|
||||||
|
|
||||||
if info.Labels[importmgr.LRootCid] != "" {
|
if info.Labels[importmgr.LRootCid] != "" {
|
||||||
@ -999,17 +1000,17 @@ func (w *lenWriter) Write(p []byte) (n int, err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (a *API) ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, error) {
|
func (a *API) ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, error) {
|
||||||
carv2FilePath, err := a.imgr().CARV2FilePathFor(root)
|
fc, err := a.imgr().FilestoreCARV2FilePathFor(root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return api.DataSize{}, xerrors.Errorf("failed to find CARv2 file for root: %w", err)
|
return api.DataSize{}, xerrors.Errorf("failed to find CARv2 file for root: %w", err)
|
||||||
}
|
}
|
||||||
if len(carv2FilePath) == 0 {
|
if len(fc) == 0 {
|
||||||
return api.DataSize{}, xerrors.New("no CARv2 file for root")
|
return api.DataSize{}, xerrors.New("no CARv2 file for root")
|
||||||
}
|
}
|
||||||
|
|
||||||
rdOnly, err := blockstore.OpenReadOnly(carv2FilePath)
|
rdOnly, err := filestorecaradapter.NewReadOnlyFileStore(fc)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return api.DataSize{}, xerrors.Errorf("failed to open read only blockstore: %w", err)
|
return api.DataSize{}, xerrors.Errorf("failed to open read only filestore: %w", err)
|
||||||
}
|
}
|
||||||
defer rdOnly.Close() //nolint:errcheck
|
defer rdOnly.Close() //nolint:errcheck
|
||||||
|
|
||||||
@ -1030,15 +1031,15 @@ func (a *API) ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, e
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (a *API) ClientDealPieceCID(ctx context.Context, root cid.Cid) (api.DataCIDSize, error) {
|
func (a *API) ClientDealPieceCID(ctx context.Context, root cid.Cid) (api.DataCIDSize, error) {
|
||||||
carv2FilePath, err := a.imgr().CARV2FilePathFor(root)
|
fc, err := a.imgr().FilestoreCARV2FilePathFor(root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return api.DataCIDSize{}, xerrors.Errorf("failed to find CARv2 file for root: %w", err)
|
return api.DataCIDSize{}, xerrors.Errorf("failed to find CARv2 file for root: %w", err)
|
||||||
}
|
}
|
||||||
if len(carv2FilePath) == 0 {
|
if len(fc) == 0 {
|
||||||
return api.DataCIDSize{}, xerrors.New("no CARv2 file for root")
|
return api.DataCIDSize{}, xerrors.New("no CARv2 file for root")
|
||||||
}
|
}
|
||||||
|
|
||||||
rdOnly, err := blockstore.OpenReadOnly(carv2FilePath)
|
rdOnly, err := filestorecaradapter.NewReadOnlyFileStore(fc)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return api.DataCIDSize{}, xerrors.Errorf("failed to open read only blockstore: %w", err)
|
return api.DataCIDSize{}, xerrors.Errorf("failed to open read only blockstore: %w", err)
|
||||||
}
|
}
|
||||||
@ -1069,7 +1070,7 @@ func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath stri
|
|||||||
}
|
}
|
||||||
defer os.Remove(tmpCARv2File) //nolint:errcheck
|
defer os.Remove(tmpCARv2File) //nolint:errcheck
|
||||||
|
|
||||||
root, err := a.importNormalFileToCARv2(ctx, id, ref.Path, tmpCARv2File)
|
root, err := a.importNormalFileToFilestoreCARv2(ctx, id, ref.Path, tmpCARv2File)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return xerrors.Errorf("failed to import normal file to CARv2")
|
return xerrors.Errorf("failed to import normal file to CARv2")
|
||||||
}
|
}
|
||||||
|
@ -6,6 +6,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
|
"github.com/filecoin-project/go-fil-markets/filestorecaradapter"
|
||||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||||
"github.com/filecoin-project/lotus/build"
|
"github.com/filecoin-project/lotus/build"
|
||||||
"github.com/filecoin-project/lotus/node/repo/importmgr"
|
"github.com/filecoin-project/lotus/node/repo/importmgr"
|
||||||
@ -24,12 +25,12 @@ import (
|
|||||||
ihelper "github.com/ipfs/go-unixfs/importer/helpers"
|
ihelper "github.com/ipfs/go-unixfs/importer/helpers"
|
||||||
"github.com/ipld/go-car"
|
"github.com/ipld/go-car"
|
||||||
carv2 "github.com/ipld/go-car/v2"
|
carv2 "github.com/ipld/go-car/v2"
|
||||||
"github.com/ipld/go-car/v2/blockstore"
|
|
||||||
"golang.org/x/xerrors"
|
"golang.org/x/xerrors"
|
||||||
)
|
)
|
||||||
|
|
||||||
// importNormalFileToCARv2 transforms the client's "normal file" to a Unixfs IPLD DAG and writes out the DAG to a CARv2 file at the given output path.
|
// importNormalFileToFilestoreCARv2 transforms the client's "normal file" to a Unixfs IPLD DAG and writes out the DAG to a CARv2 file
|
||||||
func (a *API) importNormalFileToCARv2(ctx context.Context, importID importmgr.ImportID, inputFilePath string, outputCARv2Path string) (c cid.Cid, finalErr error) {
|
// that can be used to back a filestore.
|
||||||
|
func (a *API) importNormalFileToFilestoreCARv2(ctx context.Context, importID importmgr.ImportID, inputFilePath string, outputCARv2Path string) (c cid.Cid, finalErr error) {
|
||||||
|
|
||||||
// TODO: We've currently put in a hack to create the Unixfs DAG as a CARv2 without using Badger.
|
// TODO: We've currently put in a hack to create the Unixfs DAG as a CARv2 without using Badger.
|
||||||
// We first create the Unixfs DAG using a filestore to get the root of the Unixfs DAG.
|
// We first create the Unixfs DAG using a filestore to get the root of the Unixfs DAG.
|
||||||
@ -38,7 +39,6 @@ func (a *API) importNormalFileToCARv2(ctx context.Context, importID importmgr.Im
|
|||||||
//
|
//
|
||||||
// In the second pass, we create a CARv2 file with the root present using the root node we get in the above step.
|
// In the second pass, we create a CARv2 file with the root present using the root node we get in the above step.
|
||||||
// This hack should be fixed when CARv2 allows specifying the root AFTER finishing the CARv2 streaming write.
|
// This hack should be fixed when CARv2 allows specifying the root AFTER finishing the CARv2 streaming write.
|
||||||
// TODO How do we clean this up ?
|
|
||||||
fm := filestore.NewFileManager(ds_sync.MutexWrap(datastore.NewMapDatastore()), "/")
|
fm := filestore.NewFileManager(ds_sync.MutexWrap(datastore.NewMapDatastore()), "/")
|
||||||
fm.AllowFiles = true
|
fm.AllowFiles = true
|
||||||
fstore := filestore.NewFilestore(bstore.NewMemorySync(), fm)
|
fstore := filestore.NewFilestore(bstore.NewMemorySync(), fm)
|
||||||
@ -51,14 +51,15 @@ func (a *API) importNormalFileToCARv2(ctx context.Context, importID importmgr.Im
|
|||||||
return cid.Undef, xerrors.Errorf("failed to import file to store: %w", err)
|
return cid.Undef, xerrors.Errorf("failed to import file to store: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
//------ Second Pass --- Now that we have the root of the Unixfs DAG -> write out the Unixfs DAG to a CARv2 file with the root present by using a read-write CARv2 blockstore.
|
//------ Second Pass --- Now that we have the root of the Unixfs DAG -> write out the Unixfs DAG to a CARv2 file with the root present by using a
|
||||||
rw, err := blockstore.NewReadWrite(outputCARv2Path, []cid.Cid{root})
|
// filestore backed by a read-write CARv2 blockstore.
|
||||||
|
fsb, err := filestorecaradapter.NewReadWriteFileStore(outputCARv2Path, []cid.Cid{root})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return cid.Undef, xerrors.Errorf("failed to create a CARv2 read-write blockstore: %w", err)
|
return cid.Undef, xerrors.Errorf("failed to create a CARv2 read-write blockstore: %w", err)
|
||||||
}
|
}
|
||||||
defer rw.Finalize() //nolint:errcheck
|
defer fsb.Close() //nolint:errcheck
|
||||||
|
|
||||||
bsvc = blockservice.New(rw, offline.Exchange(rw))
|
bsvc = blockservice.New(fsb, offline.Exchange(fsb))
|
||||||
root2, err := importNormalFileToUnixfsDAG(ctx, inputFilePath, merkledag.NewDAGService(bsvc))
|
root2, err := importNormalFileToUnixfsDAG(ctx, inputFilePath, merkledag.NewDAGService(bsvc))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return cid.Undef, xerrors.Errorf("failed to create Unixfs DAG with CARv2 blockstore: %w", err)
|
return cid.Undef, xerrors.Errorf("failed to create Unixfs DAG with CARv2 blockstore: %w", err)
|
||||||
|
@ -10,6 +10,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/filecoin-project/go-fil-markets/filestorecaradapter"
|
||||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||||
"github.com/filecoin-project/lotus/node/repo/importmgr"
|
"github.com/filecoin-project/lotus/node/repo/importmgr"
|
||||||
"github.com/ipfs/go-blockservice"
|
"github.com/ipfs/go-blockservice"
|
||||||
@ -81,12 +82,12 @@ func TestImportNormalFileToCARv2(t *testing.T) {
|
|||||||
outputCARv2 := genTmpFile(t)
|
outputCARv2 := genTmpFile(t)
|
||||||
defer os.Remove(outputCARv2) //nolint:errcheck
|
defer os.Remove(outputCARv2) //nolint:errcheck
|
||||||
|
|
||||||
root, err := a.importNormalFileToCARv2(ctx, importID, inputFilePath, outputCARv2)
|
root, err := a.importNormalFileToFilestoreCARv2(ctx, importID, inputFilePath, outputCARv2)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NotEqual(t, cid.Undef, root)
|
require.NotEqual(t, cid.Undef, root)
|
||||||
|
|
||||||
// convert the CARv2 to a normal file again and ensure the contents match
|
// convert the CARv2 to a normal file again and ensure the contents match
|
||||||
readOnly, err := blockstore.OpenReadOnly(outputCARv2)
|
readOnly, err := filestorecaradapter.NewReadOnlyFileStore(outputCARv2)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
defer readOnly.Close() //nolint:errcheck
|
defer readOnly.Close() //nolint:errcheck
|
||||||
dag := merkledag.NewDAGService(blockservice.New(readOnly, offline.Exchange(readOnly)))
|
dag := merkledag.NewDAGService(blockservice.New(readOnly, offline.Exchange(readOnly)))
|
||||||
|
@ -28,8 +28,6 @@ import (
|
|||||||
"github.com/libp2p/go-libp2p-core/host"
|
"github.com/libp2p/go-libp2p-core/host"
|
||||||
"github.com/libp2p/go-libp2p-core/routing"
|
"github.com/libp2p/go-libp2p-core/routing"
|
||||||
|
|
||||||
"github.com/filecoin-project/dagstore"
|
|
||||||
"github.com/filecoin-project/dagstore/mount"
|
|
||||||
"github.com/filecoin-project/go-address"
|
"github.com/filecoin-project/go-address"
|
||||||
dtimpl "github.com/filecoin-project/go-data-transfer/impl"
|
dtimpl "github.com/filecoin-project/go-data-transfer/impl"
|
||||||
dtnet "github.com/filecoin-project/go-data-transfer/network"
|
dtnet "github.com/filecoin-project/go-data-transfer/network"
|
||||||
@ -568,29 +566,21 @@ func BasicDealFilter(user dtypes.StorageDealFilter) func(onlineOk dtypes.Conside
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func DAGStoreRegistry() *mount.Registry {
|
func DAGStoreWrapper(
|
||||||
return mount.NewRegistry()
|
ds dtypes.MetadataDS,
|
||||||
}
|
r repo.LockedRepo,
|
||||||
|
pieceStore dtypes.ProviderPieceStore,
|
||||||
func DAGStore(r repo.LockedRepo, ds dtypes.MetadataDS, dsRegistry *mount.Registry) (*dagstore.DAGStore, error) {
|
rpn retrievalmarket.RetrievalProviderNode,
|
||||||
|
) (mktdagstore.DagStoreWrapper, error) {
|
||||||
dagStoreDir := filepath.Join(r.Path(), "dagstore")
|
dagStoreDir := filepath.Join(r.Path(), "dagstore")
|
||||||
dagStoreDS := namespace.Wrap(ds, datastore.NewKey("/dagstore/provider"))
|
dagStoreDS := namespace.Wrap(ds, datastore.NewKey("/dagstore/provider"))
|
||||||
return dagstore.NewDAGStore(dagstore.Config{
|
cfg := mktdagstore.MarketDAGStoreConfig{
|
||||||
TransientsDir: filepath.Join(dagStoreDir, "transients"),
|
TransientsDir: filepath.Join(dagStoreDir, "transients"),
|
||||||
IndexDir: filepath.Join(dagStoreDir, "index"),
|
IndexDir: filepath.Join(dagStoreDir, "index"),
|
||||||
Datastore: dagStoreDS,
|
Datastore: dagStoreDS,
|
||||||
MountRegistry: dsRegistry,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func DAGStoreWrapper(
|
|
||||||
pieceStore dtypes.ProviderPieceStore,
|
|
||||||
rpn retrievalmarket.RetrievalProviderNode,
|
|
||||||
dsRegistry *mount.Registry,
|
|
||||||
dagStore *dagstore.DAGStore,
|
|
||||||
) (mktdagstore.DagStoreWrapper, error) {
|
|
||||||
mountApi := mktdagstore.NewLotusMountAPI(pieceStore, rpn)
|
mountApi := mktdagstore.NewLotusMountAPI(pieceStore, rpn)
|
||||||
return mktdagstore.NewDagStoreWrapper(dsRegistry, dagStore, mountApi)
|
return mktdagstore.NewDagStoreWrapper(cfg, mountApi)
|
||||||
}
|
}
|
||||||
|
|
||||||
func StorageProvider(minerAddress dtypes.MinerAddress,
|
func StorageProvider(minerAddress dtypes.MinerAddress,
|
||||||
|
@ -32,7 +32,7 @@ const (
|
|||||||
LSource = "source" // Function which created the import
|
LSource = "source" // Function which created the import
|
||||||
LRootCid = "root" // Root CID
|
LRootCid = "root" // Root CID
|
||||||
LFileName = "filename" // Local file path
|
LFileName = "filename" // Local file path
|
||||||
LCARv2FilePath = "CARv2Path" // path of the CARv2 file.
|
LFileStoreCARv2FilePath = "CARv2Path" // path of the full CARv2 file or a CARv2 file that can serve as the backing store for a Filestore.
|
||||||
)
|
)
|
||||||
|
|
||||||
func New(ds datastore.Batching, repoPath string) *Mgr {
|
func New(ds datastore.Batching, repoPath string) *Mgr {
|
||||||
@ -129,7 +129,7 @@ func (m *Mgr) Remove(id ImportID) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Mgr) CARV2FilePathFor(dagRoot cid.Cid) (string, error) {
|
func (m *Mgr) FilestoreCARV2FilePathFor(dagRoot cid.Cid) (string, error) {
|
||||||
importIDs, err := m.List()
|
importIDs, err := m.List()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", xerrors.Errorf("failed to fetch import IDs: %w", err)
|
return "", xerrors.Errorf("failed to fetch import IDs: %w", err)
|
||||||
@ -150,7 +150,7 @@ func (m *Mgr) CARV2FilePathFor(dagRoot cid.Cid) (string, error) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if c.Equals(dagRoot) {
|
if c.Equals(dagRoot) {
|
||||||
return info.Labels[LCARv2FilePath], nil
|
return info.Labels[LFileStoreCARv2FilePath], nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user