feat: integrate dag store into lotus

This commit is contained in:
Dirk McCormick 2021-07-09 10:59:10 +02:00
parent a5ba556fff
commit 872843e491
6 changed files with 65 additions and 38 deletions

7
go.mod
View File

@ -26,6 +26,7 @@ require (
github.com/elastic/gosigar v0.12.0
github.com/etclabscore/go-openrpc-reflect v0.0.36
github.com/fatih/color v1.9.0
github.com/filecoin-project/dagstore v0.0.0-20210708130647-e413e3ad83df
github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200910194244-f640612a1a1f
github.com/filecoin-project/go-address v0.0.5
github.com/filecoin-project/go-bitfield v0.2.4
@ -34,7 +35,7 @@ require (
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
github.com/filecoin-project/go-data-transfer v1.7.0
github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a
github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210706090655-07597f8dda68
github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210708144500-db723d7ea4ff
github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec
github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20
github.com/filecoin-project/go-paramfetch v0.0.2-0.20210614165157-25a6c7769498
@ -95,7 +96,7 @@ require (
github.com/ipfs/go-unixfs v0.2.6
github.com/ipfs/interface-go-ipfs-core v0.2.3
github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d
github.com/ipld/go-car/v2 v2.0.0-20210706083137-aa61149042cd
github.com/ipld/go-car/v2 v2.0.0-20210708104948-d79de78d9567
github.com/ipld/go-ipld-prime v0.5.1-0.20201021195245-109253e8a018
github.com/kelseyhightower/envconfig v1.4.0
github.com/lib/pq v1.7.0
@ -166,5 +167,3 @@ replace github.com/filecoin-project/lotus => ./
replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi
replace github.com/filecoin-project/test-vectors => ./extern/test-vectors
replace github.com/ipld/go-car/v2 => github.com/mvdan/go-car/v2 v2.0.0-20210705214709-67212dfac7af

12
go.sum
View File

@ -256,8 +256,8 @@ github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGj
github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E=
github.com/filecoin-project/dagstore v0.0.0-20210624112130-607a5a2a5bd4 h1:AcL0EUgq37GqpsP085Ib+mJGBhNqAydeS/C1u59Prbg=
github.com/filecoin-project/dagstore v0.0.0-20210624112130-607a5a2a5bd4/go.mod h1:b8ZvtSY5ggpvS2AkeF+Hbdcu5/Xm1P6RIR5NKJj7OhE=
github.com/filecoin-project/dagstore v0.0.0-20210708130647-e413e3ad83df h1:S35PjZ9zJ/N/Oy6UshqCu8aRZujrKOwmkCtAgQaHBCU=
github.com/filecoin-project/dagstore v0.0.0-20210708130647-e413e3ad83df/go.mod h1:Qpv2Ka8Wg0iktm7cfcejJPG5hSjsKrhTy6LveOxXYYs=
github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8=
github.com/filecoin-project/go-address v0.0.5 h1:SSaFT/5aLfPXycUlFyemoHYhRgdyXClXCyDdNJKPlDM=
github.com/filecoin-project/go-address v0.0.5/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8=
@ -287,8 +287,8 @@ github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go
github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a h1:hyJ+pUm/4U4RdEZBlg6k8Ma4rDiuvqyGpoICXAxwsTg=
github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c=
github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210706090655-07597f8dda68 h1:qwVHCq2MDBGy6GmaKnMNQ8PJalAjXpVVLj5OIAGRpWw=
github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210706090655-07597f8dda68/go.mod h1:vXE4CrmWzsZ1kUswf309xOJS7injqfLv37WNURxP7SA=
github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210708144500-db723d7ea4ff h1:lRNlyyDMYbrkBBCq4BGtLmDpTllwuDCtbH+aaS/i9vs=
github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210708144500-db723d7ea4ff/go.mod h1:llRlmiWK9yCWVf0pKS13XF3oNUUzkmbX5ie0a72vKzw=
github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM=
github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24=
github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM=
@ -750,6 +750,8 @@ github.com/ipld/go-car v0.1.0/go.mod h1:RCWzaUh2i4mOEkB3W45Vc+9jnS/M6Qay5ooytiBH
github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4/go.mod h1:xrMEcuSq+D1vEwl+YAXsg/JfA98XGpXDwnkIL4Aimqw=
github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d h1:iphSzTuPqyDgH7WUVZsdqUnQNzYgIblsVr1zhVNA33U=
github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d/go.mod h1:2Gys8L8MJ6zkh1gktTSXreY63t4UbyvNp5JaudTyxHQ=
github.com/ipld/go-car/v2 v2.0.0-20210708104948-d79de78d9567 h1:81/TGRHgkIbF7xSNF95jmR8nkILx6DBn/69ujuquO8Q=
github.com/ipld/go-car/v2 v2.0.0-20210708104948-d79de78d9567/go.mod h1:Ueq4zx/SNx7yHwmfr9xKlKpXxRCMM6wyqC8B0rv9oig=
github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w=
github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e/go.mod h1:uVIwe/u0H4VdKv3kaN1ck7uCb6yD9cFLS9/ELyXbsw8=
github.com/ipld/go-ipld-prime v0.5.1-0.20200828233916-988837377a7f/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM=
@ -1293,8 +1295,6 @@ github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS
github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY=
github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
github.com/mvdan/go-car/v2 v2.0.0-20210705214709-67212dfac7af h1:8p3gLYrSJETv8fya3ybBTYT+gBBjILIPOHfSgaFyCCQ=
github.com/mvdan/go-car/v2 v2.0.0-20210705214709-67212dfac7af/go.mod h1:Ueq4zx/SNx7yHwmfr9xKlKpXxRCMM6wyqC8B0rv9oig=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU=

View File

@ -63,7 +63,7 @@ func TestDealWithMarketAndMinerNode(t *testing.T) {
ns := fmt.Sprintf("%d", n)
t.Run(ns+"-fastretrieval-CAR", func(t *testing.T) { runTest(t, n, true, true) })
t.Run(ns+"-fastretrieval-NoCAR", func(t *testing.T) { runTest(t, n, true, false) })
t.Run(ns+"-stdretrieval-CAR", func(t *testing.T) { runTest(t, n, true, false) })
t.Run(ns+"-stdretrieval-CAR", func(t *testing.T) { runTest(t, n, false, true) })
t.Run(ns+"-stdretrieval-NoCAR", func(t *testing.T) { runTest(t, n, false, false) })
}
}
@ -102,7 +102,7 @@ func TestDealCyclesConcurrent(t *testing.T) {
ns := fmt.Sprintf("%d", n)
t.Run(ns+"-fastretrieval-CAR", func(t *testing.T) { runTest(t, n, true, true) })
t.Run(ns+"-fastretrieval-NoCAR", func(t *testing.T) { runTest(t, n, true, false) })
t.Run(ns+"-stdretrieval-CAR", func(t *testing.T) { runTest(t, n, true, false) })
t.Run(ns+"-stdretrieval-CAR", func(t *testing.T) { runTest(t, n, false, true) })
t.Run(ns+"-stdretrieval-NoCAR", func(t *testing.T) { runTest(t, n, false, false) })
}
}

View File

@ -4,17 +4,17 @@ import (
"errors"
"time"
"github.com/filecoin-project/go-fil-markets/shared_testutil/dagstore"
"go.uber.org/fx"
"golang.org/x/xerrors"
"github.com/filecoin-project/dagstore"
"github.com/filecoin-project/dagstore/mount"
mktdagstore "github.com/filecoin-project/go-fil-markets/dagstore"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-fil-markets/storagemarket/impl/storedask"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/markets/retrievaladapter"
storage2 "github.com/filecoin-project/specs-storage/storage"
"github.com/filecoin-project/lotus/api"
@ -26,6 +26,7 @@ import (
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
"github.com/filecoin-project/lotus/markets/dealfilter"
"github.com/filecoin-project/lotus/markets/retrievaladapter"
"github.com/filecoin-project/lotus/markets/storageadapter"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/config"
@ -148,7 +149,9 @@ func ConfigStorageMiner(c interface{}) Option {
Override(new(dtypes.RetrievalPricingFunc), modules.RetrievalPricingFunc(cfg.Dealmaking)),
// DAG Store
Override(new(dagstore.DagStore), modules.DAGStore),
Override(new(*mount.Registry), modules.DAGStoreRegistry),
Override(new(*dagstore.DAGStore), modules.DAGStore),
Override(new(mktdagstore.DagStoreWrapper), modules.DAGStoreWrapper),
// Markets (retrieval)
Override(new(retrievalmarket.RetrievalProviderNode), retrievaladapter.NewRetrievalProviderNode),

View File

@ -7,7 +7,6 @@ import (
"path/filepath"
"time"
"github.com/filecoin-project/lotus/node/repo/retrievalstoremgr"
"go.uber.org/fx"
"golang.org/x/xerrors"
@ -17,6 +16,7 @@ import (
dtgstransport "github.com/filecoin-project/go-data-transfer/transport/graphsync"
"github.com/filecoin-project/go-fil-markets/discovery"
discoveryimpl "github.com/filecoin-project/go-fil-markets/discovery/impl"
"github.com/filecoin-project/go-fil-markets/filestore"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl"
rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network"
@ -40,6 +40,7 @@ import (
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/filecoin-project/lotus/node/repo/retrievalstoremgr"
)
func HandleMigrateClientFunds(lc fx.Lifecycle, ds dtypes.MetadataDS, wallet full.WalletAPI, fundMgr *market.FundManager) {
@ -178,22 +179,19 @@ func StorageClient(lc fx.Lifecycle, h host.Host, dataTransfer dtypes.ClientDataT
return c, nil
}
type CARStore struct {
dir string
}
func (c *CARStore) Path(key string) string {
return filepath.Join(c.dir, key)
}
// RetrievalClient creates a new retrieval client attached to the client blockstore
func RetrievalClient(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, dt dtypes.ClientDataTransfer, payAPI payapi.PaychAPI, resolver discovery.PeerResolver,
ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI, j journal.Journal) (retrievalmarket.RetrievalClient, error) {
carStore, err := getRetrievalCarStore(r.Path())
if err != nil {
return nil, err
}
adapter := retrievaladapter.NewRetrievalClientNode(payAPI, chainAPI, stateAPI)
network := rmnet.NewFromLibp2pHost(h)
client, err := retrievalimpl.NewClient(network,
&CARStore{r.Path()}, dt, adapter, resolver, namespace.Wrap(ds, datastore.NewKey("/retrievals/client")))
carStore, dt, adapter, resolver, namespace.Wrap(ds, datastore.NewKey("/retrievals/client")))
if err != nil {
return nil, err
}
@ -211,6 +209,15 @@ func RetrievalClient(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, dt dtypes.
return client, nil
}
func getRetrievalCarStore(path string) (filestore.CarFileStore, error) {
carStorePath := filepath.Join(path, "retrieval-cars")
err := os.Mkdir(carStorePath, os.ModePerm)
if err != nil {
return nil, xerrors.Errorf("could not create directory %s: %w", carStorePath, err)
}
return filestore.NewLocalCarStore(path)
}
// ClientBlockstoreRetrievalStoreManager is the default version of the RetrievalStoreManager that runs on multistore
func ClientBlockstoreRetrievalStoreManager(isIpfsRetrieval bool) func(bs dtypes.ClientBlockstore) (dtypes.ClientRetrievalStoreManager, error) {
return func(bs dtypes.ClientBlockstore) (dtypes.ClientRetrievalStoreManager, error) {

View File

@ -11,11 +11,6 @@ import (
"strings"
"time"
mktdagstore "github.com/filecoin-project/go-fil-markets/dagstore"
"github.com/filecoin-project/go-fil-markets/shared_testutil/dagstore"
"github.com/filecoin-project/lotus/markets/pricing"
"go.uber.org/fx"
"go.uber.org/multierr"
"golang.org/x/xerrors"
@ -33,10 +28,13 @@ import (
"github.com/libp2p/go-libp2p-core/host"
"github.com/libp2p/go-libp2p-core/routing"
"github.com/filecoin-project/dagstore"
"github.com/filecoin-project/dagstore/mount"
"github.com/filecoin-project/go-address"
dtimpl "github.com/filecoin-project/go-data-transfer/impl"
dtnet "github.com/filecoin-project/go-data-transfer/network"
dtgstransport "github.com/filecoin-project/go-data-transfer/transport/graphsync"
mktdagstore "github.com/filecoin-project/go-fil-markets/dagstore"
piecefilestore "github.com/filecoin-project/go-fil-markets/filestore"
piecestoreimpl "github.com/filecoin-project/go-fil-markets/piecestore/impl"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
@ -53,13 +51,13 @@ import (
"github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/go-storedcounter"
"github.com/filecoin-project/lotus/api"
sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/v0api"
"github.com/filecoin-project/lotus/api/v1api"
"github.com/filecoin-project/lotus/blockstore"
@ -71,6 +69,7 @@ import (
"github.com/filecoin-project/lotus/journal"
"github.com/filecoin-project/lotus/markets"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
"github.com/filecoin-project/lotus/markets/pricing"
lotusminer "github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/modules/dtypes"
@ -569,9 +568,29 @@ func BasicDealFilter(user dtypes.StorageDealFilter) func(onlineOk dtypes.Conside
}
}
func DAGStore(r repo.LockedRepo) (dagstore.DagStore, error) {
md := dagstore.NewMockDagStore()
return md, nil
func DAGStoreRegistry() *mount.Registry {
return mount.NewRegistry()
}
func DAGStore(r repo.LockedRepo, ds dtypes.MetadataDS, dsRegistry *mount.Registry) (*dagstore.DAGStore, error) {
dagStoreDir := filepath.Join(r.Path(), "dagstore")
dagStoreDS := namespace.Wrap(ds, datastore.NewKey("/dagstore/provider"))
return dagstore.NewDAGStore(dagstore.Config{
TransientsDir: filepath.Join(dagStoreDir, "transients"),
IndexDir: filepath.Join(dagStoreDir, "index"),
Datastore: dagStoreDS,
MountRegistry: dsRegistry,
})
}
func DAGStoreWrapper(
pieceStore dtypes.ProviderPieceStore,
rpn retrievalmarket.RetrievalProviderNode,
dsRegistry *mount.Registry,
dagStore *dagstore.DAGStore,
) (mktdagstore.DagStoreWrapper, error) {
mountApi := mktdagstore.NewLotusMountAPI(pieceStore, rpn)
return mktdagstore.NewDagStoreWrapper(dsRegistry, dagStore, mountApi)
}
func StorageProvider(minerAddress dtypes.MinerAddress,
@ -582,7 +601,7 @@ func StorageProvider(minerAddress dtypes.MinerAddress,
dataTransfer dtypes.ProviderDataTransfer,
spn storagemarket.StorageProviderNode,
df dtypes.StorageDealFilter,
dagStore dagstore.DagStore,
dagStore mktdagstore.DagStoreWrapper,
) (storagemarket.StorageProvider, error) {
net := smnet.NewFromLibp2pHost(h)
store, err := piecefilestore.NewLocalFileStore(piecefilestore.OsPath(r.Path()))
@ -657,7 +676,7 @@ func RetrievalProvider(
dt dtypes.ProviderDataTransfer,
pricingFnc dtypes.RetrievalPricingFunc,
userFilter dtypes.RetrievalDealFilter,
dagStore dagstore.DagStore,
dagStore mktdagstore.DagStoreWrapper,
) (retrievalmarket.RetrievalProvider, error) {
opt := retrievalimpl.DealDeciderOpt(retrievalimpl.DealDecider(userFilter))
return retrievalimpl.NewProvider(
@ -669,7 +688,6 @@ func RetrievalProvider(
dt,
namespace.Wrap(ds, datastore.NewKey("/retrievals/provider")),
retrievalimpl.RetrievalPricingFunc(pricingFnc),
mktdagstore.NewMount(pieceStore, adapter),
opt,
)
}