diff --git a/api/apibstore/apibstore.go b/api/apibstore/apibstore.go index 5bd0f0ad7..cf9f4f24c 100644 --- a/api/apibstore/apibstore.go +++ b/api/apibstore/apibstore.go @@ -5,8 +5,9 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/lib/blockstore" ) type ChainIO interface { diff --git a/chain/events/state/diff_adt_test.go b/chain/events/state/diff_adt_test.go index c57105179..d1b98bd11 100644 --- a/chain/events/state/diff_adt_test.go +++ b/chain/events/state/diff_adt_test.go @@ -8,14 +8,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - ds "github.com/ipfs/go-datastore" - ds_sync "github.com/ipfs/go-datastore/sync" - bstore "github.com/ipfs/go-ipfs-blockstore" cbornode "github.com/ipfs/go-ipld-cbor" typegen "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/filecoin-project/specs-actors/actors/util/adt" + + bstore "github.com/filecoin-project/lotus/lib/blockstore" ) func TestDiffAdtArray(t *testing.T) { @@ -146,7 +145,7 @@ func (t *TestAdtDiff) Remove(key uint64, val *typegen.Deferred) error { func newContextStore() *contextStore { ctx := context.Background() - bs := bstore.NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) + bs := bstore.NewTemporarySync() store := cbornode.NewCborStore(bs) return &contextStore{ ctx: ctx, diff --git a/chain/events/state/predicates_test.go b/chain/events/state/predicates_test.go index 4592cce57..2f119e3ea 100644 --- a/chain/events/state/predicates_test.go +++ b/chain/events/state/predicates_test.go @@ -8,9 +8,6 @@ import ( "golang.org/x/xerrors" "github.com/ipfs/go-cid" - ds "github.com/ipfs/go-datastore" - ds_sync "github.com/ipfs/go-datastore/sync" - bstore "github.com/ipfs/go-ipfs-blockstore" cbornode "github.com/ipfs/go-ipld-cbor" "github.com/filecoin-project/go-address" @@ -23,6 +20,7 @@ import ( tutils "github.com/filecoin-project/specs-actors/support/testing" "github.com/filecoin-project/lotus/chain/types" + bstore "github.com/filecoin-project/lotus/lib/blockstore" ) var dummyCid cid.Cid @@ -66,7 +64,7 @@ func (m mockAPI) setActor(tsk types.TipSetKey, act *types.Actor) { func TestMarketPredicates(t *testing.T) { ctx := context.Background() - bs := bstore.NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) + bs := bstore.NewTemporarySync() store := adt.WrapStore(ctx, cbornode.NewCborStore(bs)) oldDeal1 := &market.DealState{ @@ -281,7 +279,7 @@ func TestMarketPredicates(t *testing.T) { func TestMinerSectorChange(t *testing.T) { ctx := context.Background() - bs := bstore.NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) + bs := bstore.NewTemporarySync() store := adt.WrapStore(ctx, cbornode.NewCborStore(bs)) nextID := uint64(0) diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 01b874f92..b54599176 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -15,7 +15,6 @@ import ( block "github.com/ipfs/go-block-format" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" offline "github.com/ipfs/go-ipfs-exchange-offline" format "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" @@ -35,6 +34,7 @@ import ( "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/sector-storage/ffiwrapper" @@ -112,7 +112,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { return nil, xerrors.Errorf("failed to get blocks datastore: %w", err) } - bs := mybs{blockstore.NewIdStore(blockstore.NewBlockstore(bds))} + bs := mybs{blockstore.NewBlockstore(bds)} ks, err := lr.KeyStore() if err != nil { diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go index b8ede8856..bfb449b78 100644 --- a/chain/gen/genesis/genesis.go +++ b/chain/gen/genesis/genesis.go @@ -3,12 +3,12 @@ package genesis import ( "context" "encoding/json" + "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/specs-actors/actors/crypto" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" - bstore "github.com/ipfs/go-ipfs-blockstore" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" @@ -28,6 +28,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/genesis" + bstore "github.com/filecoin-project/lotus/lib/blockstore" ) const AccountStart = 100 diff --git a/chain/gen/genesis/t00_system.go b/chain/gen/genesis/t00_system.go index 581d7e788..6e6cc976a 100644 --- a/chain/gen/genesis/t00_system.go +++ b/chain/gen/genesis/t00_system.go @@ -2,12 +2,14 @@ package genesis import ( "context" + "github.com/filecoin-project/specs-actors/actors/builtin/system" - "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/builtin" - bstore "github.com/ipfs/go-ipfs-blockstore" cbor "github.com/ipfs/go-ipld-cbor" + + "github.com/filecoin-project/lotus/chain/types" + bstore "github.com/filecoin-project/lotus/lib/blockstore" ) func SetupSystemActor(bs bstore.Blockstore) (*types.Actor, error) { diff --git a/chain/gen/genesis/t01_init.go b/chain/gen/genesis/t01_init.go index a7f45482e..9f0ffaddd 100644 --- a/chain/gen/genesis/t01_init.go +++ b/chain/gen/genesis/t01_init.go @@ -9,13 +9,13 @@ import ( "github.com/filecoin-project/specs-actors/actors/util/adt" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" - bstore "github.com/ipfs/go-ipfs-blockstore" cbor "github.com/ipfs/go-ipld-cbor" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/genesis" + bstore "github.com/filecoin-project/lotus/lib/blockstore" ) func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesis.Actor) (*types.Actor, error) { diff --git a/chain/gen/genesis/t02_reward.go b/chain/gen/genesis/t02_reward.go index 0866f9fa3..2f5922fd3 100644 --- a/chain/gen/genesis/t02_reward.go +++ b/chain/gen/genesis/t02_reward.go @@ -5,12 +5,13 @@ import ( "github.com/filecoin-project/specs-actors/actors/abi/big" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/reward" - bstore "github.com/ipfs/go-ipfs-blockstore" cbor "github.com/ipfs/go-ipld-cbor" + + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/types" + bstore "github.com/filecoin-project/lotus/lib/blockstore" ) func SetupRewardActor(bs bstore.Blockstore, qaPower big.Int) (*types.Actor, error) { diff --git a/chain/gen/genesis/t03_cron.go b/chain/gen/genesis/t03_cron.go index e61a88d18..cf2c0d7a7 100644 --- a/chain/gen/genesis/t03_cron.go +++ b/chain/gen/genesis/t03_cron.go @@ -5,10 +5,10 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/cron" - bstore "github.com/ipfs/go-ipfs-blockstore" cbor "github.com/ipfs/go-ipld-cbor" "github.com/filecoin-project/lotus/chain/types" + bstore "github.com/filecoin-project/lotus/lib/blockstore" ) func SetupCronActor(bs bstore.Blockstore) (*types.Actor, error) { diff --git a/chain/gen/genesis/t04_power.go b/chain/gen/genesis/t04_power.go index b20045ca3..56660b602 100644 --- a/chain/gen/genesis/t04_power.go +++ b/chain/gen/genesis/t04_power.go @@ -8,10 +8,10 @@ import ( "github.com/filecoin-project/specs-actors/actors/abi/big" "github.com/filecoin-project/specs-actors/actors/builtin/power" - bstore "github.com/ipfs/go-ipfs-blockstore" cbor "github.com/ipfs/go-ipld-cbor" "github.com/filecoin-project/lotus/chain/types" + bstore "github.com/filecoin-project/lotus/lib/blockstore" ) func SetupStoragePowerActor(bs bstore.Blockstore) (*types.Actor, error) { diff --git a/chain/gen/genesis/t05_market.go b/chain/gen/genesis/t05_market.go index 1673f1255..615e8370b 100644 --- a/chain/gen/genesis/t05_market.go +++ b/chain/gen/genesis/t05_market.go @@ -6,10 +6,10 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/specs-actors/actors/util/adt" - bstore "github.com/ipfs/go-ipfs-blockstore" cbor "github.com/ipfs/go-ipld-cbor" "github.com/filecoin-project/lotus/chain/types" + bstore "github.com/filecoin-project/lotus/lib/blockstore" ) func SetupStorageMarketActor(bs bstore.Blockstore) (*types.Actor, error) { diff --git a/chain/gen/genesis/t06_vreg.go b/chain/gen/genesis/t06_vreg.go index 47477a733..a07e36711 100644 --- a/chain/gen/genesis/t06_vreg.go +++ b/chain/gen/genesis/t06_vreg.go @@ -4,7 +4,6 @@ import ( "context" "github.com/filecoin-project/go-address" - bstore "github.com/ipfs/go-ipfs-blockstore" cbor "github.com/ipfs/go-ipld-cbor" "github.com/filecoin-project/specs-actors/actors/builtin" @@ -12,6 +11,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/chain/types" + bstore "github.com/filecoin-project/lotus/lib/blockstore" ) var RootVerifierAddr address.Address diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index 0fc28f92c..c855c8234 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -26,11 +26,11 @@ import ( . "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/lib/blockstore" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log" cbg "github.com/whyrusleeping/cbor-gen" diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index a888e869a..0b3305ef9 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -13,20 +13,22 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/lib/blockstore" + "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/abi/big" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/util/adt" - cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" bls "github.com/filecoin-project/filecoin-ffi" "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log/v2" + cbg "github.com/whyrusleeping/cbor-gen" "go.opencensus.io/trace" ) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index e30a245de..7c8705004 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -7,7 +7,6 @@ import ( "reflect" cid "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" cbor "github.com/ipfs/go-ipld-cbor" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" @@ -38,6 +37,7 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/node/modules/dtypes" ) diff --git a/chain/store/index_test.go b/chain/store/index_test.go index 73f4901f0..9443b4c07 100644 --- a/chain/store/index_test.go +++ b/chain/store/index_test.go @@ -8,10 +8,10 @@ import ( "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types/mock" + "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/specs-actors/actors/abi" datastore "github.com/ipfs/go-datastore" syncds "github.com/ipfs/go-datastore/sync" - blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/stretchr/testify/assert" ) @@ -30,7 +30,7 @@ func TestIndexSeeks(t *testing.T) { ctx := context.TODO() - nbs := blockstore.NewBlockstore(syncds.MutexWrap(datastore.NewMapDatastore())) + nbs := blockstore.NewTemporarySync() cs := store.NewChainStore(nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil) _, err = cs.Import(bytes.NewReader(gencar)) diff --git a/chain/store/store.go b/chain/store/store.go index 13f317693..50475e6c4 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -20,7 +20,9 @@ import ( "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/journal" + bstore "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/metrics" + "go.opencensus.io/stats" "go.opencensus.io/trace" "go.uber.org/multierr" @@ -31,8 +33,6 @@ import ( block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" dstore "github.com/ipfs/go-datastore" - blockstore "github.com/ipfs/go-ipfs-blockstore" - bstore "github.com/ipfs/go-ipfs-blockstore" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log/v2" car "github.com/ipld/go-car" @@ -896,7 +896,7 @@ func (cs *ChainStore) Blockstore() bstore.Blockstore { return cs.bs } -func ActorStore(ctx context.Context, bs blockstore.Blockstore) adt.Store { +func ActorStore(ctx context.Context, bs bstore.Blockstore) adt.Store { return adt.WrapStore(ctx, cbor.NewCborStore(bs)) } @@ -1019,7 +1019,7 @@ func (cs *ChainStore) GetTipsetByHeight(ctx context.Context, h abi.ChainEpoch, t return cs.LoadTipSet(lbts.Parents()) } -func recurseLinks(bs blockstore.Blockstore, root cid.Cid, in []cid.Cid) ([]cid.Cid, error) { +func recurseLinks(bs bstore.Blockstore, root cid.Cid, in []cid.Cid) ([]cid.Cid, error) { if root.Prefix().Codec != cid.DagCBOR { return in, nil } diff --git a/chain/store/store_test.go b/chain/store/store_test.go index 939c85d20..3267feccd 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -6,7 +6,6 @@ import ( "testing" datastore "github.com/ipfs/go-datastore" - blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/abi/big" @@ -18,6 +17,7 @@ import ( "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/node/repo" ) @@ -100,7 +100,7 @@ func TestChainExportImport(t *testing.T) { t.Fatal(err) } - nbs := blockstore.NewBlockstore(datastore.NewMapDatastore()) + nbs := blockstore.NewTemporary() cs := store.NewChainStore(nbs, datastore.NewMapDatastore(), nil) root, err := cs.Import(buf) diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 875e2658f..958ed5c86 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -14,8 +14,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/util/adt" lru "github.com/hashicorp/golang-lru" "github.com/ipfs/go-cid" - dstore "github.com/ipfs/go-datastore" - bstore "github.com/ipfs/go-ipfs-blockstore" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log/v2" connmgr "github.com/libp2p/go-libp2p-core/connmgr" @@ -32,6 +30,7 @@ import ( "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/lib/bufbstore" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/metrics" @@ -239,7 +238,7 @@ func (bv *BlockValidator) isChainNearSynced() bool { func (bv *BlockValidator) validateMsgMeta(ctx context.Context, msg *types.BlockMsg) error { // TODO there has to be a simpler way to do this without the blockstore dance - store := adt.WrapStore(ctx, cbor.NewCborStore(bstore.NewBlockstore(dstore.NewMapDatastore()))) + store := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewTemporary())) bmArr := adt.MakeEmptyArray(store) smArr := adt.MakeEmptyArray(store) diff --git a/chain/sync.go b/chain/sync.go index 20475171e..b958e51e7 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -13,8 +13,6 @@ import ( "github.com/Gurpartap/async" "github.com/hashicorp/go-multierror" "github.com/ipfs/go-cid" - dstore "github.com/ipfs/go-datastore" - bstore "github.com/ipfs/go-ipfs-blockstore" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/connmgr" @@ -44,6 +42,7 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" + bstore "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/metrics" ) @@ -1394,8 +1393,7 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS for bsi := 0; bsi < len(bstout); bsi++ { // temp storage so we don't persist data we dont want to - ds := dstore.NewMapDatastore() - bs := bstore.NewBlockstore(ds) + bs := bstore.NewTemporary() blks := cbor.NewCborStore(bs) this := headers[i-bsi] diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index 4bdc00180..0ea26ef16 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -11,7 +11,6 @@ import ( block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - "github.com/multiformats/go-multihash" xerrors "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -76,8 +75,7 @@ func (blk *BlockHeader) ToStorageBlock() (block.Block, error) { return nil, err } - pref := cid.NewPrefixV1(cid.DagCBOR, multihash.BLAKE2B_MIN+31) - c, err := pref.Sum(data) + c, err := abi.CidBuilder.Sum(data) if err != nil { return nil, err } @@ -145,13 +143,12 @@ func (mm *MsgMeta) Cid() cid.Cid { } func (mm *MsgMeta) ToStorageBlock() (block.Block, error) { - buf := new(bytes.Buffer) - if err := mm.MarshalCBOR(buf); err != nil { + var buf bytes.Buffer + if err := mm.MarshalCBOR(&buf); err != nil { return nil, xerrors.Errorf("failed to marshal MsgMeta: %w", err) } - pref := cid.NewPrefixV1(cid.DagCBOR, multihash.BLAKE2B_MIN+31) - c, err := pref.Sum(buf.Bytes()) + c, err := abi.CidBuilder.Sum(buf.Bytes()) if err != nil { return nil, err } diff --git a/chain/types/message.go b/chain/types/message.go index d402490f2..1956950ad 100644 --- a/chain/types/message.go +++ b/chain/types/message.go @@ -9,7 +9,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/abi/big" block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - "github.com/multiformats/go-multihash" xerrors "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -89,8 +88,7 @@ func (m *Message) ToStorageBlock() (block.Block, error) { return nil, err } - pref := cid.NewPrefixV1(cid.DagCBOR, multihash.BLAKE2B_MIN+31) - c, err := pref.Sum(data) + c, err := abi.CidBuilder.Sum(data) if err != nil { return nil, err } diff --git a/chain/types/signedmessage.go b/chain/types/signedmessage.go index 54e82a957..77374ca38 100644 --- a/chain/types/signedmessage.go +++ b/chain/types/signedmessage.go @@ -3,10 +3,10 @@ package types import ( "bytes" + "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/crypto" block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - "github.com/multiformats/go-multihash" ) func (sm *SignedMessage) ToStorageBlock() (block.Block, error) { @@ -19,8 +19,7 @@ func (sm *SignedMessage) ToStorageBlock() (block.Block, error) { return nil, err } - pref := cid.NewPrefixV1(cid.DagCBOR, multihash.BLAKE2B_MIN+31) - c, err := pref.Sum(data) + c, err := abi.CidBuilder.Sum(data) if err != nil { return nil, err } diff --git a/chain/types/tipset_key.go b/chain/types/tipset_key.go index 638b4380e..ee1994f5a 100644 --- a/chain/types/tipset_key.go +++ b/chain/types/tipset_key.go @@ -5,8 +5,8 @@ import ( "encoding/json" "strings" + "github.com/filecoin-project/specs-actors/actors/abi" "github.com/ipfs/go-cid" - "github.com/multiformats/go-multihash" ) var EmptyTSK = TipSetKey{} @@ -15,7 +15,9 @@ var EmptyTSK = TipSetKey{} var blockHeaderCIDLen int func init() { - c, err := cid.V1Builder{Codec: cid.DagCBOR, MhType: multihash.BLAKE2B_MIN + 31}.Sum([]byte{}) + // hash a large string of zeros so we don't estimate based on inlined CIDs. + var buf [256]byte + c, err := abi.CidBuilder.Sum(buf[:]) if err != nil { panic(err) } diff --git a/chain/validation/state.go b/chain/validation/state.go index 965d0a638..2a10eb6af 100644 --- a/chain/validation/state.go +++ b/chain/validation/state.go @@ -3,20 +3,20 @@ package validation import ( "context" - "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" - blockstore "github.com/ipfs/go-ipfs-blockstore" cbor "github.com/ipfs/go-ipld-cbor" "golang.org/x/xerrors" vstate "github.com/filecoin-project/chain-validation/state" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/abi/big" "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/blockstore" ) var _ vstate.VMWrapper = &StateWrapper{} @@ -34,7 +34,7 @@ type StateWrapper struct { } func NewState() *StateWrapper { - bs := blockstore.NewBlockstore(datastore.NewMapDatastore()) + bs := blockstore.NewTemporary() cst := cbor.NewCborStore(bs) // Put EmptyObjectCid value in the store. When an actor is initially created its Head is set to this value. _, err := cst.Put(context.TODO(), map[string]string{}) diff --git a/chain/vm/vm.go b/chain/vm/vm.go index d9e056be6..4e3f7eb30 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -11,7 +11,6 @@ import ( block "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log/v2" mh "github.com/multiformats/go-multihash" @@ -30,6 +29,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors/aerrors" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/lib/bufbstore" ) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 9ea3e8d0f..747514d74 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -21,16 +21,17 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/lib/blockstore" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" + "github.com/filecoin-project/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/abi" - "golang.org/x/xerrors" "github.com/ipfs/go-datastore" badger "github.com/ipfs/go-ds-badger2" - blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" ) type TipSetExec struct { diff --git a/cmd/lotus-shed/import-car.go b/cmd/lotus-shed/import-car.go index 3347f9825..01343c4a3 100644 --- a/cmd/lotus-shed/import-car.go +++ b/cmd/lotus-shed/import-car.go @@ -5,11 +5,11 @@ import ( "io" "os" - blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipld/go-car" "github.com/urfave/cli/v2" "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/node/repo" ) @@ -48,7 +48,6 @@ var importCarCmd = &cli.Command{ } bs := blockstore.NewBlockstore(ds) - bs = blockstore.NewIdStore(bs) cr, err := car.NewCarReader(f) if err != nil { diff --git a/cmd/lotus-townhall/main.go b/cmd/lotus-townhall/main.go index 58ed36478..5f9603fba 100644 --- a/cmd/lotus-townhall/main.go +++ b/cmd/lotus-townhall/main.go @@ -10,14 +10,13 @@ import ( rice "github.com/GeertJohan/go.rice" "github.com/gorilla/websocket" - "github.com/ipfs/go-datastore" - blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipld/go-car" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p-core/peer" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/lib/blockstore" ) var topic = "/fil/headnotifs/" @@ -29,7 +28,7 @@ func init() { return } - bs := blockstore.NewBlockstore(datastore.NewMapDatastore()) + bs := blockstore.NewTemporary() c, err := car.LoadCar(bs, bytes.NewReader(genBytes)) if err != nil { diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 45a12fe98..99763dc40 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -15,7 +15,6 @@ import ( "github.com/filecoin-project/lotus/chain/types" paramfetch "github.com/filecoin-project/go-paramfetch" - blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" "github.com/urfave/cli/v2" @@ -31,6 +30,7 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/vm" lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/lib/peermgr" "github.com/filecoin-project/lotus/lib/ulimit" "github.com/filecoin-project/lotus/metrics" diff --git a/go.mod b/go.mod index 89a63ec2c..f4e5052aa 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/sector-storage v0.0.0-20200723200950-ed2e57dde6df - github.com/filecoin-project/specs-actors v0.8.1-0.20200724015154-3c690d9b7e1d + github.com/filecoin-project/specs-actors v0.8.1-0.20200724060953-964aec9ab294 github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea github.com/filecoin-project/storage-fsm v0.0.0-20200720190000-2cfe2fe3c334 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 @@ -44,7 +44,7 @@ require ( github.com/ipfs/go-bitswap v0.2.8 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.4-0.20200624145336-a978cec6e834 - github.com/ipfs/go-cid v0.0.6 + github.com/ipfs/go-cid v0.0.7 github.com/ipfs/go-cidutil v0.0.2 github.com/ipfs/go-datastore v0.4.4 github.com/ipfs/go-ds-badger2 v0.1.1-0.20200708190120-187fc06f714e diff --git a/go.sum b/go.sum index 5bafb5f91..2e1a12640 100644 --- a/go.sum +++ b/go.sum @@ -275,9 +275,8 @@ github.com/filecoin-project/specs-actors v0.7.3-0.20200716231407-60a2ae96d2e6/go github.com/filecoin-project/specs-actors v0.8.1-0.20200720115956-cd051eabf328/go.mod h1:0+CxQ5Jeii3522irTvhKRDpr4GG1bj5Erq3p/d38DzY= github.com/filecoin-project/specs-actors v0.8.1-0.20200723200253-a3c01bc62f99 h1:li6OZVhGNrQihzKhUy7x4vwKgUCExnpVSj746VMkq1I= github.com/filecoin-project/specs-actors v0.8.1-0.20200723200253-a3c01bc62f99/go.mod h1:TLvIheTVl0EIuyncuKSTVXPULaj7gzhLup5CLZ/S+uM= -github.com/filecoin-project/specs-actors v0.8.1-0.20200724015154-3c690d9b7e1d h1:dti6ssgSFG7Tk851S3RdiDr1TNbOJ26ylc6DJ9Y2Le0= -github.com/filecoin-project/specs-actors v0.8.1-0.20200724015154-3c690d9b7e1d/go.mod h1:TLvIheTVl0EIuyncuKSTVXPULaj7gzhLup5CLZ/S+uM= -github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sgOVdJbvFjOnD5w94= +github.com/filecoin-project/specs-actors v0.8.1-0.20200724060953-964aec9ab294 h1:WGSmvWiPZZ8YY8RIepfN/vQQMbaqqMtsCitKBUWX8V4= +github.com/filecoin-project/specs-actors v0.8.1-0.20200724060953-964aec9ab294/go.mod h1:TLvIheTVl0EIuyncuKSTVXPULaj7gzhLup5CLZ/S+uM= github.com/filecoin-project/specs-storage v0.1.0/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea h1:iixjULRQFPn7Q9KlIqfwLJnlAXO10bbkI+xy5GKGdLY= github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= @@ -420,7 +419,6 @@ github.com/gxed/go-shellwords v1.0.3/go.mod h1:N7paucT91ByIjmVJHhvoarjoQnmsi3Jd3 github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE= -github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099 h1:vQqOW42RRM5LoM/1K5dK940VipLqpH8lEVGrMz+mNjU= github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099/go.mod h1:WVPCl0HO/0RAL5+vBH2GMxBomlxBF70MAS78+Lu1//k= github.com/hannahhoward/cbor-gen-for v0.0.0-20200723175505-5892b522820a h1:wfqh5oiHXvn3Rk54xy8Cwqh+HnYihGnjMNzdNb3/ld0= github.com/hannahhoward/cbor-gen-for v0.0.0-20200723175505-5892b522820a/go.mod h1:jvfsLIxk0fY/2BKSQ1xf2406AKA5dwMmKKv0ADcOfN8= @@ -490,8 +488,9 @@ github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10/go.mod h1:/BYOuUoxkE github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/ipfs/go-cid v0.0.6-0.20200501230655-7c82f3b81c00/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= -github.com/ipfs/go-cid v0.0.6 h1:go0y+GcDOGeJIV01FeBsta4FHngoA4Wz7KMeLkXAhMs= github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= +github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY= +github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cidutil v0.0.2 h1:CNOboQf1t7Qp0nuNh8QMmhJs0+Q//bRL1axtCnIB1Yo= github.com/ipfs/go-cidutil v0.0.2/go.mod h1:ewllrvrxG6AMYStla3GD7Cqn+XYSLqjK0vc+086tB6s= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= @@ -660,7 +659,6 @@ github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0 github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= @@ -1127,7 +1125,6 @@ github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJE github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c h1:5bFTChQxSKNwy8ALwOebjekYExl9HTT9urdawqC95tA= github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c/go.mod h1:7qN3Y0BvzRUf4LofcoJplQL10lsFDb4PYlePTVwrP28= -github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -1334,9 +1331,7 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= @@ -1359,9 +1354,7 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200206220010-03c9665e2a66/go.mod h1:X github.com/whyrusleeping/cbor-gen v0.0.0-20200402171437-3d27c146c105/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20200414195334-429a0b5e922e/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20200501232601-351665a6e756/go.mod h1:W5MvapuoHRP8rz4vxjwCK1pDqF1aQcWsV5PZ+AHbqdg= -github.com/whyrusleeping/cbor-gen v0.0.0-20200504204219-64967432584d h1:Y25auOnuZb/GuJvqMflRSDWBz8/HBRME8fiD+H8zLfs= github.com/whyrusleeping/cbor-gen v0.0.0-20200504204219-64967432584d/go.mod h1:W5MvapuoHRP8rz4vxjwCK1pDqF1aQcWsV5PZ+AHbqdg= -github.com/whyrusleeping/cbor-gen v0.0.0-20200710004633-5379fc63235d h1:wSxKhvbN7kUoP0sfRS+w2tWr45qlU8409i94hHLOT8w= github.com/whyrusleeping/cbor-gen v0.0.0-20200710004633-5379fc63235d/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200715143311-227fab5a2377/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200723182808-cb5de1c427f5 h1:dJgLhFKggti1Xd7GczL4DetAUyx68RhpCKCfV71ongg= diff --git a/lib/blockstore/blockstore.go b/lib/blockstore/blockstore.go new file mode 100644 index 000000000..079338041 --- /dev/null +++ b/lib/blockstore/blockstore.go @@ -0,0 +1,63 @@ +// blockstore contains all the basic blockstore constructors used by lotus. Any +// blockstores not ultimately constructed out of the building blocks in this +// package may not work properly. +// +// * This package correctly wraps blockstores with the IdBlockstore. This blockstore: +// * Filters out all puts for blocks with CIDs using the "identity" hash function. +// * Extracts inlined blocks from CIDs using the identity hash function and +// returns them on get/has, ignoring the contents of the blockstore. +// * In the future, this package may enforce additional restrictions on block +// sizes, CID validity, etc. +// +// To make auditing for misuse of blockstores tractable, this package re-exports +// parts of the go-ipfs-blockstore package such that no other package needs to +// import it directly. +package blockstore + +import ( + "context" + + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + + blockstore "github.com/ipfs/go-ipfs-blockstore" +) + +// NewTemporary returns a temporary blockstore. +func NewTemporary() blockstore.Blockstore { + return NewBlockstore(ds.NewMapDatastore()) +} + +// NewTemporary returns a thread-safe temporary blockstore. +func NewTemporarySync() blockstore.Blockstore { + return NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) +} + +// Wraps the underlying blockstore in an "identity" blockstore. +func WrapIDStore(bstore blockstore.Blockstore) blockstore.Blockstore { + return blockstore.NewIdStore(bstore) +} + +// NewBlockstore creates a new blockstore wrapped by the given datastore. +func NewBlockstore(dstore ds.Batching) blockstore.Blockstore { + return WrapIDStore(blockstore.NewBlockstore(dstore)) +} + +// Alias so other packages don't have to import go-ipfs-blockstore +type Blockstore = blockstore.Blockstore +type GCBlockstore = blockstore.GCBlockstore +type CacheOpts = blockstore.CacheOpts +type GCLocker = blockstore.GCLocker + +var NewGCLocker = blockstore.NewGCLocker +var NewGCBlockstore = blockstore.NewGCBlockstore +var DefaultCacheOpts = blockstore.DefaultCacheOpts +var ErrNotFound = blockstore.ErrNotFound + +func CachedBlockstore(ctx context.Context, bs Blockstore, opts CacheOpts) (Blockstore, error) { + bs, err := blockstore.CachedBlockstore(ctx, bs, opts) + if err != nil { + return nil, err + } + return WrapIDStore(bs), nil +} diff --git a/lib/bufbstore/buf_bstore.go b/lib/bufbstore/buf_bstore.go index 4000df7d6..a766c2b52 100644 --- a/lib/bufbstore/buf_bstore.go +++ b/lib/bufbstore/buf_bstore.go @@ -6,9 +6,9 @@ import ( block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - ds "github.com/ipfs/go-datastore" - bstore "github.com/ipfs/go-ipfs-blockstore" logging "github.com/ipfs/go-log/v2" + + bstore "github.com/filecoin-project/lotus/lib/blockstore" ) var log = logging.Logger("bufbs") @@ -19,7 +19,7 @@ type BufferedBS struct { } func NewBufferedBstore(base bstore.Blockstore) *BufferedBS { - buf := bstore.NewBlockstore(ds.NewMapDatastore()) + buf := bstore.NewTemporary() if os.Getenv("LOTUS_DISABLE_VM_BUF") == "iknowitsabadidea" { log.Warn("VM BLOCKSTORE BUFFERING IS DISABLED") buf = base diff --git a/lib/cachebs/cachebs.go b/lib/cachebs/cachebs.go index 5d997137d..0880acc6e 100644 --- a/lib/cachebs/cachebs.go +++ b/lib/cachebs/cachebs.go @@ -6,9 +6,9 @@ import ( lru "github.com/hashicorp/golang-lru" block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" - bstore "github.com/ipfs/go-ipfs-blockstore" logging "github.com/ipfs/go-log/v2" + + bstore "github.com/filecoin-project/lotus/lib/blockstore" ) var log = logging.Logger("cachebs") @@ -18,15 +18,17 @@ type CacheBS struct { bs bstore.Blockstore } -func NewBufferedBstore(base blockstore.Blockstore, size int) *CacheBS { +func NewBufferedBstore(base bstore.Blockstore, size int) bstore.Blockstore { c, err := lru.NewARC(size) if err != nil { panic(err) } - return &CacheBS{ + // Wrap this in an ID blockstore to avoid caching blocks inlined into + // CIDs. + return bstore.WrapIDStore(&CacheBS{ cache: c, bs: base, - } + }) } var _ (bstore.Blockstore) = &CacheBS{} diff --git a/lib/ipfsbstore/ipfsbstore.go b/lib/ipfsbstore/ipfsbstore.go index ce756ccbb..748afee51 100644 --- a/lib/ipfsbstore/ipfsbstore.go +++ b/lib/ipfsbstore/ipfsbstore.go @@ -12,11 +12,12 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" httpapi "github.com/ipfs/go-ipfs-http-client" iface "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" + + "github.com/filecoin-project/lotus/lib/blockstore" ) type IpfsBstore struct { diff --git a/node/builder.go b/node/builder.go index 46fd260d5..6b0ea8562 100644 --- a/node/builder.go +++ b/node/builder.go @@ -5,7 +5,6 @@ import ( "errors" "time" - blockstore "github.com/ipfs/go-ipfs-blockstore" logging "github.com/ipfs/go-log" ci "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/host" @@ -40,6 +39,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/lib/peermgr" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index bbbd989b7..53495bde4 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -9,12 +9,11 @@ import ( "strings" "sync" - "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/filecoin-project/specs-actors/actors/crypto" - "github.com/filecoin-project/specs-actors/actors/util/adt" + "go.uber.org/fx" + "golang.org/x/xerrors" + "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" offline "github.com/ipfs/go-ipfs-exchange-offline" cbor "github.com/ipfs/go-ipld-cbor" ipld "github.com/ipfs/go-ipld-format" @@ -24,15 +23,17 @@ import ( "github.com/ipfs/go-path/resolver" mh "github.com/multiformats/go-multihash" cbg "github.com/whyrusleeping/cbor-gen" - "go.uber.org/fx" - "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/crypto" + "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/lib/blockstore" ) var log = logging.Logger("fullnode") diff --git a/node/modules/chain.go b/node/modules/chain.go index 380c13fe7..23f2b43fa 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -3,13 +3,11 @@ package modules import ( "bytes" "context" - "github.com/filecoin-project/lotus/chain/vm" "github.com/ipfs/go-bitswap" "github.com/ipfs/go-bitswap/network" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" - blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipld/go-car" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/routing" @@ -26,6 +24,8 @@ import ( "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" @@ -72,7 +72,7 @@ func ChainBlockstore(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo return nil, err } - return blockstore.NewIdStore(cbs), nil + return cbs, nil } func ChainGCBlockstore(bs dtypes.ChainBlockstore, gcl dtypes.ChainGCLocker) dtypes.ChainGCBlockstore { diff --git a/node/modules/client.go b/node/modules/client.go index a24709beb..eda881ba8 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -4,12 +4,8 @@ import ( "context" "time" - "github.com/filecoin-project/lotus/lib/bufbstore" - "golang.org/x/xerrors" - - blockstore "github.com/ipfs/go-ipfs-blockstore" - "github.com/libp2p/go-libp2p-core/host" "go.uber.org/fx" + "golang.org/x/xerrors" dtimpl "github.com/filecoin-project/go-data-transfer/impl" dtnet "github.com/filecoin-project/go-data-transfer/network" @@ -26,7 +22,10 @@ import ( "github.com/filecoin-project/go-storedcounter" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" + "github.com/libp2p/go-libp2p-core/host" + "github.com/filecoin-project/lotus/lib/blockstore" + "github.com/filecoin-project/lotus/lib/bufbstore" "github.com/filecoin-project/lotus/markets/retrievaladapter" "github.com/filecoin-project/lotus/node/impl/full" payapi "github.com/filecoin-project/lotus/node/impl/paych" @@ -64,9 +63,9 @@ func ClientBlockstore(imgr dtypes.ClientImportMgr) dtypes.ClientBlockstore { // TODO: This isn't.. the best // - If it's easy to pass per-retrieval blockstores with markets we don't need this // - If it's not easy, we need to store this in a separate datastore on disk - defaultWrite := blockstore.NewBlockstore(datastore.NewMapDatastore()) + defaultWrite := blockstore.NewTemporary() - return blockstore.NewIdStore(bufbstore.NewTieredBstore(imgr.Blockstore, defaultWrite)) + return bufbstore.NewTieredBstore(imgr.Blockstore, defaultWrite) } // RegisterClientValidator is an initialization hook that registers the client diff --git a/node/modules/dtypes/storage.go b/node/modules/dtypes/storage.go index 7ede8aaab..b19905e14 100644 --- a/node/modules/dtypes/storage.go +++ b/node/modules/dtypes/storage.go @@ -4,7 +4,6 @@ import ( bserv "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" "github.com/ipfs/go-graphsync" - blockstore "github.com/ipfs/go-ipfs-blockstore" exchange "github.com/ipfs/go-ipfs-exchange-interface" format "github.com/ipfs/go-ipld-format" @@ -13,6 +12,8 @@ import ( datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-statestore" + + "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/node/repo/importmgr" ) diff --git a/node/modules/ipfsclient.go b/node/modules/ipfsclient.go index 93bedf598..c672ca1cf 100644 --- a/node/modules/ipfsclient.go +++ b/node/modules/ipfsclient.go @@ -4,9 +4,9 @@ import ( "go.uber.org/fx" "golang.org/x/xerrors" - blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/multiformats/go-multiaddr" + "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/lib/bufbstore" "github.com/filecoin-project/lotus/lib/ipfsbstore" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -20,7 +20,7 @@ import ( func IpfsClientBlockstore(ipfsMaddr string, useForRetrieval bool) func(helpers.MetricsCtx, fx.Lifecycle, dtypes.ClientImportMgr) (dtypes.ClientBlockstore, error) { return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, localStore dtypes.ClientImportMgr) (dtypes.ClientBlockstore, error) { var err error - var ipfsbs *ipfsbstore.IpfsBstore + var ipfsbs blockstore.Blockstore if ipfsMaddr != "" { var ma multiaddr.Multiaddr ma, err = multiaddr.NewMultiaddr(ipfsMaddr) @@ -34,10 +34,11 @@ func IpfsClientBlockstore(ipfsMaddr string, useForRetrieval bool) func(helpers.M if err != nil { return nil, xerrors.Errorf("constructing ipfs blockstore: %w", err) } + ipfsbs = blockstore.WrapIDStore(ipfsbs) var ws blockstore.Blockstore ws = ipfsbs if !useForRetrieval { - ws = blockstore.NewIdStore(localStore.Blockstore) + ws = blockstore.WrapIDStore(localStore.Blockstore) } return bufbstore.NewTieredBstore(ipfsbs, ws), nil } diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 1a22fdd95..35002ed15 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -7,6 +7,10 @@ import ( "net/http" "time" + "go.uber.org/fx" + "go.uber.org/multierr" + "golang.org/x/xerrors" + "github.com/ipfs/go-bitswap" "github.com/ipfs/go-bitswap/network" "github.com/ipfs/go-blockservice" @@ -16,13 +20,9 @@ import ( graphsync "github.com/ipfs/go-graphsync/impl" gsnet "github.com/ipfs/go-graphsync/network" "github.com/ipfs/go-graphsync/storeutil" - blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-merkledag" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/routing" - "go.uber.org/fx" - "go.uber.org/multierr" - "golang.org/x/xerrors" "github.com/filecoin-project/go-address" dtimpl "github.com/filecoin-project/go-data-transfer/impl" @@ -42,7 +42,6 @@ import ( paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/go-storedcounter" - "github.com/filecoin-project/lotus/node/config" sectorstorage "github.com/filecoin-project/sector-storage" "github.com/filecoin-project/sector-storage/ffiwrapper" "github.com/filecoin-project/sector-storage/stores" @@ -53,8 +52,10 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/markets/retrievaladapter" "github.com/filecoin-project/lotus/miner" + "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" @@ -252,10 +253,7 @@ func StagingBlockstore(r repo.LockedRepo) (dtypes.StagingBlockstore, error) { return nil, err } - bs := blockstore.NewBlockstore(stagingds) - ibs := blockstore.NewIdStore(bs) - - return ibs, nil + return blockstore.NewBlockstore(stagingds), nil } // StagingDAG is a DAGService for the StagingBlockstore diff --git a/node/repo/importmgr/mbstore.go b/node/repo/importmgr/mbstore.go index 3b6058bee..e93bafeac 100644 --- a/node/repo/importmgr/mbstore.go +++ b/node/repo/importmgr/mbstore.go @@ -8,7 +8,8 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" + + "github.com/filecoin-project/lotus/lib/blockstore" ) type multiReadBs struct { diff --git a/node/repo/importmgr/mgr.go b/node/repo/importmgr/mgr.go index d3139918e..1a8e117a7 100644 --- a/node/repo/importmgr/mgr.go +++ b/node/repo/importmgr/mgr.go @@ -8,7 +8,8 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" - blockstore "github.com/ipfs/go-ipfs-blockstore" + + "github.com/filecoin-project/lotus/lib/blockstore" ) type Mgr struct { diff --git a/node/repo/importmgr/store.go b/node/repo/importmgr/store.go index 78bb7462b..bfe515776 100644 --- a/node/repo/importmgr/store.go +++ b/node/repo/importmgr/store.go @@ -5,10 +5,11 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" "github.com/ipfs/go-filestore" - blockstore "github.com/ipfs/go-ipfs-blockstore" offline "github.com/ipfs/go-ipfs-exchange-offline" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" + + "github.com/filecoin-project/lotus/lib/blockstore" ) type Store struct { @@ -31,7 +32,7 @@ func openStore(ds datastore.Batching) (*Store, error) { fm.AllowFiles = true fstore := filestore.NewFilestore(bs, fm) - ibs := blockstore.NewIdStore(fstore) + ibs := blockstore.WrapIDStore(fstore) bsvc := blockservice.New(ibs, offline.Exchange(ibs)) dag := merkledag.NewDAGService(bsvc)