cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

don't include not-yet-accessed blocks in car generation

Browse Source
This commit is contained in:
Will Scott 2020-09-16 12:22:38 -07:00
parent 77a956cda4
commit ed7423b2b5
2 changed files with 25 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
tvx/extract_msg.go
View File

@ -184,6 +184,16 @@ func runExtractMsg(c *cli.Context) error {
return err return err
} }
// don't fetch additional content that wasn't accessed yet during car spidering / generation.
type onlineblockstore interface {
SetOnline(bool)
}
if ob, ok := pst.Blockstore.(onlineblockstore); ok {
ob.SetOnline(false)
}
out := new(bytes.Buffer) out := new(bytes.Buffer)
gw := gzip.NewWriter(out) gw := gzip.NewWriter(out)
if err := g.WriteCAR(gw, preroot, postroot); err != nil { if err := g.WriteCAR(gw, preroot, postroot); err != nil {
@ -209,7 +219,7 @@ func runExtractMsg(c *cli.Context) error {
ID: "TK", ID: "TK",
Version: "TK", Version: "TK",
Gen: []schema.GenerationData{schema.GenerationData{ Gen: []schema.GenerationData{schema.GenerationData{
Source: "TK", Source: msg.Cid().String(),
Version: version.String(), Version: version.String(),
}}, }},
}, },

15
tvx/state/store.go
View File

@ -2,6 +2,7 @@ package state
import ( import (
"context" "context"
"sync"
"github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/lib/blockstore"
@ -63,13 +64,18 @@ type proxyingBlockstore struct {
ctx context.Context ctx context.Context
api api.FullNode api api.FullNode
online bool
lock sync.RWMutex
blockstore.Blockstore blockstore.Blockstore
} }
func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) { func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
if block, err := pb.Blockstore.Get(cid); err == nil { pb.lock.RLock()
if block, err := pb.Blockstore.Get(cid); (err == nil || !pb.online) {
pb.lock.RUnlock();
return block, err return block, err
} }
pb.lock.RUnlock();
// fmt.Printf("fetching cid via rpc: %v\n", cid) // fmt.Printf("fetching cid via rpc: %v\n", cid)
item, err := pb.api.ChainReadObj(pb.ctx, cid) item, err := pb.api.ChainReadObj(pb.ctx, cid)
@ -81,6 +87,8 @@ func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
return nil, err return nil, err
} }
pb.lock.Lock();
defer pb.lock.Unlock();
err = pb.Blockstore.Put(block) err = pb.Blockstore.Put(block)
if err != nil { if err != nil {
return nil, err return nil, err
@ -89,6 +97,10 @@ func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
return block, nil return block, nil
} }
func (pb *proxyingBlockstore) SetOnline(online bool) {
pb.online = online
}
// NewProxyingStore is a Stores that proxies get requests for unknown CIDs // NewProxyingStore is a Stores that proxies get requests for unknown CIDs
// to a Filecoin node, via the ChainReadObj RPC. // to a Filecoin node, via the ChainReadObj RPC.
func NewProxyingStore(ctx context.Context, api api.FullNode) *Stores { func NewProxyingStore(ctx context.Context, api api.FullNode) *Stores {
@ -97,6 +109,7 @@ func NewProxyingStore(ctx context.Context, api api.FullNode) *Stores {
bs := &proxyingBlockstore{ bs := &proxyingBlockstore{
ctx: ctx, ctx: ctx,
api: api, api: api,
online: true,
Blockstore: blockstore.NewBlockstore(ds), Blockstore: blockstore.NewBlockstore(ds),
} }