cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

don't include not-yet-accessed blocks in car generation

Browse Source
This commit is contained in:
Will Scott 2020-09-16 12:22:38 -07:00
parent 77a956cda4
commit ed7423b2b5
2 changed files with 25 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
tvx/extract_msg.go
View File

@ -184,6 +184,16 @@ func runExtractMsg(c *cli.Context) error {
return err
}
// don't fetch additional content that wasn't accessed yet during car spidering / generation.
type onlineblockstore interface {
SetOnline(bool)
}
if ob, ok := pst.Blockstore.(onlineblockstore); ok {
ob.SetOnline(false)
}
out := new(bytes.Buffer)
gw := gzip.NewWriter(out)
if err := g.WriteCAR(gw, preroot, postroot); err != nil {
@ -209,7 +219,7 @@ func runExtractMsg(c *cli.Context) error {
ID: "TK",
Version: "TK",
Gen: []schema.GenerationData{schema.GenerationData{
Source: "TK",
Source: msg.Cid().String(),
Version: version.String(),
}},
},

15
tvx/state/store.go
View File

@ -2,6 +2,7 @@ package state
import (
"context"
"sync"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/blockstore"
@ -63,13 +64,18 @@ type proxyingBlockstore struct {
ctx context.Context
api api.FullNode
online bool
lock sync.RWMutex
blockstore.Blockstore
}
func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
if block, err := pb.Blockstore.Get(cid); err == nil {
pb.lock.RLock()
if block, err := pb.Blockstore.Get(cid); (err == nil || !pb.online) {
pb.lock.RUnlock();
return block, err
}
pb.lock.RUnlock();
// fmt.Printf("fetching cid via rpc: %v\n", cid)
item, err := pb.api.ChainReadObj(pb.ctx, cid)
@ -81,6 +87,8 @@ func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
return nil, err
}
pb.lock.Lock();
defer pb.lock.Unlock();
err = pb.Blockstore.Put(block)
if err != nil {
return nil, err
@ -89,6 +97,10 @@ func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
return block, nil
}
func (pb *proxyingBlockstore) SetOnline(online bool) {
pb.online = online
}
// NewProxyingStore is a Stores that proxies get requests for unknown CIDs
// to a Filecoin node, via the ChainReadObj RPC.
func NewProxyingStore(ctx context.Context, api api.FullNode) *Stores {
@ -97,6 +109,7 @@ func NewProxyingStore(ctx context.Context, api api.FullNode) *Stores {
bs := &proxyingBlockstore{
ctx: ctx,
api: api,
online: true,
Blockstore: blockstore.NewBlockstore(ds),
}