add batching to vm Copy call

This commit is contained in:
whyrusleeping 2019-11-17 11:50:04 -06:00
parent 9269c04b21
commit cac44367aa

View File

@ -549,6 +549,32 @@ func linksForObj(blk block.Block) ([]cid.Cid, error) {
} }
func Copy(from, to blockstore.Blockstore, root cid.Cid) error { func Copy(from, to blockstore.Blockstore, root cid.Cid) error {
var batch []block.Block
batchCp := func(blk block.Block) error {
batch = append(batch, blk)
if len(batch) > 100 {
if err := to.PutMany(batch); err != nil {
return xerrors.Errorf("batch put in copy: %w", err)
}
batch = batch[:0]
}
return nil
}
if err := copyRec(from, to, root, batchCp); err != nil {
return err
}
if len(batch) > 0 {
if err := to.PutMany(batch); err != nil {
return xerrors.Errorf("batch put in copy: %w", err)
}
}
return nil
}
func copyRec(from, to blockstore.Blockstore, root cid.Cid, cp func(block.Block) error) error {
if root.Prefix().MhType == 0 { if root.Prefix().MhType == 0 {
// identity cid, skip // identity cid, skip
return nil return nil
@ -577,12 +603,12 @@ func Copy(from, to blockstore.Blockstore, root cid.Cid) error {
continue continue
} }
if err := Copy(from, to, link); err != nil { if err := copyRec(from, to, link, cp); err != nil {
return err return err
} }
} }
if err := to.Put(blk); err != nil { if err := cp(blk); err != nil {
return err return err
} }
return nil return nil