cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #3533 from filecoin-project/feat/fast-export

Browse Source 
Make chain export ~1000x times faster
This commit is contained in:
Łukasz Magiera 2020-09-03 22:47:20 +02:00 committed by GitHub
commit 9151f8b492
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
node/impl/full/chain.go
View File

@ -1,6 +1,7 @@
package full
import (
"bufio"
"bytes"
"context"
"encoding/json"
@ -503,7 +504,11 @@ func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk t
out := make(chan []byte)
go func() {
defer w.Close() //nolint:errcheck // it is a pipe
if err := a.Chain.Export(ctx, ts, nroots, w); err != nil {
bw := bufio.NewWriterSize(w, 1<<20)
defer bw.Flush() //nolint:errcheck // it is a write to a pipe
if err := a.Chain.Export(ctx, ts, nroots, bw); err != nil {
log.Errorf("chain export call failed: %s", err)
return
}
@ -512,7 +517,7 @@ func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk t
go func() {
defer close(out)
for {
buf := make([]byte, 4096)
buf := make([]byte, 1<<20)
n, err := r.Read(buf)
if err != nil && err != io.EOF {
log.Errorf("chain export pipe read failed: %s", err)
@ -522,6 +527,7 @@ func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk t
case out <- buf[:n]:
case <-ctx.Done():
log.Warnf("export writer failed: %s", ctx.Err())
return
}
if err == io.EOF {
return