cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #3533 from filecoin-project/feat/fast-export

Browse Source 
Make chain export ~1000x times faster
This commit is contained in:
Łukasz Magiera 2020-09-03 22:47:20 +02:00 committed by GitHub
commit 9151f8b492
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
node/impl/full/chain.go
View File

@ -1,6 +1,7 @@
package full package full
import ( import (
"bufio"
"bytes" "bytes"
"context" "context"
"encoding/json" "encoding/json"
@ -503,7 +504,11 @@ func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk t
out := make(chan []byte) out := make(chan []byte)
go func() { go func() {
defer w.Close() //nolint:errcheck // it is a pipe defer w.Close() //nolint:errcheck // it is a pipe
if err := a.Chain.Export(ctx, ts, nroots, w); err != nil {
bw := bufio.NewWriterSize(w, 1<<20)
defer bw.Flush() //nolint:errcheck // it is a write to a pipe
if err := a.Chain.Export(ctx, ts, nroots, bw); err != nil {
log.Errorf("chain export call failed: %s", err) log.Errorf("chain export call failed: %s", err)
return return
} }
@ -512,7 +517,7 @@ func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk t
go func() { go func() {
defer close(out) defer close(out)
for { for {
buf := make([]byte, 4096) buf := make([]byte, 1<<20)
n, err := r.Read(buf) n, err := r.Read(buf)
if err != nil && err != io.EOF { if err != nil && err != io.EOF {
log.Errorf("chain export pipe read failed: %s", err) log.Errorf("chain export pipe read failed: %s", err)
@ -522,6 +527,7 @@ func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk t
case out <- buf[:n]: case out <- buf[:n]:
case <-ctx.Done(): case <-ctx.Done():
log.Warnf("export writer failed: %s", ctx.Err()) log.Warnf("export writer failed: %s", ctx.Err())
return
} }
if err == io.EOF { if err == io.EOF {
return return