cerc-io/lotus
15
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #7681 from filecoin-project/feat/storage-tar-buf

Browse Source 
storage: Use 1M buffers for Tar transfers
This commit is contained in:
Łukasz Magiera 2021-11-24 20:52:02 +01:00 committed by GitHub
commit ab55a6fbbe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 9 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
extern/sector-storage/stores/http_handler.go vendored
View File

@ -2,7 +2,6 @@ package stores
import (
"encoding/json"
"io"
"net/http"
"os"
"strconv"
@ -139,17 +138,12 @@ func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Requ
return
}
rd, err := tarutil.TarDirectory(path)
if err != nil {
log.Errorf("%+v", err)
w.WriteHeader(500)
return
}
w.Header().Set("Content-Type", "application/x-tar")
w.WriteHeader(200)
if _, err := io.CopyBuffer(w, rd, make([]byte, CopyBuf)); err != nil {
log.Errorf("%+v", err)
err := tarutil.TarDirectory(path, w, make([]byte, CopyBuf))
if err != nil {
log.Errorf("send tar: %+v", err)
return
}
} else {

2
extern/sector-storage/stores/remote.go vendored
View File

@ -281,7 +281,7 @@ func (r *Remote) fetch(ctx context.Context, url, outname string) error {
switch mediatype {
case "application/x-tar":
return tarutil.ExtractTar(resp.Body, outname)
return tarutil.ExtractTar(resp.Body, outname, make([]byte, CopyBuf))
case "application/octet-stream":
f, err := os.Create(outname)
if err != nil {

18
extern/sector-storage/tarutil/systar.go vendored
View File

@ -14,7 +14,7 @@ import (
var log = logging.Logger("tarutil") // nolint
func ExtractTar(body io.Reader, dir string) error {
func ExtractTar(body io.Reader, dir string, buf []byte) error {
if err := os.MkdirAll(dir, 0755); err != nil { // nolint
return xerrors.Errorf("mkdir: %w", err)
}
@ -38,7 +38,7 @@ func ExtractTar(body io.Reader, dir string) error {
// This data is coming from a trusted source, no need to check the size.
//nolint:gosec
if _, err := io.Copy(f, tr); err != nil {
if _, err := io.CopyBuffer(f, tr, buf); err != nil {
return err
}
@ -48,17 +48,7 @@ func ExtractTar(body io.Reader, dir string) error {
}
}
func TarDirectory(dir string) (io.ReadCloser, error) {
r, w := io.Pipe()
go func() {
_ = w.CloseWithError(writeTarDirectory(dir, w))
}()
return r, nil
}
func writeTarDirectory(dir string, w io.Writer) error {
func TarDirectory(dir string, w io.Writer, buf []byte) error {
tw := tar.NewWriter(w)
files, err := ioutil.ReadDir(dir)
@ -81,7 +71,7 @@ func writeTarDirectory(dir string, w io.Writer) error {
return xerrors.Errorf("opening %s for reading: %w", file.Name(), err)
}
if _, err := io.Copy(tw, f); err != nil {
if _, err := io.CopyBuffer(tw, f, buf); err != nil {
return xerrors.Errorf("copy data for file %s: %w", file.Name(), err)
}