106 lines
2.8 KiB
Go
106 lines
2.8 KiB
Go
package paths
|
|
|
|
import (
|
|
"context"
|
|
"io"
|
|
"mime"
|
|
"net/http"
|
|
"os"
|
|
"time"
|
|
|
|
"github.com/hashicorp/go-multierror"
|
|
"golang.org/x/xerrors"
|
|
|
|
"github.com/filecoin-project/lotus/storage/sealer/tarutil"
|
|
)
|
|
|
|
func fetch(ctx context.Context, url, outname string, header http.Header) (rerr error) {
|
|
log.Infof("Fetch %s -> %s", url, outname)
|
|
|
|
req, err := http.NewRequest("GET", url, nil)
|
|
if err != nil {
|
|
return xerrors.Errorf("request: %w", err)
|
|
}
|
|
req.Header = header
|
|
req = req.WithContext(ctx)
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return xerrors.Errorf("do request: %w", err)
|
|
}
|
|
defer resp.Body.Close() // nolint
|
|
|
|
if resp.StatusCode != 200 {
|
|
return xerrors.Errorf("non-200 code: %d", resp.StatusCode)
|
|
}
|
|
|
|
start := time.Now()
|
|
var bytes int64
|
|
defer func() {
|
|
took := time.Now().Sub(start)
|
|
mibps := float64(bytes) / 1024 / 1024 * float64(time.Second) / float64(took)
|
|
log.Infow("Fetch done", "url", url, "out", outname, "took", took.Round(time.Millisecond), "bytes", bytes, "MiB/s", mibps, "err", rerr)
|
|
}()
|
|
|
|
mediatype, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
return xerrors.Errorf("parse media type: %w", err)
|
|
}
|
|
|
|
if err := os.RemoveAll(outname); err != nil {
|
|
return xerrors.Errorf("removing dest: %w", err)
|
|
}
|
|
|
|
switch mediatype {
|
|
case "application/x-tar":
|
|
bytes, err = tarutil.ExtractTar(resp.Body, outname, make([]byte, CopyBuf))
|
|
return err
|
|
case "application/octet-stream":
|
|
f, err := os.Create(outname)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
bytes, err = io.CopyBuffer(f, resp.Body, make([]byte, CopyBuf))
|
|
if err != nil {
|
|
f.Close() // nolint
|
|
return err
|
|
}
|
|
return f.Close()
|
|
default:
|
|
return xerrors.Errorf("unknown content type: '%s'", mediatype)
|
|
}
|
|
}
|
|
|
|
// FetchWithTemp fetches data into a temp 'fetching' directory, then moves the file to destination
|
|
// The set of URLs must refer to the same object, if one fails, another one will be tried.
|
|
func FetchWithTemp(ctx context.Context, urls []string, dest string, header http.Header) (string, error) {
|
|
var merr error
|
|
for _, url := range urls {
|
|
tempDest, err := tempFetchDest(dest, true)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if err := os.RemoveAll(dest); err != nil {
|
|
return "", xerrors.Errorf("removing dest: %w", err)
|
|
}
|
|
|
|
err = fetch(ctx, url, tempDest, header)
|
|
if err != nil {
|
|
merr = multierror.Append(merr, xerrors.Errorf("fetch error %s -> %s: %w", url, tempDest, err))
|
|
continue
|
|
}
|
|
|
|
if err := Move(tempDest, dest); err != nil {
|
|
return "", xerrors.Errorf("fetch move error %s -> %s: %w", tempDest, dest, err)
|
|
}
|
|
|
|
if merr != nil {
|
|
log.Warnw("acquireFromRemote encountered errors when fetching sector from remote", "errors", merr)
|
|
}
|
|
return url, nil
|
|
}
|
|
|
|
return "", xerrors.Errorf("failed to fetch sector file (tried %v): %w", urls, merr)
|
|
}
|