164 lines
4.6 KiB
Go
164 lines
4.6 KiB
Go
package tarutil
|
|
|
|
import (
|
|
"archive/tar"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
logging "github.com/ipfs/go-log/v2"
|
|
"golang.org/x/xerrors"
|
|
)
|
|
|
|
var log = logging.Logger("tarutil") // nolint
|
|
|
|
var CacheFileConstraints = map[string]int64{
|
|
"p_aux": 64,
|
|
"t_aux": 10240,
|
|
|
|
"sc-02-data-tree-r-last.dat": 10_000_000, // small sectors
|
|
|
|
"sc-02-data-tree-r-last-0.dat": 10_000_000,
|
|
"sc-02-data-tree-r-last-1.dat": 10_000_000,
|
|
"sc-02-data-tree-r-last-2.dat": 10_000_000,
|
|
"sc-02-data-tree-r-last-3.dat": 10_000_000,
|
|
"sc-02-data-tree-r-last-4.dat": 10_000_000,
|
|
"sc-02-data-tree-r-last-5.dat": 10_000_000,
|
|
"sc-02-data-tree-r-last-6.dat": 10_000_000,
|
|
"sc-02-data-tree-r-last-7.dat": 10_000_000,
|
|
|
|
"sc-02-data-tree-r-last-8.dat": 10_000_000,
|
|
"sc-02-data-tree-r-last-9.dat": 10_000_000,
|
|
"sc-02-data-tree-r-last-10.dat": 10_000_000,
|
|
"sc-02-data-tree-r-last-11.dat": 10_000_000,
|
|
"sc-02-data-tree-r-last-12.dat": 10_000_000,
|
|
"sc-02-data-tree-r-last-13.dat": 10_000_000,
|
|
"sc-02-data-tree-r-last-14.dat": 10_000_000,
|
|
"sc-02-data-tree-r-last-15.dat": 10_000_000,
|
|
|
|
"sc-02-data-layer-1.dat": 65 << 30, // 1x sector size + small buffer
|
|
"sc-02-data-layer-2.dat": 65 << 30, // 1x sector size + small buffer
|
|
"sc-02-data-layer-3.dat": 65 << 30, // 1x sector size + small buffer
|
|
"sc-02-data-layer-4.dat": 65 << 30, // 1x sector size + small buffer
|
|
"sc-02-data-layer-5.dat": 65 << 30, // 1x sector size + small buffer
|
|
"sc-02-data-layer-6.dat": 65 << 30, // 1x sector size + small buffer
|
|
"sc-02-data-layer-7.dat": 65 << 30, // 1x sector size + small buffer
|
|
"sc-02-data-layer-8.dat": 65 << 30, // 1x sector size + small buffer
|
|
"sc-02-data-layer-9.dat": 65 << 30, // 1x sector size + small buffer
|
|
"sc-02-data-layer-10.dat": 65 << 30, // 1x sector size + small buffer
|
|
"sc-02-data-layer-11.dat": 65 << 30, // 1x sector size + small buffer
|
|
|
|
"sc-02-data-tree-c-0.dat": 5 << 30, // ~4.6G
|
|
"sc-02-data-tree-c-1.dat": 5 << 30,
|
|
"sc-02-data-tree-c-2.dat": 5 << 30,
|
|
"sc-02-data-tree-c-3.dat": 5 << 30,
|
|
"sc-02-data-tree-c-4.dat": 5 << 30,
|
|
"sc-02-data-tree-c-5.dat": 5 << 30,
|
|
"sc-02-data-tree-c-6.dat": 5 << 30,
|
|
"sc-02-data-tree-c-7.dat": 5 << 30,
|
|
|
|
"sc-02-data-tree-c-8.dat": 5 << 30,
|
|
"sc-02-data-tree-c-9.dat": 5 << 30,
|
|
"sc-02-data-tree-c-10.dat": 5 << 30,
|
|
"sc-02-data-tree-c-11.dat": 5 << 30,
|
|
"sc-02-data-tree-c-12.dat": 5 << 30,
|
|
"sc-02-data-tree-c-13.dat": 5 << 30,
|
|
"sc-02-data-tree-c-14.dat": 5 << 30,
|
|
"sc-02-data-tree-c-15.dat": 5 << 30,
|
|
|
|
"sc-02-data-tree-d.dat": 130 << 30, // 2x sector size, ~130G accunting for small buffer on 64G sectors
|
|
}
|
|
|
|
func ExtractTar(body io.Reader, dir string, buf []byte) (int64, error) {
|
|
if err := os.MkdirAll(dir, 0755); err != nil { // nolint
|
|
return 0, xerrors.Errorf("mkdir: %w", err)
|
|
}
|
|
|
|
tr := tar.NewReader(body)
|
|
var read int64
|
|
for {
|
|
header, err := tr.Next()
|
|
switch err {
|
|
default:
|
|
return read, err
|
|
case io.EOF:
|
|
return read, nil
|
|
|
|
case nil:
|
|
}
|
|
|
|
sz, found := CacheFileConstraints[header.Name]
|
|
if !found {
|
|
return read, xerrors.Errorf("tar file %#v isn't expected")
|
|
}
|
|
if header.Size > sz {
|
|
return read, xerrors.Errorf("tar file %#v is bigger than expected: %d > %d", header.Name, header.Size, sz)
|
|
}
|
|
|
|
out := filepath.Join(dir, header.Name) //nolint:gosec
|
|
|
|
if !strings.HasPrefix(out, filepath.Clean(dir)) {
|
|
return read, xerrors.Errorf("unsafe tar path %#v (must be within %#v)", out, filepath.Clean(dir))
|
|
}
|
|
|
|
f, err := os.Create(out)
|
|
if err != nil {
|
|
return read, xerrors.Errorf("creating file %s: %w", out, err)
|
|
}
|
|
|
|
ltr := io.LimitReader(tr, header.Size)
|
|
|
|
r, err := io.CopyBuffer(f, ltr, buf)
|
|
read += r
|
|
if err != nil {
|
|
return read, err
|
|
}
|
|
|
|
if err := f.Close(); err != nil {
|
|
return read, err
|
|
}
|
|
}
|
|
}
|
|
|
|
func TarDirectory(dir string, w io.Writer, buf []byte) error {
|
|
tw := tar.NewWriter(w)
|
|
|
|
files, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, file := range files {
|
|
info, err := file.Info()
|
|
if err != nil {
|
|
return xerrors.Errorf("getting file info for file %s: %w", file.Name(), err)
|
|
}
|
|
|
|
h, err := tar.FileInfoHeader(info, "")
|
|
if err != nil {
|
|
return xerrors.Errorf("getting header for file %s: %w", file.Name(), err)
|
|
}
|
|
|
|
if err := tw.WriteHeader(h); err != nil {
|
|
return xerrors.Errorf("wiritng header for file %s: %w", file.Name(), err)
|
|
}
|
|
|
|
f, err := os.OpenFile(filepath.Join(dir, file.Name()), os.O_RDONLY, 644) // nolint
|
|
if err != nil {
|
|
return xerrors.Errorf("opening %s for reading: %w", file.Name(), err)
|
|
}
|
|
|
|
if _, err := io.CopyBuffer(tw, f, buf); err != nil {
|
|
return xerrors.Errorf("copy data for file %s: %w", file.Name(), err)
|
|
}
|
|
|
|
if err := f.Close(); err != nil {
|
|
return err
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
}
|