cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

tarutil: Strict header checks

Browse Source
This commit is contained in:
Łukasz Magiera 2022-09-09 13:31:03 +02:00
parent c17ab761c5
commit 474c60203b
2 changed files with 66 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
storage/sealer/storiface/storage.go
View File

@ -131,7 +131,7 @@ type SectorData struct {
// URL to the sector data // URL to the sector data
// For sealed/unsealed sector, lotus expects octet-stream // For sealed/unsealed sector, lotus expects octet-stream
// For cache, lotus expects a tar archive with cache files (todo maybe use not-tar; specify what files with what paths must be present) // For cache, lotus expects a tar archive with cache files
// Valid schemas: // Valid schemas:
// - http:// / https:// // - http:// / https://
URL string URL string

68
storage/sealer/tarutil/systar.go
View File

@ -12,6 +12,63 @@ import (
var log = logging.Logger("tarutil") // nolint var log = logging.Logger("tarutil") // nolint
var CacheFileConstraints = map[string]int64{
"p_aux": 64,
"t_aux": 10240,
"sc-02-data-tree-r-last.dat": 10_000_000, // small sectors
"sc-02-data-tree-r-last-0.dat": 10_000_000,
"sc-02-data-tree-r-last-1.dat": 10_000_000,
"sc-02-data-tree-r-last-2.dat": 10_000_000,
"sc-02-data-tree-r-last-3.dat": 10_000_000,
"sc-02-data-tree-r-last-4.dat": 10_000_000,
"sc-02-data-tree-r-last-5.dat": 10_000_000,
"sc-02-data-tree-r-last-6.dat": 10_000_000,
"sc-02-data-tree-r-last-7.dat": 10_000_000,
"sc-02-data-tree-r-last-8.dat": 10_000_000,
"sc-02-data-tree-r-last-9.dat": 10_000_000,
"sc-02-data-tree-r-last-10.dat": 10_000_000,
"sc-02-data-tree-r-last-11.dat": 10_000_000,
"sc-02-data-tree-r-last-12.dat": 10_000_000,
"sc-02-data-tree-r-last-13.dat": 10_000_000,
"sc-02-data-tree-r-last-14.dat": 10_000_000,
"sc-02-data-tree-r-last-15.dat": 10_000_000,
"sc-02-data-layer-1.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-2.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-3.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-4.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-5.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-6.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-7.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-8.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-9.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-10.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-11.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-tree-c-0.dat": 5 << 30, // ~4.6G
"sc-02-data-tree-c-1.dat": 5 << 30,
"sc-02-data-tree-c-2.dat": 5 << 30,
"sc-02-data-tree-c-3.dat": 5 << 30,
"sc-02-data-tree-c-4.dat": 5 << 30,
"sc-02-data-tree-c-5.dat": 5 << 30,
"sc-02-data-tree-c-6.dat": 5 << 30,
"sc-02-data-tree-c-7.dat": 5 << 30,
"sc-02-data-tree-c-8.dat": 5 << 30,
"sc-02-data-tree-c-9.dat": 5 << 30,
"sc-02-data-tree-c-10.dat": 5 << 30,
"sc-02-data-tree-c-11.dat": 5 << 30,
"sc-02-data-tree-c-12.dat": 5 << 30,
"sc-02-data-tree-c-13.dat": 5 << 30,
"sc-02-data-tree-c-14.dat": 5 << 30,
"sc-02-data-tree-c-15.dat": 5 << 30,
"sc-02-data-tree-d.dat": 130 << 30, // 2x sector size, ~130G accunting for small buffer on 64G sectors
}
func ExtractTar(body io.Reader, dir string, buf []byte) (int64, error) { func ExtractTar(body io.Reader, dir string, buf []byte) (int64, error) {
if err := os.MkdirAll(dir, 0755); err != nil { // nolint if err := os.MkdirAll(dir, 0755); err != nil { // nolint
return 0, xerrors.Errorf("mkdir: %w", err) return 0, xerrors.Errorf("mkdir: %w", err)
@ -37,9 +94,14 @@ func ExtractTar(body io.Reader, dir string, buf []byte) (int64, error) {
return read, xerrors.Errorf("creating file %s: %w", filepath.Join(dir, header.Name), err) return read, xerrors.Errorf("creating file %s: %w", filepath.Join(dir, header.Name), err)
} }
// This data is coming from a trusted source, no need to check the size. sz, found := CacheFileConstraints[header.Name]
// TODO: now it's actually not coming from a trusted source, check size / paths if !found {
//nolint:gosec return read, xerrors.Errorf("tar file %#v isn't expected")
}
if header.Size > sz {
return read, xerrors.Errorf("tar file %#v is bigger than expected: %d > %d", header.Name, header.Size, sz)
}
r, err := io.CopyBuffer(f, tr, buf) r, err := io.CopyBuffer(f, tr, buf)
read += r read += r
if err != nil { if err != nil {