diff --git a/storage/sealer/storiface/storage.go b/storage/sealer/storiface/storage.go index 975011a0b..5246a1a1a 100644 --- a/storage/sealer/storiface/storage.go +++ b/storage/sealer/storiface/storage.go @@ -131,7 +131,7 @@ type SectorData struct { // URL to the sector data // For sealed/unsealed sector, lotus expects octet-stream - // For cache, lotus expects a tar archive with cache files (todo maybe use not-tar; specify what files with what paths must be present) + // For cache, lotus expects a tar archive with cache files // Valid schemas: // - http:// / https:// URL string diff --git a/storage/sealer/tarutil/systar.go b/storage/sealer/tarutil/systar.go index 96dcce875..f7437d1f2 100644 --- a/storage/sealer/tarutil/systar.go +++ b/storage/sealer/tarutil/systar.go @@ -12,6 +12,63 @@ import ( var log = logging.Logger("tarutil") // nolint +var CacheFileConstraints = map[string]int64{ + "p_aux": 64, + "t_aux": 10240, + + "sc-02-data-tree-r-last.dat": 10_000_000, // small sectors + + "sc-02-data-tree-r-last-0.dat": 10_000_000, + "sc-02-data-tree-r-last-1.dat": 10_000_000, + "sc-02-data-tree-r-last-2.dat": 10_000_000, + "sc-02-data-tree-r-last-3.dat": 10_000_000, + "sc-02-data-tree-r-last-4.dat": 10_000_000, + "sc-02-data-tree-r-last-5.dat": 10_000_000, + "sc-02-data-tree-r-last-6.dat": 10_000_000, + "sc-02-data-tree-r-last-7.dat": 10_000_000, + + "sc-02-data-tree-r-last-8.dat": 10_000_000, + "sc-02-data-tree-r-last-9.dat": 10_000_000, + "sc-02-data-tree-r-last-10.dat": 10_000_000, + "sc-02-data-tree-r-last-11.dat": 10_000_000, + "sc-02-data-tree-r-last-12.dat": 10_000_000, + "sc-02-data-tree-r-last-13.dat": 10_000_000, + "sc-02-data-tree-r-last-14.dat": 10_000_000, + "sc-02-data-tree-r-last-15.dat": 10_000_000, + + "sc-02-data-layer-1.dat": 65 << 30, // 1x sector size + small buffer + "sc-02-data-layer-2.dat": 65 << 30, // 1x sector size + small buffer + "sc-02-data-layer-3.dat": 65 << 30, // 1x sector size + small buffer + "sc-02-data-layer-4.dat": 65 << 30, // 1x sector size + small buffer + "sc-02-data-layer-5.dat": 65 << 30, // 1x sector size + small buffer + "sc-02-data-layer-6.dat": 65 << 30, // 1x sector size + small buffer + "sc-02-data-layer-7.dat": 65 << 30, // 1x sector size + small buffer + "sc-02-data-layer-8.dat": 65 << 30, // 1x sector size + small buffer + "sc-02-data-layer-9.dat": 65 << 30, // 1x sector size + small buffer + "sc-02-data-layer-10.dat": 65 << 30, // 1x sector size + small buffer + "sc-02-data-layer-11.dat": 65 << 30, // 1x sector size + small buffer + + "sc-02-data-tree-c-0.dat": 5 << 30, // ~4.6G + "sc-02-data-tree-c-1.dat": 5 << 30, + "sc-02-data-tree-c-2.dat": 5 << 30, + "sc-02-data-tree-c-3.dat": 5 << 30, + "sc-02-data-tree-c-4.dat": 5 << 30, + "sc-02-data-tree-c-5.dat": 5 << 30, + "sc-02-data-tree-c-6.dat": 5 << 30, + "sc-02-data-tree-c-7.dat": 5 << 30, + + "sc-02-data-tree-c-8.dat": 5 << 30, + "sc-02-data-tree-c-9.dat": 5 << 30, + "sc-02-data-tree-c-10.dat": 5 << 30, + "sc-02-data-tree-c-11.dat": 5 << 30, + "sc-02-data-tree-c-12.dat": 5 << 30, + "sc-02-data-tree-c-13.dat": 5 << 30, + "sc-02-data-tree-c-14.dat": 5 << 30, + "sc-02-data-tree-c-15.dat": 5 << 30, + + "sc-02-data-tree-d.dat": 130 << 30, // 2x sector size, ~130G accunting for small buffer on 64G sectors +} + func ExtractTar(body io.Reader, dir string, buf []byte) (int64, error) { if err := os.MkdirAll(dir, 0755); err != nil { // nolint return 0, xerrors.Errorf("mkdir: %w", err) @@ -37,9 +94,14 @@ func ExtractTar(body io.Reader, dir string, buf []byte) (int64, error) { return read, xerrors.Errorf("creating file %s: %w", filepath.Join(dir, header.Name), err) } - // This data is coming from a trusted source, no need to check the size. - // TODO: now it's actually not coming from a trusted source, check size / paths - //nolint:gosec + sz, found := CacheFileConstraints[header.Name] + if !found { + return read, xerrors.Errorf("tar file %#v isn't expected") + } + if header.Size > sz { + return read, xerrors.Errorf("tar file %#v is bigger than expected: %d > %d", header.Name, header.Size, sz) + } + r, err := io.CopyBuffer(f, tr, buf) read += r if err != nil {