cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

flexible bundle loading

Browse Source
This commit is contained in:
vyzo 2022-05-17 20:38:32 +03:00
parent e9ece8eb3f
commit daf452a180
5 changed files with 133 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
build/README-bundle.md
View File

@ -6,7 +6,7 @@ The bundles are specified in build/bundles.toml using the following syntax:
```toml
[[bundles]]
version = X # actors version
release = tag # release gag
release = tag # release tag
```
This will add a bundle for version `X`, using the github release `tag`
@ -30,6 +30,18 @@ path = /path/to/builtin-actors.car
development = true
```
## Additional Options for Bundles
- You can also specify a URL, together with a sha256 checksum to avoid downloading from
github.
- You can also specify an EnvVar, to provide the path dynamically at runtime.
The precedence for bundle fetching/loading is as folllows:
- Check the EnvVar; use the bundle specified by it.
- Check the Path; use the bundle specified by it.
- Check the URL; use the bundle specified by it, and verify the checksum which must be present.
- Otherwise, use the release tag and download from github.
## Local Storage
Bundles downloaded from github will be stored in

8
build/bundle.go
View File

@ -20,8 +20,14 @@ type Bundle struct {
Version actors.Version
// Release is the release id
Release string
// Path is the (optional) bundle path; uses the appropriate release bundle if unset
// EnvVar is the (optional) env var specifying the bundle path; takes precdence over path
EnvVar string
// Path is the (optional) bundle path; takes precedence over url
Path string
// URL is the (optional) bundle URL; takes precdence over github release
URL string
// CHecksum is the bundle sha256 checksume in hex, when specifying a URL.
Checksum string
// Devlopment indicates whether this is a development version; when set, in conjunction with path,
// it will always load the bundle to the blockstore, without recording the manifest CID in the
// datastore.

62
node/bundle/bundle.go
View File

@ -33,7 +33,7 @@ func NewBundleFetcher(basepath string) (*BundleFetcher, error) {
return &BundleFetcher{path: path}, nil
}
func (b *BundleFetcher) Fetch(version int, release, netw string) (path string, err error) {
func (b *BundleFetcher) FetchFromRelease(version int, release, netw string) (path string, err error) {
bundleName := fmt.Sprintf("builtin-actors-%s", netw)
bundleFile := fmt.Sprintf("%s.car", bundleName)
bundleHash := fmt.Sprintf("%s.sha256", bundleName)
@ -46,7 +46,7 @@ func (b *BundleFetcher) Fetch(version int, release, netw string) (path string, e
// check if it exists; if it does, check the hash
bundleFilePath := filepath.Join(bundleBasePath, bundleFile)
if _, err := os.Stat(bundleFilePath); err == nil {
err := b.check(bundleBasePath, bundleFile, bundleHash)
err := b.checkRelease(bundleBasePath, bundleFile, bundleHash)
if err == nil {
return bundleFilePath, nil
}
@ -55,12 +55,46 @@ func (b *BundleFetcher) Fetch(version int, release, netw string) (path string, e
}
log.Infof("fetching bundle %s", bundleFile)
if err := b.fetch(release, bundleBasePath, bundleFile, bundleHash); err != nil {
if err := b.fetchFromRelease(release, bundleBasePath, bundleFile, bundleHash); err != nil {
log.Errorf("error fetching bundle %s: %s", bundleName, err)
return "", xerrors.Errorf("error fetching bundle: %w", err)
}
if err := b.check(bundleBasePath, bundleFile, bundleHash); err != nil {
if err := b.checkRelease(bundleBasePath, bundleFile, bundleHash); err != nil {
log.Errorf("error checking bundle %s: %s", bundleName, err)
return "", xerrors.Errorf("error checking bundle: %s", err)
}
return bundleFilePath, nil
}
func (b *BundleFetcher) FetchFromURL(version int, release, netw, url, cksum string) (path string, err error) {
bundleName := fmt.Sprintf("builtin-actors-%s", netw)
bundleFile := fmt.Sprintf("%s.car", bundleName)
bundleBasePath := filepath.Join(b.path, fmt.Sprintf("v%d", version), release)
if err := os.MkdirAll(bundleBasePath, 0755); err != nil {
return "", xerrors.Errorf("error making bundle directory %s: %w", bundleBasePath, err)
}
// check if it exists; if it does, check the hash
bundleFilePath := filepath.Join(bundleBasePath, bundleFile)
if _, err := os.Stat(bundleFilePath); err == nil {
err := b.checkHash(bundleBasePath, bundleFile, cksum)
if err == nil {
return bundleFilePath, nil
}
log.Warnf("invalid bundle %s: %s; refetching", bundleName, err)
}
log.Infof("fetching bundle %s", bundleFile)
if err := b.fetchFromURL(bundleBasePath, bundleFile, url); err != nil {
log.Errorf("error fetching bundle %s: %s", bundleName, err)
return "", xerrors.Errorf("error fetching bundle: %w", err)
}
if err := b.checkHash(bundleBasePath, bundleFile, cksum); err != nil {
log.Errorf("error checking bundle %s: %s", bundleName, err)
return "", xerrors.Errorf("error checking bundle: %s", err)
}
@ -105,7 +139,7 @@ func (b *BundleFetcher) fetchURL(url, path string) error {
return xerrors.Errorf("all attempts to fetch %s failed", url)
}
func (b *BundleFetcher) fetch(release, bundleBasePath, bundleFile, bundleHash string) error {
func (b *BundleFetcher) fetchFromRelease(release, bundleBasePath, bundleFile, bundleHash string) error {
bundleHashUrl := fmt.Sprintf("https://github.com/filecoin-project/builtin-actors/releases/download/%s/%s",
release, bundleHash)
bundleHashPath := filepath.Join(bundleBasePath, bundleHash)
@ -123,7 +157,12 @@ func (b *BundleFetcher) fetch(release, bundleBasePath, bundleFile, bundleHash st
return nil
}
func (b *BundleFetcher) check(bundleBasePath, bundleFile, bundleHash string) error {
func (b *BundleFetcher) fetchFromURL(bundleBasePath, bundleFile, url string) error {
bundleFilePath := filepath.Join(bundleBasePath, bundleFile)
return b.fetchURL(url, bundleFilePath)
}
func (b *BundleFetcher) checkRelease(bundleBasePath, bundleFile, bundleHash string) error {
bundleHashPath := filepath.Join(bundleBasePath, bundleHash)
f, err := os.Open(bundleHashPath)
if err != nil {
@ -138,13 +177,18 @@ func (b *BundleFetcher) check(bundleBasePath, bundleFile, bundleHash string) err
parts := strings.Split(string(bs), " ")
hashHex := parts[0]
expectedDigest, err := hex.DecodeString(hashHex)
return b.checkHash(bundleBasePath, bundleFile, hashHex)
}
func (b *BundleFetcher) checkHash(bundleBasePath, bundleFile, cksum string) error {
expectedDigest, err := hex.DecodeString(cksum)
if err != nil {
return xerrors.Errorf("error decoding digest from %s: %w", bundleHashPath, err)
return xerrors.Errorf("error decoding digest from %s: %w", cksum, err)
}
bundleFilePath := filepath.Join(bundleBasePath, bundleFile)
f, err = os.Open(bundleFilePath)
f, err := os.Open(bundleFilePath)
if err != nil {
return xerrors.Errorf("error opening %s: %w", bundleFilePath, err)
}

42
node/bundle/manifest.go
View File

@ -17,13 +17,27 @@ import (
"github.com/mitchellh/go-homedir"
)
func FetchAndLoadBundle(ctx context.Context, basePath string, bs blockstore.Blockstore, av actors.Version, rel, netw string) (cid.Cid, error) {
func FetchAndLoadBundleFromRelease(ctx context.Context, basePath string, bs blockstore.Blockstore, av actors.Version, rel, netw string) (cid.Cid, error) {
fetcher, err := NewBundleFetcher(basePath)
if err != nil {
return cid.Undef, xerrors.Errorf("error creating fetcher for builtin-actors version %d: %w", av, err)
}
path, err := fetcher.Fetch(int(av), rel, netw)
path, err := fetcher.FetchFromRelease(int(av), rel, netw)
if err != nil {
return cid.Undef, xerrors.Errorf("error fetching bundle for builtin-actors version %d: %w", av, err)
}
return LoadBundle(ctx, bs, path, av)
}
func FetchAndLoadBundleFromURL(ctx context.Context, basePath string, bs blockstore.Blockstore, av actors.Version, rel, netw, url, cksum string) (cid.Cid, error) {
fetcher, err := NewBundleFetcher(basePath)
if err != nil {
return cid.Undef, xerrors.Errorf("error creating fetcher for builtin-actors version %d: %w", av, err)
}
path, err := fetcher.FetchFromURL(int(av), rel, netw, url, cksum)
if err != nil {
return cid.Undef, xerrors.Errorf("error fetching bundle for builtin-actors version %d: %w", av, err)
}
@ -72,12 +86,30 @@ func FetchAndLoadBundles(ctx context.Context, bs blockstore.Blockstore, bar map[
}
for av, bd := range bar {
if bd.Path != "" {
switch {
case bd.EnvVar != "":
// this is a local bundle, specified by an env var to load from the filesystem
path := os.Getenv(bd.EnvVar)
if path == "" {
return xerrors.Errorf("bundle envvar is empty: %s", bd.EnvVar)
}
if _, err := LoadBundle(ctx, bs, path, av); err != nil {
return err
}
case bd.Path != "":
if _, err := LoadBundle(ctx, bs, bd.Path, av); err != nil {
return err
}
} else {
if _, err := FetchAndLoadBundle(ctx, path, bs, av, bd.Release, netw); err != nil {
case bd.URL != "":
if _, err := FetchAndLoadBundleFromURL(ctx, path, bs, av, bd.Release, netw, bd.URL, bd.Checksum); err != nil {
return err
}
case bd.Release != "":
if _, err := FetchAndLoadBundleFromRelease(ctx, path, bs, av, bd.Release, netw); err != nil {
return err
}
}

26
node/modules/builtin_actors.go
View File

@ -2,6 +2,7 @@ package modules
import (
"fmt"
"os"
"path/filepath"
"sync"
@ -69,6 +70,18 @@ func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRe
// we haven't recorded it in the datastore, so we need to load it
var mfCid cid.Cid
switch {
case bd.EnvVar != "":
// this is a local bundle, specified by an env var to load from the filesystem
path := os.Getenv(bd.EnvVar)
if path == "" {
return result, xerrors.Errorf("bundle envvar is empty: %s", bd.EnvVar)
}
mfCid, err = bundle.LoadBundle(ctx, bs, path, av)
if err != nil {
return result, err
}
case bd.Path != "":
// this is a local bundle, load it directly from the filessystem
mfCid, err = bundle.LoadBundle(ctx, bs, bd.Path, av)
@ -76,9 +89,16 @@ func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRe
return result, err
}
case bd.URL != "":
// fetch it from the specified URL
mfCid, err = bundle.FetchAndLoadBundleFromURL(ctx, r.Path(), bs, av, bd.Release, netw, bd.URL, bd.Checksum)
if err != nil {
return result, err
}
case bd.Release != "":
// fetch it and add it to the blockstore
mfCid, err = bundle.FetchAndLoadBundle(ctx, r.Path(), bs, av, bd.Release, netw)
mfCid, err = bundle.FetchAndLoadBundleFromRelease(ctx, r.Path(), bs, av, bd.Release, netw)
if err != nil {
return result, err
}
@ -135,12 +155,12 @@ func LoadBuiltinActorsTesting(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtype
case bd.Release != "":
const basePath = "/tmp/lotus-testing"
if _, err := bundle.FetchAndLoadBundle(ctx, basePath, bs, av, bd.Release, netw); err != nil {
if _, err := bundle.FetchAndLoadBundleFromRelease(ctx, basePath, bs, av, bd.Release, netw); err != nil {
return result, xerrors.Errorf("error loading testing bundle for builtin-actors version %d/%s: %w", av, netw, err)
}
default:
return result, xerrors.Errorf("no path or release specified for version %d bundle", av)
return result, xerrors.Errorf("no path or release specified for version %d testing bundle", av)
}
}