cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #8666 from filecoin-project/feat/nv16-flexibundle

Browse Source 
[nv16] flexible bundle loading
This commit is contained in:
vyzo 2022-05-17 22:12:48 +03:00 committed by GitHub
commit 86e1144b65
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 146 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
build/README-bundle.md
View File

@ -6,7 +6,7 @@ The bundles are specified in build/bundles.toml using the following syntax:
```toml
[[bundles]]
version = X # actors version
release = tag # release gag
release = tag # release tag
```
This will add a bundle for version `X`, using the github release `tag`
@ -30,6 +30,18 @@ path = /path/to/builtin-actors.car
development = true
```
## Additional Options for Bundles
- You can also specify a URL, together with a sha256 checksum to avoid downloading from
github.
- You can also specify an environment variable (`LOTUS_BUILTIN_ACTORS_VX_BUNDLE`), to provide the path dynamically at runtime.
The precedence for bundle fetching/loading is as folllows:
- Check the environment variable `LOTUS_BUILTIN_ACTORS_VX_BUNDLE` for version X bundle; use it if set.
- Check the Path; use the bundle specified by it.
- Check the URL; use the bundle specified by it, and verify the checksum which must be present.
- Otherwise, use the release tag and download from github.
## Local Storage
Bundles downloaded from github will be stored in

13
build/bundle.go
View File

@ -20,10 +20,19 @@ type Bundle struct {
Version actors.Version
// Release is the release id
Release string
// Path is the (optional) bundle path; uses the appropriate release bundle if unset
Path string
// Path is the (optional) bundle path; takes precedence over url
Path map[string]string
// URL is the (optional) bundle URL; takes precdence over github release
URL map[string]BundleURL
// Devlopment indicates whether this is a development version; when set, in conjunction with path,
// it will always load the bundle to the blockstore, without recording the manifest CID in the
// datastore.
Development bool
}
type BundleURL struct {
// URL is the url of the bundle
URL string
// Checksum is the sha256 checksum of the bundle
Checksum string
}

2
build/bundles.toml
View File

@ -1,3 +1,3 @@
[[bundles]]
version = 8
release = "b71c2ec785aec23d"
release = "dev/20220517"

62
node/bundle/bundle.go
View File

@ -33,7 +33,7 @@ func NewBundleFetcher(basepath string) (*BundleFetcher, error) {
return &BundleFetcher{path: path}, nil
}
func (b *BundleFetcher) Fetch(version int, release, netw string) (path string, err error) {
func (b *BundleFetcher) FetchFromRelease(version int, release, netw string) (path string, err error) {
bundleName := fmt.Sprintf("builtin-actors-%s", netw)
bundleFile := fmt.Sprintf("%s.car", bundleName)
bundleHash := fmt.Sprintf("%s.sha256", bundleName)
@ -46,7 +46,7 @@ func (b *BundleFetcher) Fetch(version int, release, netw string) (path string, e
// check if it exists; if it does, check the hash
bundleFilePath := filepath.Join(bundleBasePath, bundleFile)
if _, err := os.Stat(bundleFilePath); err == nil {
err := b.check(bundleBasePath, bundleFile, bundleHash)
err := b.checkRelease(bundleBasePath, bundleFile, bundleHash)
if err == nil {
return bundleFilePath, nil
}
@ -55,12 +55,46 @@ func (b *BundleFetcher) Fetch(version int, release, netw string) (path string, e
}
log.Infof("fetching bundle %s", bundleFile)
if err := b.fetch(release, bundleBasePath, bundleFile, bundleHash); err != nil {
if err := b.fetchFromRelease(release, bundleBasePath, bundleFile, bundleHash); err != nil {
log.Errorf("error fetching bundle %s: %s", bundleName, err)
return "", xerrors.Errorf("error fetching bundle: %w", err)
}
if err := b.check(bundleBasePath, bundleFile, bundleHash); err != nil {
if err := b.checkRelease(bundleBasePath, bundleFile, bundleHash); err != nil {
log.Errorf("error checking bundle %s: %s", bundleName, err)
return "", xerrors.Errorf("error checking bundle: %s", err)
}
return bundleFilePath, nil
}
func (b *BundleFetcher) FetchFromURL(version int, release, netw, url, cksum string) (path string, err error) {
bundleName := fmt.Sprintf("builtin-actors-%s", netw)
bundleFile := fmt.Sprintf("%s.car", bundleName)
bundleBasePath := filepath.Join(b.path, fmt.Sprintf("v%d", version), release)
if err := os.MkdirAll(bundleBasePath, 0755); err != nil {
return "", xerrors.Errorf("error making bundle directory %s: %w", bundleBasePath, err)
}
// check if it exists; if it does, check the hash
bundleFilePath := filepath.Join(bundleBasePath, bundleFile)
if _, err := os.Stat(bundleFilePath); err == nil {
err := b.checkHash(bundleBasePath, bundleFile, cksum)
if err == nil {
return bundleFilePath, nil
}
log.Warnf("invalid bundle %s: %s; refetching", bundleName, err)
}
log.Infof("fetching bundle %s", bundleFile)
if err := b.fetchFromURL(bundleBasePath, bundleFile, url); err != nil {
log.Errorf("error fetching bundle %s: %s", bundleName, err)
return "", xerrors.Errorf("error fetching bundle: %w", err)
}
if err := b.checkHash(bundleBasePath, bundleFile, cksum); err != nil {
log.Errorf("error checking bundle %s: %s", bundleName, err)
return "", xerrors.Errorf("error checking bundle: %s", err)
}
@ -105,7 +139,7 @@ func (b *BundleFetcher) fetchURL(url, path string) error {
return xerrors.Errorf("all attempts to fetch %s failed", url)
}
func (b *BundleFetcher) fetch(release, bundleBasePath, bundleFile, bundleHash string) error {
func (b *BundleFetcher) fetchFromRelease(release, bundleBasePath, bundleFile, bundleHash string) error {
bundleHashUrl := fmt.Sprintf("https://github.com/filecoin-project/builtin-actors/releases/download/%s/%s",
release, bundleHash)
bundleHashPath := filepath.Join(bundleBasePath, bundleHash)
@ -123,7 +157,12 @@ func (b *BundleFetcher) fetch(release, bundleBasePath, bundleFile, bundleHash st
return nil
}
func (b *BundleFetcher) check(bundleBasePath, bundleFile, bundleHash string) error {
func (b *BundleFetcher) fetchFromURL(bundleBasePath, bundleFile, url string) error {
bundleFilePath := filepath.Join(bundleBasePath, bundleFile)
return b.fetchURL(url, bundleFilePath)
}
func (b *BundleFetcher) checkRelease(bundleBasePath, bundleFile, bundleHash string) error {
bundleHashPath := filepath.Join(bundleBasePath, bundleHash)
f, err := os.Open(bundleHashPath)
if err != nil {
@ -138,13 +177,18 @@ func (b *BundleFetcher) check(bundleBasePath, bundleFile, bundleHash string) err
parts := strings.Split(string(bs), " ")
hashHex := parts[0]
expectedDigest, err := hex.DecodeString(hashHex)
return b.checkHash(bundleBasePath, bundleFile, hashHex)
}
func (b *BundleFetcher) checkHash(bundleBasePath, bundleFile, cksum string) error {
expectedDigest, err := hex.DecodeString(cksum)
if err != nil {
return xerrors.Errorf("error decoding digest from %s: %w", bundleHashPath, err)
return xerrors.Errorf("error decoding digest from %s: %w", cksum, err)
}
bundleFilePath := filepath.Join(bundleBasePath, bundleFile)
f, err = os.Open(bundleFilePath)
f, err := os.Open(bundleFilePath)
if err != nil {
return xerrors.Errorf("error opening %s: %w", bundleFilePath, err)
}

43
node/bundle/manifest.go
View File

@ -2,6 +2,7 @@ package bundle
import (
"context"
"fmt"
"io"
"os"
@ -17,13 +18,27 @@ import (
"github.com/mitchellh/go-homedir"
)
func FetchAndLoadBundle(ctx context.Context, basePath string, bs blockstore.Blockstore, av actors.Version, rel, netw string) (cid.Cid, error) {
func FetchAndLoadBundleFromRelease(ctx context.Context, basePath string, bs blockstore.Blockstore, av actors.Version, rel, netw string) (cid.Cid, error) {
fetcher, err := NewBundleFetcher(basePath)
if err != nil {
return cid.Undef, xerrors.Errorf("error creating fetcher for builtin-actors version %d: %w", av, err)
}
path, err := fetcher.Fetch(int(av), rel, netw)
path, err := fetcher.FetchFromRelease(int(av), rel, netw)
if err != nil {
return cid.Undef, xerrors.Errorf("error fetching bundle for builtin-actors version %d: %w", av, err)
}
return LoadBundle(ctx, bs, path, av)
}
func FetchAndLoadBundleFromURL(ctx context.Context, basePath string, bs blockstore.Blockstore, av actors.Version, rel, netw, url, cksum string) (cid.Cid, error) {
fetcher, err := NewBundleFetcher(basePath)
if err != nil {
return cid.Undef, xerrors.Errorf("error creating fetcher for builtin-actors version %d: %w", av, err)
}
path, err := fetcher.FetchFromURL(int(av), rel, netw, url, cksum)
if err != nil {
return cid.Undef, xerrors.Errorf("error fetching bundle for builtin-actors version %d: %w", av, err)
}
@ -72,12 +87,28 @@ func FetchAndLoadBundles(ctx context.Context, bs blockstore.Blockstore, bar map[
}
for av, bd := range bar {
if bd.Path != "" {
if _, err := LoadBundle(ctx, bs, bd.Path, av); err != nil {
envvar := fmt.Sprintf("LOTUS_BUILTIN_ACTORS_V%d_BUNDLE", av)
switch {
case os.Getenv(envvar) != "":
// this is a local bundle, specified by an env var to load from the filesystem
path := os.Getenv(envvar)
if _, err := LoadBundle(ctx, bs, path, av); err != nil {
return err
}
} else {
if _, err := FetchAndLoadBundle(ctx, path, bs, av, bd.Release, netw); err != nil {
case bd.Path[netw] != "":
if _, err := LoadBundle(ctx, bs, bd.Path[netw], av); err != nil {
return err
}
case bd.URL[netw].URL != "":
if _, err := FetchAndLoadBundleFromURL(ctx, path, bs, av, bd.Release, netw, bd.URL[netw].URL, bd.URL[netw].Checksum); err != nil {
return err
}
case bd.Release != "":
if _, err := FetchAndLoadBundleFromRelease(ctx, path, bs, av, bd.Release, netw); err != nil {
return err
}
}

43
node/modules/builtin_actors.go
View File

@ -2,7 +2,7 @@ package modules
import (
"fmt"
"path/filepath"
"os"
"sync"
"go.uber.org/fx"
@ -67,18 +67,34 @@ func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRe
}
// we haven't recorded it in the datastore, so we need to load it
envvar := fmt.Sprintf("LOTUS_BUILTIN_ACTORS_V%d_BUNDLE", av)
var mfCid cid.Cid
switch {
case bd.Path != "":
case os.Getenv(envvar) != "":
path := os.Getenv(envvar)
mfCid, err = bundle.LoadBundle(ctx, bs, path, av)
if err != nil {
return result, err
}
case bd.Path[netw] != "":
// this is a local bundle, load it directly from the filessystem
mfCid, err = bundle.LoadBundle(ctx, bs, bd.Path, av)
mfCid, err = bundle.LoadBundle(ctx, bs, bd.Path[netw], av)
if err != nil {
return result, err
}
case bd.URL[netw].URL != "":
// fetch it from the specified URL
mfCid, err = bundle.FetchAndLoadBundleFromURL(ctx, r.Path(), bs, av, bd.Release, netw, bd.URL[netw].URL, bd.URL[netw].Checksum)
if err != nil {
return result, err
}
case bd.Release != "":
// fetch it and add it to the blockstore
mfCid, err = bundle.FetchAndLoadBundle(ctx, r.Path(), bs, av, bd.Release, netw)
mfCid, err = bundle.FetchAndLoadBundleFromRelease(ctx, r.Path(), bs, av, bd.Release, netw)
if err != nil {
return result, err
}
@ -123,24 +139,27 @@ func LoadBuiltinActorsTesting(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtype
testingBundleMx.Lock()
defer testingBundleMx.Unlock()
const basePath = "/tmp/lotus-testing"
for av, bd := range build.BuiltinActorReleases {
switch {
case bd.Path != "":
// we need the appropriate bundle for tests; it should live next to the main bundle, with the
// appropriate network name
path := filepath.Join(filepath.Dir(bd.Path), fmt.Sprintf("builtin-actors-%s.car", netw))
if _, err := bundle.LoadBundle(ctx, bs, path, av); err != nil {
case bd.Path[netw] != "":
if _, err := bundle.LoadBundle(ctx, bs, bd.Path[netw], av); err != nil {
return result, xerrors.Errorf("error loading testing bundle for builtin-actors version %d/%s: %w", av, netw, err)
}
case bd.URL[netw].URL != "":
// fetch it from the specified URL
if _, err := bundle.FetchAndLoadBundleFromURL(ctx, basePath, bs, av, bd.Release, netw, bd.URL[netw].URL, bd.URL[netw].Checksum); err != nil {
return result, err
}
case bd.Release != "":
const basePath = "/tmp/lotus-testing"
if _, err := bundle.FetchAndLoadBundle(ctx, basePath, bs, av, bd.Release, netw); err != nil {
if _, err := bundle.FetchAndLoadBundleFromRelease(ctx, basePath, bs, av, bd.Release, netw); err != nil {
return result, xerrors.Errorf("error loading testing bundle for builtin-actors version %d/%s: %w", av, netw, err)
}
default:
return result, xerrors.Errorf("no path or release specified for version %d bundle", av)
return result, xerrors.Errorf("no path or release specified for version %d testing bundle", av)
}
}