Merge pull request #8658 from filecoin-project/feat/nv16-dev-bundle

[nv16] niceties for development bundle loading

Makefile

@@ -75,7 +75,7 @@ debug: build-devnets
 2k: build-devnets
 
 calibnet: GOFLAGS+=-tags=calibnet
-calibnet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=calibrationnet
+calibnet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=calibnet
 calibnet: build-devnets
 
 butterflynet: GOFLAGS+=-tags=butterflynet
@@ -83,7 +83,7 @@ butterflynet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.Netwo
 butterflynet: build-devnets
 
 interopnet: GOFLAGS+=-tags=interopnet
-interopnet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=caterpillarnet
+interopnet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=interopnet
 interopnet: build-devnets
 
 lotus: $(BUILD_DEPS)

build/README-bundle.md

@@ -0,0 +1,59 @@
+# Builtin Actor Bundles
+
+With NV16, builtin actor bundles must be loaded into lotus for the FVM to operate.
+
+The bundles are specified in build/bundles.toml using the following syntax:
+```toml
+[[bundles]]
+version = X   # actors version
+release = tag # release tag
+```
+
+This will add a bundle for version `X`, using the github release `tag`
+to fetch the bundles at first startup.
+
+If you don't want to fetch the bundle from github, you can specify an explicit path to the bundle (which must be appropriate for your network, typically mainnet):
+```toml
+[[bundles]]
+version = X   # actors version
+release = tag # release tag
+path = /path/to/builtin-actors.car
+```
+
+For development bundles, you can also specify `development = true` so that the bundle is not
+recorded in the datastore and reloaded every time the daemon starts up:
+```toml
+[[bundles]]
+version = X   # actors version
+release = tag # release gag
+path = /path/to/builtin-actors.car
+development = true
+```
+
+## Additional Options for Bundles
+
+- You can also specify a URL, together with a sha256 checksum to avoid downloading from
+  github.
+- You can also specify an environment variable (`LOTUS_BUILTIN_ACTORS_VX_BUNDLE`), to provide the path dynamically at runtime.
+
+The precedence for bundle fetching/loading is as folllows:
+- Check the environment variable `LOTUS_BUILTIN_ACTORS_VX_BUNDLE` for version X bundle; use it if set.
+- Check the Path; use the bundle specified by it.
+- Check the URL; use the bundle specified by it, and verify the checksum which must be present.
+- Otherwise, use the release tag and download from github.
+
+## Local Storage
+
+Bundles downloaded from github will be stored in
+`$LOTUS_PATH/builtin-actors/vXXX/YYY/builtin-actors-ZZZ.car``, where
+`XXX` is the actors version, `YYY` is the release tag, and `ZZZ` is
+the network bundle name.
+
+The sha256 sum of the bundle will be stored next to it, in
+`$LOTUS_PATH/builtin-actors/vXXX/YYY/builtin-actors-ZZZ.sha256`
+
+On startup, if a bundle is recorded as loaded the manifest CID will be
+checked for presence in the blockstore.  If the manifest is missing,
+then the bundle will be reloaded from the local file (if it exists) or
+refetched from github.  The sha256 sum is always checked before
+loading the bundle.

build/builtin_actors.go

@@ -8,10 +8,10 @@ import (
 	"github.com/BurntSushi/toml"
 )
 
-var BuiltinActorReleases map[actors.Version]string
+var BuiltinActorReleases map[actors.Version]Bundle
 
 func init() {
-	BuiltinActorReleases = make(map[actors.Version]string)
+	BuiltinActorReleases = make(map[actors.Version]Bundle)
 
 	spec := BundleSpec{}
 
@@ -22,6 +22,6 @@ func init() {
 	}
 
 	for _, b := range spec.Bundles {
-		BuiltinActorReleases[b.Version] = b.Release
+		BuiltinActorReleases[b.Version] = b
 	}
 }

build/bundle.go

@@ -8,6 +8,21 @@ import (
 
 var NetworkBundle string
 
+func GetNetworkBundle() string {
+	switch NetworkBundle {
+	case "devnet":
+		return "devnet"
+	case "calibnet", "calibrationnet":
+		return "calibrationnet"
+	case "butterflynet":
+		return "butterflynet"
+	case "interopnet", "caterpillarnet":
+		return "caterpillarnet"
+	default:
+		return "mainnet"
+	}
+}
+
 //go:embed bundles.toml
 var BuiltinActorBundles []byte
 
@@ -16,6 +31,23 @@ type BundleSpec struct {
 }
 
 type Bundle struct {
+	// Version is the actors version in this bundle
 	Version actors.Version
+	// Release is the release id
 	Release string
+	// Path is the (optional) bundle path; takes precedence over url
+	Path map[string]string
+	// URL is the (optional) bundle URL; takes precdence over github release
+	URL map[string]BundleURL
+	// Devlopment indicates whether this is a development version; when set, in conjunction with path,
+	// it will always load the bundle to the blockstore, without recording the manifest CID in the
+	// datastore.
+	Development bool
+}
+
+type BundleURL struct {
+	// URL is the url of the bundle
+	URL string
+	// Checksum is the sha256 checksum of the bundle
+	Checksum string
 }

build/bundles.toml

@@ -1,3 +1,3 @@
 [[bundles]]
 version = 8
-release = "b71c2ec785aec23d"
+release = "dev/20220517"

node/bundle/bundle.go

@@ -33,7 +33,7 @@ func NewBundleFetcher(basepath string) (*BundleFetcher, error) {
 	return &BundleFetcher{path: path}, nil
 }
 
-func (b *BundleFetcher) Fetch(version int, release, netw string) (path string, err error) {
+func (b *BundleFetcher) FetchFromRelease(version int, release, netw string) (path string, err error) {
 	bundleName := fmt.Sprintf("builtin-actors-%s", netw)
 	bundleFile := fmt.Sprintf("%s.car", bundleName)
 	bundleHash := fmt.Sprintf("%s.sha256", bundleName)
@@ -46,7 +46,7 @@ func (b *BundleFetcher) Fetch(version int, release, netw string) (path string, e
 	// check if it exists; if it does, check the hash
 	bundleFilePath := filepath.Join(bundleBasePath, bundleFile)
 	if _, err := os.Stat(bundleFilePath); err == nil {
-		err := b.check(bundleBasePath, bundleFile, bundleHash)
+		err := b.checkRelease(bundleBasePath, bundleFile, bundleHash)
 		if err == nil {
 			return bundleFilePath, nil
 		}
@@ -55,12 +55,46 @@ func (b *BundleFetcher) Fetch(version int, release, netw string) (path string, e
 	}
 
 	log.Infof("fetching bundle %s", bundleFile)
-	if err := b.fetch(release, bundleBasePath, bundleFile, bundleHash); err != nil {
+	if err := b.fetchFromRelease(release, bundleBasePath, bundleFile, bundleHash); err != nil {
 		log.Errorf("error fetching bundle %s: %s", bundleName, err)
 		return "", xerrors.Errorf("error fetching bundle: %w", err)
 	}
 
-	if err := b.check(bundleBasePath, bundleFile, bundleHash); err != nil {
+	if err := b.checkRelease(bundleBasePath, bundleFile, bundleHash); err != nil {
+		log.Errorf("error checking bundle %s: %s", bundleName, err)
+		return "", xerrors.Errorf("error checking bundle: %s", err)
+	}
+
+	return bundleFilePath, nil
+}
+
+func (b *BundleFetcher) FetchFromURL(version int, release, netw, url, cksum string) (path string, err error) {
+	bundleName := fmt.Sprintf("builtin-actors-%s", netw)
+	bundleFile := fmt.Sprintf("%s.car", bundleName)
+	bundleBasePath := filepath.Join(b.path, fmt.Sprintf("v%d", version), release)
+
+	if err := os.MkdirAll(bundleBasePath, 0755); err != nil {
+		return "", xerrors.Errorf("error making bundle directory %s: %w", bundleBasePath, err)
+	}
+
+	// check if it exists; if it does, check the hash
+	bundleFilePath := filepath.Join(bundleBasePath, bundleFile)
+	if _, err := os.Stat(bundleFilePath); err == nil {
+		err := b.checkHash(bundleBasePath, bundleFile, cksum)
+		if err == nil {
+			return bundleFilePath, nil
+		}
+
+		log.Warnf("invalid bundle %s: %s; refetching", bundleName, err)
+	}
+
+	log.Infof("fetching bundle %s", bundleFile)
+	if err := b.fetchFromURL(bundleBasePath, bundleFile, url); err != nil {
+		log.Errorf("error fetching bundle %s: %s", bundleName, err)
+		return "", xerrors.Errorf("error fetching bundle: %w", err)
+	}
+
+	if err := b.checkHash(bundleBasePath, bundleFile, cksum); err != nil {
 		log.Errorf("error checking bundle %s: %s", bundleName, err)
 		return "", xerrors.Errorf("error checking bundle: %s", err)
 	}
@@ -105,7 +139,7 @@ func (b *BundleFetcher) fetchURL(url, path string) error {
 	return xerrors.Errorf("all attempts to fetch %s failed", url)
 }
 
-func (b *BundleFetcher) fetch(release, bundleBasePath, bundleFile, bundleHash string) error {
+func (b *BundleFetcher) fetchFromRelease(release, bundleBasePath, bundleFile, bundleHash string) error {
 	bundleHashUrl := fmt.Sprintf("https://github.com/filecoin-project/builtin-actors/releases/download/%s/%s",
 		release, bundleHash)
 	bundleHashPath := filepath.Join(bundleBasePath, bundleHash)
@@ -123,7 +157,12 @@ func (b *BundleFetcher) fetch(release, bundleBasePath, bundleFile, bundleHash st
 	return nil
 }
 
-func (b *BundleFetcher) check(bundleBasePath, bundleFile, bundleHash string) error {
+func (b *BundleFetcher) fetchFromURL(bundleBasePath, bundleFile, url string) error {
+	bundleFilePath := filepath.Join(bundleBasePath, bundleFile)
+	return b.fetchURL(url, bundleFilePath)
+}
+
+func (b *BundleFetcher) checkRelease(bundleBasePath, bundleFile, bundleHash string) error {
 	bundleHashPath := filepath.Join(bundleBasePath, bundleHash)
 	f, err := os.Open(bundleHashPath)
 	if err != nil {
@@ -138,13 +177,18 @@ func (b *BundleFetcher) check(bundleBasePath, bundleFile, bundleHash string) err
 
 	parts := strings.Split(string(bs), " ")
 	hashHex := parts[0]
-	expectedDigest, err := hex.DecodeString(hashHex)
+
+	return b.checkHash(bundleBasePath, bundleFile, hashHex)
+}
+
+func (b *BundleFetcher) checkHash(bundleBasePath, bundleFile, cksum string) error {
+	expectedDigest, err := hex.DecodeString(cksum)
 	if err != nil {
-		return xerrors.Errorf("error decoding digest from %s: %w", bundleHashPath, err)
+		return xerrors.Errorf("error decoding digest from %s: %w", cksum, err)
 	}
 
 	bundleFilePath := filepath.Join(bundleBasePath, bundleFile)
-	f, err = os.Open(bundleFilePath)
+	f, err := os.Open(bundleFilePath)
 	if err != nil {
 		return xerrors.Errorf("error opening %s: %w", bundleFilePath, err)
 	}

node/bundle/manifest.go

@@ -2,6 +2,7 @@ package bundle
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"os"
 
@@ -17,17 +18,35 @@ import (
 	"github.com/mitchellh/go-homedir"
 )
 
-func FetchAndLoadBundle(ctx context.Context, basePath string, bs blockstore.Blockstore, av actors.Version, rel, netw string) (cid.Cid, error) {
+func FetchAndLoadBundleFromRelease(ctx context.Context, basePath string, bs blockstore.Blockstore, av actors.Version, rel, netw string) (cid.Cid, error) {
 	fetcher, err := NewBundleFetcher(basePath)
 	if err != nil {
 		return cid.Undef, xerrors.Errorf("error creating fetcher for builtin-actors version %d: %w", av, err)
 	}
 
-	path, err := fetcher.Fetch(int(av), rel, netw)
+	path, err := fetcher.FetchFromRelease(int(av), rel, netw)
 	if err != nil {
 		return cid.Undef, xerrors.Errorf("error fetching bundle for builtin-actors version %d: %w", av, err)
 	}
 
+	return LoadBundle(ctx, bs, path, av)
+}
+
+func FetchAndLoadBundleFromURL(ctx context.Context, basePath string, bs blockstore.Blockstore, av actors.Version, rel, netw, url, cksum string) (cid.Cid, error) {
+	fetcher, err := NewBundleFetcher(basePath)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("error creating fetcher for builtin-actors version %d: %w", av, err)
+	}
+
+	path, err := fetcher.FetchFromURL(int(av), rel, netw, url, cksum)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("error fetching bundle for builtin-actors version %d: %w", av, err)
+	}
+
+	return LoadBundle(ctx, bs, path, av)
+}
+
+func LoadBundle(ctx context.Context, bs blockstore.Blockstore, path string, av actors.Version) (cid.Cid, error) {
 	f, err := os.Open(path)
 	if err != nil {
 		return cid.Undef, xerrors.Errorf("error opening bundle for builtin-actors vresion %d: %w", av, err)
@@ -52,11 +71,8 @@ func FetchAndLoadBundle(ctx context.Context, basePath string, bs blockstore.Bloc
 }
 
 // utility for blanket loading outside DI
-func FetchAndLoadBundles(ctx context.Context, bs blockstore.Blockstore, bar map[actors.Version]string) error {
-	netw := build.NetworkBundle
-	if netw == "" {
-		netw = "mainnet"
-	}
+func FetchAndLoadBundles(ctx context.Context, bs blockstore.Blockstore, bar map[actors.Version]build.Bundle) error {
+	netw := build.GetNetworkBundle()
 
 	path := os.Getenv("LOTUS_PATH")
 	if path == "" {
@@ -67,10 +83,32 @@ func FetchAndLoadBundles(ctx context.Context, bs blockstore.Blockstore, bar map[
 		}
 	}
 
-	for av, rel := range bar {
-		if _, err := FetchAndLoadBundle(ctx, path, bs, av, rel, netw); err != nil {
+	for av, bd := range bar {
+		envvar := fmt.Sprintf("LOTUS_BUILTIN_ACTORS_V%d_BUNDLE", av)
+		switch {
+		case os.Getenv(envvar) != "":
+			// this is a local bundle, specified by an env var to load from the filesystem
+			path := os.Getenv(envvar)
+
+			if _, err := LoadBundle(ctx, bs, path, av); err != nil {
 				return err
 			}
+
+		case bd.Path[netw] != "":
+			if _, err := LoadBundle(ctx, bs, bd.Path[netw], av); err != nil {
+				return err
+			}
+
+		case bd.URL[netw].URL != "":
+			if _, err := FetchAndLoadBundleFromURL(ctx, path, bs, av, bd.Release, netw, bd.URL[netw].URL, bd.URL[netw].Checksum); err != nil {
+				return err
+			}
+
+		case bd.Release != "":
+			if _, err := FetchAndLoadBundleFromRelease(ctx, path, bs, av, bd.Release, netw); err != nil {
+				return err
+			}
+		}
 	}
 
 	cborStore := cbor.NewCborStore(bs)

node/modules/builtin_actors.go

@@ -2,6 +2,7 @@ package modules
 
 import (
 	"fmt"
+	"os"
 	"sync"
 
 	"go.uber.org/fx"
@@ -24,14 +25,11 @@ func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRe
 
 	// We can't put it as a dep in inputs causes a stack overflow in DI from circular dependency
 	// So we pass it through ldflags instead
-	netw := build.NetworkBundle
-	if netw == "" {
-		netw = "mainnet"
-	}
+	netw := build.GetNetworkBundle()
 
-	for av, rel := range build.BuiltinActorReleases {
+	for av, bd := range build.BuiltinActorReleases {
 		// first check to see if we know this release
-		key := dstore.NewKey(fmt.Sprintf("/builtin-actors/v%d/%s", av, rel))
+		key := dstore.NewKey(fmt.Sprintf("/builtin-actors/v%d/%s", av, bd.Release))
 
 		data, err := ds.Get(ctx, key)
 		switch err {
@@ -65,12 +63,48 @@ func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRe
 			return result, xerrors.Errorf("error loading %s from datastore: %w", key, err)
 		}
 
-		// ok, we don't have it -- fetch it and add it to the blockstore
-		mfCid, err := bundle.FetchAndLoadBundle(ctx, r.Path(), bs, av, rel, netw)
+		// we haven't recorded it in the datastore, so we need to load it
+		envvar := fmt.Sprintf("LOTUS_BUILTIN_ACTORS_V%d_BUNDLE", av)
+		var mfCid cid.Cid
+		switch {
+		case os.Getenv(envvar) != "":
+			path := os.Getenv(envvar)
+
+			mfCid, err = bundle.LoadBundle(ctx, bs, path, av)
 			if err != nil {
 				return result, err
 			}
 
+		case bd.Path[netw] != "":
+			// this is a local bundle, load it directly from the filessystem
+			mfCid, err = bundle.LoadBundle(ctx, bs, bd.Path[netw], av)
+			if err != nil {
+				return result, err
+			}
+
+		case bd.URL[netw].URL != "":
+			// fetch it from the specified URL
+			mfCid, err = bundle.FetchAndLoadBundleFromURL(ctx, r.Path(), bs, av, bd.Release, netw, bd.URL[netw].URL, bd.URL[netw].Checksum)
+			if err != nil {
+				return result, err
+			}
+
+		case bd.Release != "":
+			// fetch it and add it to the blockstore
+			mfCid, err = bundle.FetchAndLoadBundleFromRelease(ctx, r.Path(), bs, av, bd.Release, netw)
+			if err != nil {
+				return result, err
+			}
+
+		default:
+			return result, xerrors.Errorf("no release or path specified for version %d bundle", av)
+		}
+
+		if bd.Development || bd.Release == "" {
+			// don't store the release key so that we always load development bundles
+			continue
+		}
+
 		// add the release key with the manifest to avoid reloading it in next restart.
 		if err := ds.Put(ctx, key, mfCid.Bytes()); err != nil {
 			return result, xerrors.Errorf("error storing manifest CID for builtin-actors vrsion %d to the datastore: %w", av, err)
@@ -102,11 +136,27 @@ func LoadBuiltinActorsTesting(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtype
 	testingBundleMx.Lock()
 	defer testingBundleMx.Unlock()
 
-	for av, rel := range build.BuiltinActorReleases {
 	const basePath = "/tmp/lotus-testing"
+	for av, bd := range build.BuiltinActorReleases {
+		switch {
+		case bd.Path[netw] != "":
+			if _, err := bundle.LoadBundle(ctx, bs, bd.Path[netw], av); err != nil {
+				return result, xerrors.Errorf("error loading testing bundle for builtin-actors version %d/%s: %w", av, netw, err)
+			}
 
-		if _, err := bundle.FetchAndLoadBundle(ctx, basePath, bs, av, rel, netw); err != nil {
-			return result, xerrors.Errorf("error loading bundle for builtin-actors vresion %d: %w", av, err)
+		case bd.URL[netw].URL != "":
+			// fetch it from the specified URL
+			if _, err := bundle.FetchAndLoadBundleFromURL(ctx, basePath, bs, av, bd.Release, netw, bd.URL[netw].URL, bd.URL[netw].Checksum); err != nil {
+				return result, err
+			}
+
+		case bd.Release != "":
+			if _, err := bundle.FetchAndLoadBundleFromRelease(ctx, basePath, bs, av, bd.Release, netw); err != nil {
+				return result, xerrors.Errorf("error loading testing bundle for builtin-actors version %d/%s: %w", av, netw, err)
+			}
+
+		default:
+			return result, xerrors.Errorf("no path or release specified for version %d testing bundle", av)
 		}
 	}