diff --git a/Makefile b/Makefile index 2f729447d..141a90dda 100644 --- a/Makefile +++ b/Makefile @@ -75,7 +75,7 @@ debug: build-devnets 2k: build-devnets calibnet: GOFLAGS+=-tags=calibnet -calibnet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=calibrationnet +calibnet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=calibnet calibnet: build-devnets butterflynet: GOFLAGS+=-tags=butterflynet @@ -83,7 +83,7 @@ butterflynet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.Netwo butterflynet: build-devnets interopnet: GOFLAGS+=-tags=interopnet -interopnet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=caterpillarnet +interopnet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=interopnet interopnet: build-devnets lotus: $(BUILD_DEPS) diff --git a/build/README-bundle.md b/build/README-bundle.md new file mode 100644 index 000000000..4dc96c687 --- /dev/null +++ b/build/README-bundle.md @@ -0,0 +1,59 @@ +# Builtin Actor Bundles + +With NV16, builtin actor bundles must be loaded into lotus for the FVM to operate. + +The bundles are specified in build/bundles.toml using the following syntax: +```toml +[[bundles]] +version = X # actors version +release = tag # release tag +``` + +This will add a bundle for version `X`, using the github release `tag` +to fetch the bundles at first startup. + +If you don't want to fetch the bundle from github, you can specify an explicit path to the bundle (which must be appropriate for your network, typically mainnet): +```toml +[[bundles]] +version = X # actors version +release = tag # release tag +path = /path/to/builtin-actors.car +``` + +For development bundles, you can also specify `development = true` so that the bundle is not +recorded in the datastore and reloaded every time the daemon starts up: +```toml +[[bundles]] +version = X # actors version +release = tag # release gag +path = /path/to/builtin-actors.car +development = true +``` + +## Additional Options for Bundles + +- You can also specify a URL, together with a sha256 checksum to avoid downloading from + github. +- You can also specify an environment variable (`LOTUS_BUILTIN_ACTORS_VX_BUNDLE`), to provide the path dynamically at runtime. + +The precedence for bundle fetching/loading is as folllows: +- Check the environment variable `LOTUS_BUILTIN_ACTORS_VX_BUNDLE` for version X bundle; use it if set. +- Check the Path; use the bundle specified by it. +- Check the URL; use the bundle specified by it, and verify the checksum which must be present. +- Otherwise, use the release tag and download from github. + +## Local Storage + +Bundles downloaded from github will be stored in +`$LOTUS_PATH/builtin-actors/vXXX/YYY/builtin-actors-ZZZ.car``, where +`XXX` is the actors version, `YYY` is the release tag, and `ZZZ` is +the network bundle name. + +The sha256 sum of the bundle will be stored next to it, in +`$LOTUS_PATH/builtin-actors/vXXX/YYY/builtin-actors-ZZZ.sha256` + +On startup, if a bundle is recorded as loaded the manifest CID will be +checked for presence in the blockstore. If the manifest is missing, +then the bundle will be reloaded from the local file (if it exists) or +refetched from github. The sha256 sum is always checked before +loading the bundle. diff --git a/build/builtin_actors.go b/build/builtin_actors.go index e65ebabf9..c6ed38c5b 100644 --- a/build/builtin_actors.go +++ b/build/builtin_actors.go @@ -8,10 +8,10 @@ import ( "github.com/BurntSushi/toml" ) -var BuiltinActorReleases map[actors.Version]string +var BuiltinActorReleases map[actors.Version]Bundle func init() { - BuiltinActorReleases = make(map[actors.Version]string) + BuiltinActorReleases = make(map[actors.Version]Bundle) spec := BundleSpec{} @@ -22,6 +22,6 @@ func init() { } for _, b := range spec.Bundles { - BuiltinActorReleases[b.Version] = b.Release + BuiltinActorReleases[b.Version] = b } } diff --git a/build/bundle.go b/build/bundle.go index 1a6fbcd93..dc2faff24 100644 --- a/build/bundle.go +++ b/build/bundle.go @@ -8,6 +8,21 @@ import ( var NetworkBundle string +func GetNetworkBundle() string { + switch NetworkBundle { + case "devnet": + return "devnet" + case "calibnet", "calibrationnet": + return "calibrationnet" + case "butterflynet": + return "butterflynet" + case "interopnet", "caterpillarnet": + return "caterpillarnet" + default: + return "mainnet" + } +} + //go:embed bundles.toml var BuiltinActorBundles []byte @@ -16,6 +31,23 @@ type BundleSpec struct { } type Bundle struct { + // Version is the actors version in this bundle Version actors.Version + // Release is the release id Release string + // Path is the (optional) bundle path; takes precedence over url + Path map[string]string + // URL is the (optional) bundle URL; takes precdence over github release + URL map[string]BundleURL + // Devlopment indicates whether this is a development version; when set, in conjunction with path, + // it will always load the bundle to the blockstore, without recording the manifest CID in the + // datastore. + Development bool +} + +type BundleURL struct { + // URL is the url of the bundle + URL string + // Checksum is the sha256 checksum of the bundle + Checksum string } diff --git a/build/bundles.toml b/build/bundles.toml index a3ef18414..347b81149 100644 --- a/build/bundles.toml +++ b/build/bundles.toml @@ -1,3 +1,3 @@ [[bundles]] version = 8 -release = "b71c2ec785aec23d" +release = "dev/20220517" diff --git a/node/bundle/bundle.go b/node/bundle/bundle.go index 99f214383..a3a14f468 100644 --- a/node/bundle/bundle.go +++ b/node/bundle/bundle.go @@ -33,7 +33,7 @@ func NewBundleFetcher(basepath string) (*BundleFetcher, error) { return &BundleFetcher{path: path}, nil } -func (b *BundleFetcher) Fetch(version int, release, netw string) (path string, err error) { +func (b *BundleFetcher) FetchFromRelease(version int, release, netw string) (path string, err error) { bundleName := fmt.Sprintf("builtin-actors-%s", netw) bundleFile := fmt.Sprintf("%s.car", bundleName) bundleHash := fmt.Sprintf("%s.sha256", bundleName) @@ -46,7 +46,7 @@ func (b *BundleFetcher) Fetch(version int, release, netw string) (path string, e // check if it exists; if it does, check the hash bundleFilePath := filepath.Join(bundleBasePath, bundleFile) if _, err := os.Stat(bundleFilePath); err == nil { - err := b.check(bundleBasePath, bundleFile, bundleHash) + err := b.checkRelease(bundleBasePath, bundleFile, bundleHash) if err == nil { return bundleFilePath, nil } @@ -55,12 +55,46 @@ func (b *BundleFetcher) Fetch(version int, release, netw string) (path string, e } log.Infof("fetching bundle %s", bundleFile) - if err := b.fetch(release, bundleBasePath, bundleFile, bundleHash); err != nil { + if err := b.fetchFromRelease(release, bundleBasePath, bundleFile, bundleHash); err != nil { log.Errorf("error fetching bundle %s: %s", bundleName, err) return "", xerrors.Errorf("error fetching bundle: %w", err) } - if err := b.check(bundleBasePath, bundleFile, bundleHash); err != nil { + if err := b.checkRelease(bundleBasePath, bundleFile, bundleHash); err != nil { + log.Errorf("error checking bundle %s: %s", bundleName, err) + return "", xerrors.Errorf("error checking bundle: %s", err) + } + + return bundleFilePath, nil +} + +func (b *BundleFetcher) FetchFromURL(version int, release, netw, url, cksum string) (path string, err error) { + bundleName := fmt.Sprintf("builtin-actors-%s", netw) + bundleFile := fmt.Sprintf("%s.car", bundleName) + bundleBasePath := filepath.Join(b.path, fmt.Sprintf("v%d", version), release) + + if err := os.MkdirAll(bundleBasePath, 0755); err != nil { + return "", xerrors.Errorf("error making bundle directory %s: %w", bundleBasePath, err) + } + + // check if it exists; if it does, check the hash + bundleFilePath := filepath.Join(bundleBasePath, bundleFile) + if _, err := os.Stat(bundleFilePath); err == nil { + err := b.checkHash(bundleBasePath, bundleFile, cksum) + if err == nil { + return bundleFilePath, nil + } + + log.Warnf("invalid bundle %s: %s; refetching", bundleName, err) + } + + log.Infof("fetching bundle %s", bundleFile) + if err := b.fetchFromURL(bundleBasePath, bundleFile, url); err != nil { + log.Errorf("error fetching bundle %s: %s", bundleName, err) + return "", xerrors.Errorf("error fetching bundle: %w", err) + } + + if err := b.checkHash(bundleBasePath, bundleFile, cksum); err != nil { log.Errorf("error checking bundle %s: %s", bundleName, err) return "", xerrors.Errorf("error checking bundle: %s", err) } @@ -105,7 +139,7 @@ func (b *BundleFetcher) fetchURL(url, path string) error { return xerrors.Errorf("all attempts to fetch %s failed", url) } -func (b *BundleFetcher) fetch(release, bundleBasePath, bundleFile, bundleHash string) error { +func (b *BundleFetcher) fetchFromRelease(release, bundleBasePath, bundleFile, bundleHash string) error { bundleHashUrl := fmt.Sprintf("https://github.com/filecoin-project/builtin-actors/releases/download/%s/%s", release, bundleHash) bundleHashPath := filepath.Join(bundleBasePath, bundleHash) @@ -123,7 +157,12 @@ func (b *BundleFetcher) fetch(release, bundleBasePath, bundleFile, bundleHash st return nil } -func (b *BundleFetcher) check(bundleBasePath, bundleFile, bundleHash string) error { +func (b *BundleFetcher) fetchFromURL(bundleBasePath, bundleFile, url string) error { + bundleFilePath := filepath.Join(bundleBasePath, bundleFile) + return b.fetchURL(url, bundleFilePath) +} + +func (b *BundleFetcher) checkRelease(bundleBasePath, bundleFile, bundleHash string) error { bundleHashPath := filepath.Join(bundleBasePath, bundleHash) f, err := os.Open(bundleHashPath) if err != nil { @@ -138,13 +177,18 @@ func (b *BundleFetcher) check(bundleBasePath, bundleFile, bundleHash string) err parts := strings.Split(string(bs), " ") hashHex := parts[0] - expectedDigest, err := hex.DecodeString(hashHex) + + return b.checkHash(bundleBasePath, bundleFile, hashHex) +} + +func (b *BundleFetcher) checkHash(bundleBasePath, bundleFile, cksum string) error { + expectedDigest, err := hex.DecodeString(cksum) if err != nil { - return xerrors.Errorf("error decoding digest from %s: %w", bundleHashPath, err) + return xerrors.Errorf("error decoding digest from %s: %w", cksum, err) } bundleFilePath := filepath.Join(bundleBasePath, bundleFile) - f, err = os.Open(bundleFilePath) + f, err := os.Open(bundleFilePath) if err != nil { return xerrors.Errorf("error opening %s: %w", bundleFilePath, err) } diff --git a/node/bundle/manifest.go b/node/bundle/manifest.go index 45c632bdb..d726b0260 100644 --- a/node/bundle/manifest.go +++ b/node/bundle/manifest.go @@ -2,6 +2,7 @@ package bundle import ( "context" + "fmt" "io" "os" @@ -17,17 +18,35 @@ import ( "github.com/mitchellh/go-homedir" ) -func FetchAndLoadBundle(ctx context.Context, basePath string, bs blockstore.Blockstore, av actors.Version, rel, netw string) (cid.Cid, error) { +func FetchAndLoadBundleFromRelease(ctx context.Context, basePath string, bs blockstore.Blockstore, av actors.Version, rel, netw string) (cid.Cid, error) { fetcher, err := NewBundleFetcher(basePath) if err != nil { return cid.Undef, xerrors.Errorf("error creating fetcher for builtin-actors version %d: %w", av, err) } - path, err := fetcher.Fetch(int(av), rel, netw) + path, err := fetcher.FetchFromRelease(int(av), rel, netw) if err != nil { return cid.Undef, xerrors.Errorf("error fetching bundle for builtin-actors version %d: %w", av, err) } + return LoadBundle(ctx, bs, path, av) +} + +func FetchAndLoadBundleFromURL(ctx context.Context, basePath string, bs blockstore.Blockstore, av actors.Version, rel, netw, url, cksum string) (cid.Cid, error) { + fetcher, err := NewBundleFetcher(basePath) + if err != nil { + return cid.Undef, xerrors.Errorf("error creating fetcher for builtin-actors version %d: %w", av, err) + } + + path, err := fetcher.FetchFromURL(int(av), rel, netw, url, cksum) + if err != nil { + return cid.Undef, xerrors.Errorf("error fetching bundle for builtin-actors version %d: %w", av, err) + } + + return LoadBundle(ctx, bs, path, av) +} + +func LoadBundle(ctx context.Context, bs blockstore.Blockstore, path string, av actors.Version) (cid.Cid, error) { f, err := os.Open(path) if err != nil { return cid.Undef, xerrors.Errorf("error opening bundle for builtin-actors vresion %d: %w", av, err) @@ -52,11 +71,8 @@ func FetchAndLoadBundle(ctx context.Context, basePath string, bs blockstore.Bloc } // utility for blanket loading outside DI -func FetchAndLoadBundles(ctx context.Context, bs blockstore.Blockstore, bar map[actors.Version]string) error { - netw := build.NetworkBundle - if netw == "" { - netw = "mainnet" - } +func FetchAndLoadBundles(ctx context.Context, bs blockstore.Blockstore, bar map[actors.Version]build.Bundle) error { + netw := build.GetNetworkBundle() path := os.Getenv("LOTUS_PATH") if path == "" { @@ -67,9 +83,31 @@ func FetchAndLoadBundles(ctx context.Context, bs blockstore.Blockstore, bar map[ } } - for av, rel := range bar { - if _, err := FetchAndLoadBundle(ctx, path, bs, av, rel, netw); err != nil { - return err + for av, bd := range bar { + envvar := fmt.Sprintf("LOTUS_BUILTIN_ACTORS_V%d_BUNDLE", av) + switch { + case os.Getenv(envvar) != "": + // this is a local bundle, specified by an env var to load from the filesystem + path := os.Getenv(envvar) + + if _, err := LoadBundle(ctx, bs, path, av); err != nil { + return err + } + + case bd.Path[netw] != "": + if _, err := LoadBundle(ctx, bs, bd.Path[netw], av); err != nil { + return err + } + + case bd.URL[netw].URL != "": + if _, err := FetchAndLoadBundleFromURL(ctx, path, bs, av, bd.Release, netw, bd.URL[netw].URL, bd.URL[netw].Checksum); err != nil { + return err + } + + case bd.Release != "": + if _, err := FetchAndLoadBundleFromRelease(ctx, path, bs, av, bd.Release, netw); err != nil { + return err + } } } diff --git a/node/modules/builtin_actors.go b/node/modules/builtin_actors.go index 05b6e9e32..2ee63a9ce 100644 --- a/node/modules/builtin_actors.go +++ b/node/modules/builtin_actors.go @@ -2,6 +2,7 @@ package modules import ( "fmt" + "os" "sync" "go.uber.org/fx" @@ -24,14 +25,11 @@ func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRe // We can't put it as a dep in inputs causes a stack overflow in DI from circular dependency // So we pass it through ldflags instead - netw := build.NetworkBundle - if netw == "" { - netw = "mainnet" - } + netw := build.GetNetworkBundle() - for av, rel := range build.BuiltinActorReleases { + for av, bd := range build.BuiltinActorReleases { // first check to see if we know this release - key := dstore.NewKey(fmt.Sprintf("/builtin-actors/v%d/%s", av, rel)) + key := dstore.NewKey(fmt.Sprintf("/builtin-actors/v%d/%s", av, bd.Release)) data, err := ds.Get(ctx, key) switch err { @@ -65,10 +63,46 @@ func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRe return result, xerrors.Errorf("error loading %s from datastore: %w", key, err) } - // ok, we don't have it -- fetch it and add it to the blockstore - mfCid, err := bundle.FetchAndLoadBundle(ctx, r.Path(), bs, av, rel, netw) - if err != nil { - return result, err + // we haven't recorded it in the datastore, so we need to load it + envvar := fmt.Sprintf("LOTUS_BUILTIN_ACTORS_V%d_BUNDLE", av) + var mfCid cid.Cid + switch { + case os.Getenv(envvar) != "": + path := os.Getenv(envvar) + + mfCid, err = bundle.LoadBundle(ctx, bs, path, av) + if err != nil { + return result, err + } + + case bd.Path[netw] != "": + // this is a local bundle, load it directly from the filessystem + mfCid, err = bundle.LoadBundle(ctx, bs, bd.Path[netw], av) + if err != nil { + return result, err + } + + case bd.URL[netw].URL != "": + // fetch it from the specified URL + mfCid, err = bundle.FetchAndLoadBundleFromURL(ctx, r.Path(), bs, av, bd.Release, netw, bd.URL[netw].URL, bd.URL[netw].Checksum) + if err != nil { + return result, err + } + + case bd.Release != "": + // fetch it and add it to the blockstore + mfCid, err = bundle.FetchAndLoadBundleFromRelease(ctx, r.Path(), bs, av, bd.Release, netw) + if err != nil { + return result, err + } + + default: + return result, xerrors.Errorf("no release or path specified for version %d bundle", av) + } + + if bd.Development || bd.Release == "" { + // don't store the release key so that we always load development bundles + continue } // add the release key with the manifest to avoid reloading it in next restart. @@ -102,11 +136,27 @@ func LoadBuiltinActorsTesting(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtype testingBundleMx.Lock() defer testingBundleMx.Unlock() - for av, rel := range build.BuiltinActorReleases { - const basePath = "/tmp/lotus-testing" + const basePath = "/tmp/lotus-testing" + for av, bd := range build.BuiltinActorReleases { + switch { + case bd.Path[netw] != "": + if _, err := bundle.LoadBundle(ctx, bs, bd.Path[netw], av); err != nil { + return result, xerrors.Errorf("error loading testing bundle for builtin-actors version %d/%s: %w", av, netw, err) + } - if _, err := bundle.FetchAndLoadBundle(ctx, basePath, bs, av, rel, netw); err != nil { - return result, xerrors.Errorf("error loading bundle for builtin-actors vresion %d: %w", av, err) + case bd.URL[netw].URL != "": + // fetch it from the specified URL + if _, err := bundle.FetchAndLoadBundleFromURL(ctx, basePath, bs, av, bd.Release, netw, bd.URL[netw].URL, bd.URL[netw].Checksum); err != nil { + return result, err + } + + case bd.Release != "": + if _, err := bundle.FetchAndLoadBundleFromRelease(ctx, basePath, bs, av, bd.Release, netw); err != nil { + return result, xerrors.Errorf("error loading testing bundle for builtin-actors version %d/%s: %w", av, netw, err) + } + + default: + return result, xerrors.Errorf("no path or release specified for version %d testing bundle", av) } }