Merge pull request #8642 from filecoin-project/feat/nv16-bundles

[nv16] dynamic loading of builtin actor bundles

.circleci/config.yml

@@ -48,7 +48,6 @@ commands:
             - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
       - run: git submodule sync
       - run: git submodule update --init
-      - fetch_builtin_actors
   download-params:
     steps:
       - restore_cache:
@@ -84,12 +83,6 @@ commands:
           name: fetch all tags
           command: |
             git fetch --all
-  fetch_builtin_actors:
-    steps:
-      - run:
-          name: fetch builtin actor bundles
-          command: |
-            build/builtin-actors/fetch-bundles.sh
   packer_build:
     description: "Run a packer build"
     parameters:

.circleci/template.yml

@@ -48,7 +48,6 @@ commands:
             - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
       - run: git submodule sync
       - run: git submodule update --init
-      - fetch_builtin_actors
   download-params:
     steps:
       - restore_cache:
@@ -84,12 +83,6 @@ commands:
           name: fetch all tags
           command: |
             git fetch --all
-  fetch_builtin_actors:
-    steps:
-      - run:
-          name: fetch builtin actor bundles
-          command: |
-            build/builtin-actors/fetch-bundles.sh
   packer_build:
     description: "Run a packer build"
     parameters:

Makefile

@@ -57,14 +57,6 @@ build/.update-modules:
 
 # end git modules
 
-# builtin actor bundles
-builtin-actor-bundles:
-	./build/builtin-actors/fetch-bundles.sh
-
-BUILD_DEPS+=builtin-actor-bundles
-
-.PHONY: builtin-actor-bundles
-
 ## MAIN BINARIES
 
 CLEAN+=build/.update-modules
@@ -79,15 +71,19 @@ debug: GOFLAGS+=-tags=debug
 debug: build-devnets
 
 2k: GOFLAGS+=-tags=2k
+2k: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=devnet
 2k: build-devnets
 
 calibnet: GOFLAGS+=-tags=calibnet
+calibnet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=calibrationnet
 calibnet: build-devnets
 
 butterflynet: GOFLAGS+=-tags=butterflynet
+butterflynet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=butterflynet
 butterflynet: build-devnets
 
 interopnet: GOFLAGS+=-tags=interopnet
+interopnet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=caterpillarnet
 interopnet: build-devnets
 
 lotus: $(BUILD_DEPS)

build/builtin-actors/README.md

@@ -1,3 +0,0 @@
-This directory contains the builtin actors v8 bundle, to be emdedded in the binary.
-To change your actor bundle, prior to fixing for nv16 upgrade, generate a bundle using the actor
-bundler and place it in this directory, in a file named builtin-actors-v8.car

build/builtin-actors/bundles.env

@@ -1,2 +0,0 @@
-actors7_release=""
-actors8_release=c718ae4f957b1806

build/builtin-actors/fetch-bundles.sh

@@ -1,85 +0,0 @@
-#!/bin/bash
-set -e
-
-cd "$(dirname "$0")"
-
-. bundles.env
-
-die() {
-    echo "$1"
-    exit 1
-}
-
-fetch() {
-    ver=$1
-    rel=$2
-
-    if [ ! -e $ver ]; then
-        mkdir $ver
-    fi
-
-    if [ -e $ver/release ]; then
-       cur=$(cat $ver/release)
-       if [ $cur == $rel ]; then
-           return 0
-       fi
-    fi
-
-    for net in mainnet caterpillarnet butterflynet calibrationnet devnet testing testing-fake-proofs; do
-        fetch_bundle $ver $rel $net
-    done
-
-    # remember the current release so that we don't have to hit github unless we have modified it
-    echo $rel > $ver/release
-}
-
-fetch_bundle() {
-    ver=$1
-    rel=$2
-    net=$3
-
-    target=builtin-actors-$net.car
-    hash=builtin-actors-$net.sha256
-
-    pushd $ver
-
-    # fetch the hash first and check if it matches what we (may) already have
-    curl -L --retry 3 https://github.com/filecoin-project/builtin-actors/releases/download/$rel/$hash -o $hash || die "error fetching hash for $ver/$net"
-    if [ -e $target ]; then
-        if (shasum -a 256 --check $hash); then
-            popd
-            return 0
-        fi
-    fi
-
-    # we don't have the (correct) bundle, fetch it
-    curl -L --retry 3 https://github.com/filecoin-project/builtin-actors/releases/download/$rel/$target -o $target || die "error fetching bundle for $ver/$net"
-    # verify
-    shasum -a 256 --check $hash || die "hash mismatch"
-    # all good
-    popd
-}
-
-touch_bundles() {
-    ver=$1
-
-    if [ ! -e $ver ]; then
-        mkdir $ver
-    fi
-
-    for net in mainnet caterpillarnet butterflynet calibrationnet devnet testing testing-fake-proofs; do
-        touch $ver/builtin-actors-$net.car
-    done
-}
-
-if [ -n "$actors7_release" ]; then
-    fetch v7 "$actors7_release"
-else
-    touch_bundles v7
-fi
-
-if [ -n "$actors8_release" ]; then
-    fetch v8 "$actors8_release"
-else
-    touch_bundles v8
-fi

build/builtin_actors.go

@@ -0,0 +1,27 @@
+package build
+
+import (
+	"bytes"
+
+	"github.com/filecoin-project/lotus/chain/actors"
+
+	"github.com/BurntSushi/toml"
+)
+
+var BuiltinActorReleases map[actors.Version]string
+
+func init() {
+	BuiltinActorReleases = make(map[actors.Version]string)
+
+	spec := BundleSpec{}
+
+	r := bytes.NewReader(BuiltinActorBundles)
+	_, err := toml.DecodeReader(r, &spec)
+	if err != nil {
+		panic(err)
+	}
+
+	for _, b := range spec.Bundles {
+		BuiltinActorReleases[b.Version] = b.Release
+	}
+}

build/builtin_actors_2k.go

@@ -1,22 +0,0 @@
-//go:build debug || 2k || testground
-// +build debug 2k testground
-
-package build
-
-import (
-	_ "embed"
-)
-
-//go:embed builtin-actors/v8/builtin-actors-devnet.car
-var actorsv8 []byte
-
-func BuiltinActorsV8Bundle() []byte {
-	return actorsv8
-}
-
-//go:embed builtin-actors/v7/builtin-actors-devnet.car
-var actorsv7 []byte
-
-func BuiltinActorsV7Bundle() []byte {
-	return actorsv7
-}

build/builtin_actors_butterfly.go

@@ -1,22 +0,0 @@
-//go:build butterflynet
-// +build butterflynet
-
-package build
-
-import (
-	_ "embed"
-)
-
-//go:embed builtin-actors/v8/builtin-actors-butterflynet.car
-var actorsv8 []byte
-
-func BuiltinActorsV8Bundle() []byte {
-	return actorsv8
-}
-
-//go:embed builtin-actors/v7/builtin-actors-butterflynet.car
-var actorsv7 []byte
-
-func BuiltinActorsV7Bundle() []byte {
-	return actorsv7
-}

build/builtin_actors_calibnet.go

@@ -1,22 +0,0 @@
-//go:build calibnet
-// +build calibnet
-
-package build
-
-import (
-	_ "embed"
-)
-
-//go:embed builtin-actors/v8/builtin-actors-calibrationnet.car
-var actorsv8 []byte
-
-func BuiltinActorsV8Bundle() []byte {
-	return actorsv8
-}
-
-//go:embed builtin-actors/v7/builtin-actors-calibrationnet.car
-var actorsv7 []byte
-
-func BuiltinActorsV7Bundle() []byte {
-	return actorsv7
-}

build/builtin_actors_interop.go

@@ -1,22 +0,0 @@
-//go:build interopnet
-// +build interopnet
-
-package build
-
-import (
-	_ "embed"
-)
-
-//go:embed builtin-actors/v8/builtin-actors-caterpillarnet.car
-var actorsv8 []byte
-
-func BuiltinActorsV8Bundle() []byte {
-	return actorsv8
-}
-
-//go:embed builtin-actors/v7/builtin-actors-caterpillarnet.car
-var actorsv7 []byte
-
-func BuiltinActorsV7Bundle() []byte {
-	return actorsv7
-}

build/builtin_actors_mainnet.go

@@ -1,22 +0,0 @@
-//go:build !debug && !2k && !testground && !calibnet && !nerpanet && !butterflynet && !interopnet
-// +build !debug,!2k,!testground,!calibnet,!nerpanet,!butterflynet,!interopnet
-
-package build
-
-import (
-	_ "embed"
-)
-
-//go:embed builtin-actors/v8/builtin-actors-mainnet.car
-var actorsv8 []byte
-
-func BuiltinActorsV8Bundle() []byte {
-	return actorsv8
-}
-
-//go:embed builtin-actors/v7/builtin-actors-mainnet.car
-var actorsv7 []byte
-
-func BuiltinActorsV7Bundle() []byte {
-	return actorsv7
-}

build/bundle.go

@@ -0,0 +1,21 @@
+package build
+
+import (
+	_ "embed"
+
+	"github.com/filecoin-project/lotus/chain/actors"
+)
+
+var NetworkBundle string
+
+//go:embed bundles.toml
+var BuiltinActorBundles []byte
+
+type BundleSpec struct {
+	Bundles []Bundle
+}
+
+type Bundle struct {
+	Version actors.Version
+	Release string
+}

build/bundles.toml

@@ -0,0 +1,3 @@
+[[bundles]]
+version = 8
+release = "b71c2ec785aec23d"

chain/actors/manifest.go

@@ -130,12 +130,3 @@ func LoadBundle(ctx context.Context, bs blockstore.Blockstore, av Version, data
 
 	return nil
 }
-
-func LoadManifestFromBundle(ctx context.Context, bs blockstore.Blockstore, av Version, data []byte) error {
-	if err := LoadBundle(ctx, bs, av, data); err != nil {
-		return err
-	}
-
-	cborStore := cbor.NewCborStore(bs)
-	return LoadManifests(ctx, cborStore)
-}

cli/init.go

@@ -2,21 +2,20 @@ package cli
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/node/bundle"
 )
 
 func init() {
 	// preload manifest so that we have the correct code CID inventory for cli since that doesn't
 	// go through CI
-	if len(build.BuiltinActorsV8Bundle()) > 0 {
+	// TODO loading the bundle in every cli invocation adds some latency; we should figure out a way
+	//      to load actor CIDs without incurring this hit.
 	bs := blockstore.NewMemory()
 
-		if err := actors.LoadManifestFromBundle(context.TODO(), bs, actors.Version8, build.BuiltinActorsV8Bundle()); err != nil {
+	if err := bundle.FetchAndLoadBundles(context.Background(), bs, build.BuiltinActorReleases); err != nil {
-			panic(fmt.Errorf("error loading actor manifest: %w", err))
+		panic(err)
-		}
 	}
 }

cmd/lotus-miner/init.go

@@ -58,6 +58,7 @@ import (
 	"github.com/filecoin-project/lotus/journal"
 	"github.com/filecoin-project/lotus/journal/fsjournal"
 	storageminer "github.com/filecoin-project/lotus/miner"
+	"github.com/filecoin-project/lotus/node/bundle"
 	"github.com/filecoin-project/lotus/node/modules"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/repo"
@@ -216,12 +217,11 @@ var initCmd = &cli.Command{
 				return err
 			}
 
-			if len(build.BuiltinActorsV8Bundle()) > 0 {
+			// load bundles
 			bs := blockstore.NewMemory()
 
-				if err := actors.LoadManifestFromBundle(context.TODO(), bs, actors.Version8, build.BuiltinActorsV8Bundle()); err != nil {
+			if err := bundle.FetchAndLoadBundles(ctx, bs, build.BuiltinActorReleases); err != nil {
-					return xerrors.Errorf("error loading actor manifest: %w", err)
+				return err
-				}
 			}
 
 			var localPaths []stores.LocalPath

cmd/lotus-seed/genesis.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"context"
 	"encoding/csv"
 	"encoding/json"
 	"fmt"
@@ -13,10 +12,10 @@ import (
 	"github.com/filecoin-project/go-state-types/network"
 
 	"github.com/filecoin-project/lotus/blockstore"
-	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 	"github.com/filecoin-project/lotus/journal"
+	"github.com/filecoin-project/lotus/node/bundle"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/modules/testing"
 	"github.com/google/uuid"
@@ -580,11 +579,12 @@ var genesisCarCmd = &cli.Command{
 		jrnl := journal.NilJournal()
 		bstor := blockstore.WrapIDStore(blockstore.NewMemorySync())
 		sbldr := vm.Syscalls(ffiwrapper.ProofVerifier)
-		if len(build.BuiltinActorsV8Bundle()) > 0 {
+
-			if err := actors.LoadManifestFromBundle(context.TODO(), bstor, actors.Version8, build.BuiltinActorsV8Bundle()); err != nil {
+		// load appropriate bundles
-				return xerrors.Errorf("error loading actor manifest: %w", err)
+		if err := bundle.FetchAndLoadBundles(c.Context, bstor, build.BuiltinActorReleases); err != nil {
-			}
+			return err
 		}
+
 		_, err := testing.MakeGenesis(ofile, c.Args().First())(bstor, sbldr, jrnl, dtypes.BuiltinActorsLoaded{})()
 		return err
 	},

node/builder_chain.go

@@ -53,7 +53,7 @@ var ChainNode = Options(
 	//      right manifest cid.
 	//      This restriction will be lifted once we have the final actors v8 bundle and we know
 	//      the manifest cid.
-	Override(new(dtypes.BuiltinActorsLoaded), modules.LoadBultinActors),
+	Override(new(dtypes.BuiltinActorsLoaded), modules.LoadBuiltinActors),
 
 	// Consensus settings
 	Override(new(dtypes.DrandSchedule), modules.BuiltinDrandConfig),

node/builder_miner.go

@@ -53,7 +53,7 @@ var MinerNode = Options(
 	Override(new(*storage.AddressSelector), modules.AddressSelector(nil)),
 
 	// builtin actors manifest
-	Override(new(dtypes.BuiltinActorsLoaded), modules.LoadBultinActors),
+	Override(new(dtypes.BuiltinActorsLoaded), modules.LoadBuiltinActors),
 	Override(new(dtypes.UniversalBlockstore), modules.MemoryBlockstore),
 )
 

node/bundle/bundle.go

@@ -0,0 +1,172 @@
+package bundle
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"golang.org/x/xerrors"
+
+	logging "github.com/ipfs/go-log/v2"
+)
+
+var log = logging.Logger("bundle-fetcher")
+
+type BundleFetcher struct {
+	path string
+}
+
+func NewBundleFetcher(basepath string) (*BundleFetcher, error) {
+	path := filepath.Join(basepath, "builtin-actors")
+	if err := os.MkdirAll(path, 0755); err != nil {
+		return nil, xerrors.Errorf("error making bundle directory %s: %w", path, err)
+	}
+
+	return &BundleFetcher{path: path}, nil
+}
+
+func (b *BundleFetcher) Fetch(version int, release, netw string) (path string, err error) {
+	bundleName := fmt.Sprintf("builtin-actors-%s", netw)
+	bundleFile := fmt.Sprintf("%s.car", bundleName)
+	bundleHash := fmt.Sprintf("%s.sha256", bundleName)
+	bundleBasePath := filepath.Join(b.path, fmt.Sprintf("v%d", version), release)
+
+	if err := os.MkdirAll(bundleBasePath, 0755); err != nil {
+		return "", xerrors.Errorf("error making bundle directory %s: %w", bundleBasePath, err)
+	}
+
+	// check if it exists; if it does, check the hash
+	bundleFilePath := filepath.Join(bundleBasePath, bundleFile)
+	if _, err := os.Stat(bundleFilePath); err == nil {
+		err := b.check(bundleBasePath, bundleFile, bundleHash)
+		if err == nil {
+			return bundleFilePath, nil
+		}
+
+		log.Warnf("invalid bundle %s: %s; refetching", bundleName, err)
+	}
+
+	log.Infof("fetching bundle %s", bundleFile)
+	if err := b.fetch(release, bundleBasePath, bundleFile, bundleHash); err != nil {
+		log.Errorf("error fetching bundle %s: %s", bundleName, err)
+		return "", xerrors.Errorf("error fetching bundle: %w", err)
+	}
+
+	if err := b.check(bundleBasePath, bundleFile, bundleHash); err != nil {
+		log.Errorf("error checking bundle %s: %s", bundleName, err)
+		return "", xerrors.Errorf("error checking bundle: %s", err)
+	}
+
+	return bundleFilePath, nil
+}
+
+func (b *BundleFetcher) fetchURL(url, path string) error {
+	log.Infof("fetching URL: %s", url)
+
+	for i := 0; i < 3; i++ {
+		resp, err := http.Get(url) //nolint
+		if err != nil {
+			if isTemporary(err) {
+				log.Warnf("temporary error fetching %s: %s; retrying in 1s", url, err)
+				time.Sleep(time.Second)
+				continue
+			}
+			return xerrors.Errorf("error fetching %s: %w", url, err)
+		}
+		defer resp.Body.Close() //nolint
+
+		if resp.StatusCode != http.StatusOK {
+			log.Warnf("unexpected response fetching %s: %s (%d); retrying in 1s", url, resp.Status, resp.StatusCode)
+			time.Sleep(time.Second)
+			continue
+		}
+
+		f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+		if err != nil {
+			return xerrors.Errorf("error opening %s for writing: %w", path, err)
+		}
+		defer f.Close() //nolint
+
+		if _, err := io.Copy(f, resp.Body); err != nil {
+			return xerrors.Errorf("error writing %s: %w", path, err)
+		}
+
+		return nil
+	}
+
+	return xerrors.Errorf("all attempts to fetch %s failed", url)
+}
+
+func (b *BundleFetcher) fetch(release, bundleBasePath, bundleFile, bundleHash string) error {
+	bundleHashUrl := fmt.Sprintf("https://github.com/filecoin-project/builtin-actors/releases/download/%s/%s",
+		release, bundleHash)
+	bundleHashPath := filepath.Join(bundleBasePath, bundleHash)
+	if err := b.fetchURL(bundleHashUrl, bundleHashPath); err != nil {
+		return err
+	}
+
+	bundleFileUrl := fmt.Sprintf("https://github.com/filecoin-project/builtin-actors/releases/download/%s/%s",
+		release, bundleFile)
+	bundleFilePath := filepath.Join(bundleBasePath, bundleFile)
+	if err := b.fetchURL(bundleFileUrl, bundleFilePath); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (b *BundleFetcher) check(bundleBasePath, bundleFile, bundleHash string) error {
+	bundleHashPath := filepath.Join(bundleBasePath, bundleHash)
+	f, err := os.Open(bundleHashPath)
+	if err != nil {
+		return xerrors.Errorf("error opening %s: %w", bundleHashPath, err)
+	}
+	defer f.Close() //nolint
+
+	bs, err := io.ReadAll(f)
+	if err != nil {
+		return xerrors.Errorf("error reading %s: %w", bundleHashPath, err)
+	}
+
+	parts := strings.Split(string(bs), " ")
+	hashHex := parts[0]
+	expectedDigest, err := hex.DecodeString(hashHex)
+	if err != nil {
+		return xerrors.Errorf("error decoding digest from %s: %w", bundleHashPath, err)
+	}
+
+	bundleFilePath := filepath.Join(bundleBasePath, bundleFile)
+	f, err = os.Open(bundleFilePath)
+	if err != nil {
+		return xerrors.Errorf("error opening %s: %w", bundleFilePath, err)
+	}
+	defer f.Close() //nolint
+
+	h256 := sha256.New()
+	if _, err := io.Copy(h256, f); err != nil {
+		return xerrors.Errorf("error computing digest for %s: %w", bundleFilePath, err)
+	}
+	digest := h256.Sum(nil)
+
+	if !bytes.Equal(digest, expectedDigest) {
+		return xerrors.Errorf("hash mismatch")
+	}
+
+	return nil
+}
+
+func isTemporary(err error) bool {
+	if ne, ok := err.(net.Error); ok {
+		return ne.Temporary()
+	}
+
+	return false
+}

node/bundle/manifest.go

@@ -0,0 +1,82 @@
+package bundle
+
+import (
+	"context"
+	"io"
+	"os"
+
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/lotus/blockstore"
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors"
+
+	cid "github.com/ipfs/go-cid"
+	cbor "github.com/ipfs/go-ipld-cbor"
+
+	"github.com/mitchellh/go-homedir"
+)
+
+func FetchAndLoadBundle(ctx context.Context, basePath string, bs blockstore.Blockstore, av actors.Version, rel, netw string) (cid.Cid, error) {
+	fetcher, err := NewBundleFetcher(basePath)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("error creating fetcher for builtin-actors version %d: %w", av, err)
+	}
+
+	path, err := fetcher.Fetch(int(av), rel, netw)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("error fetching bundle for builtin-actors version %d: %w", av, err)
+	}
+
+	f, err := os.Open(path)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("error opening bundle for builtin-actors vresion %d: %w", av, err)
+	}
+	defer f.Close() //nolint
+
+	data, err := io.ReadAll(f)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("error reading bundle for builtin-actors vresion %d: %w", av, err)
+	}
+
+	if err := actors.LoadBundle(ctx, bs, av, data); err != nil {
+		return cid.Undef, xerrors.Errorf("error loading bundle for builtin-actors vresion %d: %w", av, err)
+	}
+
+	mfCid, ok := actors.GetManifest(av)
+	if !ok {
+		return cid.Undef, xerrors.Errorf("missing manifest CID for builtin-actors vrsion %d", av)
+	}
+
+	return mfCid, nil
+}
+
+// utility for blanket loading outside DI
+func FetchAndLoadBundles(ctx context.Context, bs blockstore.Blockstore, bar map[actors.Version]string) error {
+	netw := build.NetworkBundle
+	if netw == "" {
+		netw = "mainnet"
+	}
+
+	path := os.Getenv("LOTUS_PATH")
+	if path == "" {
+		var err error
+		path, err = homedir.Expand("~/.lotus")
+		if err != nil {
+			return err
+		}
+	}
+
+	for av, rel := range bar {
+		if _, err := FetchAndLoadBundle(ctx, path, bs, av, rel, netw); err != nil {
+			return err
+		}
+	}
+
+	cborStore := cbor.NewCborStore(bs)
+	if err := actors.LoadManifests(ctx, cborStore); err != nil {
+		return err
+	}
+
+	return nil
+}

node/modules/builtin_actors.go

@@ -2,40 +2,82 @@ package modules
 
 import (
 	"fmt"
-	"io"
+	"sync"
-	"os"
 
 	"go.uber.org/fx"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/node/bundle"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/modules/helpers"
+	"github.com/filecoin-project/lotus/node/repo"
 
+	cid "github.com/ipfs/go-cid"
+	dstore "github.com/ipfs/go-datastore"
 	cbor "github.com/ipfs/go-ipld-cbor"
 )
 
-func LoadBultinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtypes.UniversalBlockstore) (result dtypes.BuiltinActorsLoaded, err error) {
+func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo, bs dtypes.UniversalBlockstore, ds dtypes.MetadataDS) (result dtypes.BuiltinActorsLoaded, err error) {
 	ctx := helpers.LifecycleCtx(mctx, lc)
 
-	// TODO eventually we want this to start with bundle/manifest CIDs and fetch them from IPFS if
+	// We can't put it as a dep in inputs causes a stack overflow in DI from circular dependency
-	//      not already loaded.
+	// So we pass it through ldflags instead
-	//      For now, we just embed the v8 bundle and adjust the manifest CIDs for the migration/actor
+	netw := build.NetworkBundle
-	//      metadata.
+	if netw == "" {
-	if len(build.BuiltinActorsV8Bundle()) > 0 {
+		netw = "mainnet"
-		if err := actors.LoadBundle(ctx, bs, actors.Version8, build.BuiltinActorsV8Bundle()); err != nil {
+	}
-			return result, err
+
-		}
+	for av, rel := range build.BuiltinActorReleases {
-	}
+		// first check to see if we know this release
-
+		key := dstore.NewKey(fmt.Sprintf("/builtin-actors/v%d/%s", av, rel))
-	// for testing -- need to also set LOTUS_USE_FVM_CUSTOM_BUNDLE=1 to force the fvm to use it.
+
-	if len(build.BuiltinActorsV7Bundle()) > 0 {
+		data, err := ds.Get(ctx, key)
-		if err := actors.LoadBundle(ctx, bs, actors.Version7, build.BuiltinActorsV7Bundle()); err != nil {
+		switch err {
+		case nil:
+			// ok, we do, this should be the manifest cid
+			mfCid, err := cid.Cast(data)
+			if err != nil {
+				return result, xerrors.Errorf("error parsing cid for %s: %w", key, err)
+			}
+
+			// check the blockstore for existence of the manifest
+			has, err := bs.Has(ctx, mfCid)
+			if err != nil {
+				return result, xerrors.Errorf("error checking blockstore for manifest cid %s: %w", mfCid, err)
+			}
+
+			if has {
+				// it's there, no need to reload the bundle to the blockstore; just add it to the manifest list.
+				actors.AddManifest(av, mfCid)
+				continue
+			}
+
+			// we have a release key but don't have the manifest in the blockstore; maybe the user
+			// nuked his blockstore to restart from a snapshot. So fallthrough to refetch (if necessary)
+			// and reload the bundle.
+
+		case dstore.ErrNotFound:
+			// we don't have a release key, we need to load the bundle
+
+		default:
+			return result, xerrors.Errorf("error loading %s from datastore: %w", key, err)
+		}
+
+		// ok, we don't have it -- fetch it and add it to the blockstore
+		mfCid, err := bundle.FetchAndLoadBundle(ctx, r.Path(), bs, av, rel, netw)
+		if err != nil {
 			return result, err
 		}
+
+		// add the release key with the manifest to avoid reloading it in next restart.
+		if err := ds.Put(ctx, key, mfCid.Bytes()); err != nil {
+			return result, xerrors.Errorf("error storing manifest CID for builtin-actors vrsion %d to the datastore: %w", av, err)
+		}
 	}
 
+	// we've loaded all the bundles, now load the manifests to get actor code CIDs.
 	cborStore := cbor.NewCborStore(bs)
 	if err := actors.LoadManifests(ctx, cborStore); err != nil {
 		return result, xerrors.Errorf("error loading actor manifests: %w", err)
@@ -45,38 +87,26 @@ func LoadBultinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtypes.Univer
 }
 
 // for itests
+var testingBundleMx sync.Mutex
+
 func LoadBuiltinActorsTesting(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtypes.UniversalBlockstore) (result dtypes.BuiltinActorsLoaded, err error) {
 	ctx := helpers.LifecycleCtx(mctx, lc)
 
-	base := os.Getenv("LOTUS_SRC_DIR")
+	var netw string
-	if base == "" {
-		base = "."
-	}
-
-	var template string
 	if build.InsecurePoStValidation {
-		template = "%s/build/builtin-actors/v%d/builtin-actors-testing-fake-proofs.car"
+		netw = "testing-fake-proofs"
 	} else {
-		template = "%s/build/builtin-actors/v%d/builtin-actors-testing.car"
+		netw = "testing"
 	}
 
-	for _, ver := range []actors.Version{actors.Version8} {
+	testingBundleMx.Lock()
-		path := fmt.Sprintf(template, base, ver)
+	defer testingBundleMx.Unlock()
 
-		log.Infof("loading testing bundle: %s", path)
+	for av, rel := range build.BuiltinActorReleases {
+		const basePath = "/tmp/lotus-testing"
 
-		file, err := os.Open(path)
+		if _, err := bundle.FetchAndLoadBundle(ctx, basePath, bs, av, rel, netw); err != nil {
-		if err != nil {
+			return result, xerrors.Errorf("error loading bundle for builtin-actors vresion %d: %w", av, err)
-			return result, xerrors.Errorf("error opening v%d bundle: %w", ver, err)
-		}
-
-		bundle, err := io.ReadAll(file)
-		if err != nil {
-			return result, xerrors.Errorf("error reading v%d bundle: %w", ver, err)
-		}
-
-		if err := actors.LoadBundle(ctx, bs, actors.Version8, bundle); err != nil {
-			return result, xerrors.Errorf("error loading v%d bundle: %w", ver, err)
 		}
 	}