Merge pull request #8509 from filecoin-project/feat/actor-bundles

.circleci/config.yml

@@ -86,22 +86,10 @@ commands:
             git fetch --all
   fetch_builtin_actors:
     steps:
-      - restore_cache:
-          name: restore builtin actors bundle cache
-          keys:
-            - 'builtin-actors-v8.car'
-          paths:
-            - build/builtin-actors/
       - run:
           name: fetch builtin actor bundles
           command: |
             build/builtin-actors/fetch-bundles.sh
-      - save_cache:
-          name: save builtin actors bundle cache
-          key: 'builtin-actors-v8.car'
-          paths:
-            - build/builtin-actors/
-
   packer_build:
     description: "Run a packer build"
     parameters:
@@ -214,6 +202,7 @@ jobs:
           environment:
             TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >>
             SKIP_CONFORMANCE: "1"
+            LOTUS_SRC_DIR: /home/circleci/project
           command: |
             mkdir -p /tmp/test-reports/<< parameters.suite >>
             mkdir -p /tmp/test-artifacts
@@ -1125,4 +1114,4 @@ workflows:
               only:
                 - master
     jobs:
-      - publish-packer-snap
+      - publish-packer-snap

.circleci/template.yml

@@ -86,22 +86,10 @@ commands:
             git fetch --all
   fetch_builtin_actors:
     steps:
-      - restore_cache:
-          name: restore builtin actors bundle cache
-          keys:
-            - 'builtin-actors-v8.car'
-          paths:
-            - build/builtin-actors/
       - run:
           name: fetch builtin actor bundles
           command: |
             build/builtin-actors/fetch-bundles.sh
-      - save_cache:
-          name: save builtin actors bundle cache
-          key: 'builtin-actors-v8.car'
-          paths:
-            - build/builtin-actors/
-
   packer_build:
     description: "Run a packer build"
     parameters:
@@ -214,6 +202,7 @@ jobs:
           environment:
             TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >>
             SKIP_CONFORMANCE: "1"
+            LOTUS_SRC_DIR: /home/circleci/project
           command: |
             mkdir -p /tmp/test-reports/<< parameters.suite >>
             mkdir -p /tmp/test-artifacts
@@ -930,4 +919,4 @@ workflows:
               only:
                 - master
     jobs:
-      - publish-packer-snap
+      - publish-packer-snap

.gitignore

@@ -49,4 +49,4 @@ bin/tmp/*
 .idea
 scratchpad
 
-build/builtin-actors/*.car
+build/builtin-actors/v*

build/builtin-actors/bundles.env

@@ -0,0 +1,2 @@
+actors7_release=""
+actors8_release=a9635268e3b359bd

build/builtin-actors/fetch-bundles.sh

@@ -3,50 +3,83 @@ set -e
 
 cd "$(dirname "$0")"
 
-# gateway to use
-dweb="dweb.link"
-
-actors7_cid=""
-actors7_hash=""
-actors8_cid="bafybeictmywrut5tprz5fnoti6adfwuvixvrfardhqwldxosmdsfavc56e"
-actors8_hash="687b38f59b0c32800f55a8f1f303de214ec173c90e653984d67f393bc41c1416"
+. bundles.env
 
 die() {
     echo "$1"
     exit 1
 }
 
-check() {
-    file=$1
-    hash=$2
-    if [ -e "$file" ]; then
-        echo "$hash  $file" | shasum -a 256 --check
-    else
-        return 1
-    fi
-}
-
 fetch() {
-    output=$1
-    cid=$2
-    hash=$3
-    if (check "$output" "$hash"); then
-        return 0
-    else
-        echo "fetching $cid to $output"
-        curl --retry 3 -k "https://$dweb/ipfs/$cid" -o "$output"
-        check "$output" "$hash" || die "hash mismatch"
+    ver=$1
+    rel=$2
+
+    if [ ! -e $ver ]; then
+        mkdir $ver
     fi
+
+    if [ -e $ver/release ]; then
+       cur=$(cat $ver/release)
+       if [ $cur == $rel ]; then
+           return 0
+       fi
+    fi
+
+    for net in mainnet caterpillarnet butterflynet calibrationnet devnet testing; do
+        fetch_bundle $ver $rel $net
+    done
+
+    # remember the current release so that we don't have to hit github unless we have modified it
+    echo $rel > $ver/release
 }
 
-if [ -n "$actors7_cid" ]; then
-    fetch builtin-actors-v7.car "$actors7_cid" "$actors7_hash"
+fetch_bundle() {
+    ver=$1
+    rel=$2
+    net=$3
+
+    target=builtin-actors-$net.car
+    hash=builtin-actors-$net.sha256
+
+    pushd $ver
+
+    # fetch the hash first and check if it matches what we (may) already have
+    curl -L --retry 3 https://github.com/filecoin-project/builtin-actors/releases/download/$rel/$hash -o $hash || die "error fetching hash for $ver/$net"
+    if [ -e $target ]; then
+        if (shasum -a 256 --check $hash); then
+            popd
+            return 0
+        fi
+    fi
+
+    # we don't have the (correct) bundle, fetch it
+    curl -L --retry 3 https://github.com/filecoin-project/builtin-actors/releases/download/$rel/$target -o $target || die "error fetching bundle for $ver/$net"
+    # verify
+    shasum -a 256 --check $hash || die "hash mismatch"
+    # all good
+    popd
+}
+
+touch_bundles() {
+    ver=$1
+
+    if [ ! -e $ver ]; then
+        mkdir $ver
+    fi
+
+    for net in mainnet caterpillarnet butterflynet calibrationnet devnet testing; do
+        touch $ver/builtin-actors-$net.car
+    done
+}
+
+if [ -n "$actors7_release" ]; then
+    fetch v7 "$actors7_release"
 else
-    touch builtin-actors-v7.car
+    touch_bundles v7
 fi
 
-if [ -n "$actors8_cid" ]; then
-    fetch builtin-actors-v8.car "$actors8_cid" "$actors8_hash"
+if [ -n "$actors8_release" ]; then
+    fetch v8 "$actors8_release"
 else
-    touch builtin-actors-v8.car
+    touch_bundles v8
 fi

build/builtin_actors_2k.go

@@ -0,0 +1,22 @@
+//go:build debug || 2k || testground
+// +build debug 2k testground
+
+package build
+
+import (
+	_ "embed"
+)
+
+//go:embed builtin-actors/v8/builtin-actors-devnet.car
+var actorsv8 []byte
+
+func BuiltinActorsV8Bundle() []byte {
+	return actorsv8
+}
+
+//go:embed builtin-actors/v7/builtin-actors-devnet.car
+var actorsv7 []byte
+
+func BuiltinActorsV7Bundle() []byte {
+	return actorsv7
+}

build/builtin_actors_butterfly.go

@@ -0,0 +1,22 @@
+//go:build butterflynet
+// +build butterflynet
+
+package build
+
+import (
+	_ "embed"
+)
+
+//go:embed builtin-actors/v8/builtin-actors-butterflynet.car
+var actorsv8 []byte
+
+func BuiltinActorsV8Bundle() []byte {
+	return actorsv8
+}
+
+//go:embed builtin-actors/v7/builtin-actors-butterflynet.car
+var actorsv7 []byte
+
+func BuiltinActorsV7Bundle() []byte {
+	return actorsv7
+}

build/builtin_actors_calibnet.go

@@ -1,17 +1,20 @@
+//go:build calibnet
+// +build calibnet
+
 package build
 
 import (
 	_ "embed"
 )
 
-//go:embed builtin-actors/builtin-actors-v8.car
+//go:embed builtin-actors/v8/builtin-actors-calibrationnet.car
 var actorsv8 []byte
 
 func BuiltinActorsV8Bundle() []byte {
 	return actorsv8
 }
 
-//go:embed builtin-actors/builtin-actors-v7.car
+//go:embed builtin-actors/v7/builtin-actors-calibrationnet.car
 var actorsv7 []byte
 
 func BuiltinActorsV7Bundle() []byte {

build/builtin_actors_interop.go

@@ -0,0 +1,22 @@
+//go:build interopnet
+// +build interopnet
+
+package build
+
+import (
+	_ "embed"
+)
+
+//go:embed builtin-actors/v8/builtin-actors-caterpillarnet.car
+var actorsv8 []byte
+
+func BuiltinActorsV8Bundle() []byte {
+	return actorsv8
+}
+
+//go:embed builtin-actors/v7/builtin-actors-caterpillarnet.car
+var actorsv7 []byte
+
+func BuiltinActorsV7Bundle() []byte {
+	return actorsv7
+}

build/builtin_actors_mainnet.go

@@ -0,0 +1,22 @@
+//go:build !debug && !2k && !testground && !calibnet && !nerpanet && !butterflynet && !interopnet
+// +build !debug,!2k,!testground,!calibnet,!nerpanet,!butterflynet,!interopnet
+
+package build
+
+import (
+	_ "embed"
+)
+
+//go:embed builtin-actors/v8/builtin-actors-mainnet.car
+var actorsv8 []byte
+
+func BuiltinActorsV8Bundle() []byte {
+	return actorsv8
+}
+
+//go:embed builtin-actors/v7/builtin-actors-mainnet.car
+var actorsv7 []byte
+
+func BuiltinActorsV7Bundle() []byte {
+	return actorsv7
+}

cmd/lotus-seed/genesis.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"encoding/csv"
 	"encoding/json"
 	"fmt"
@@ -12,9 +13,11 @@ import (
 	"github.com/filecoin-project/go-state-types/network"
 
 	"github.com/filecoin-project/lotus/blockstore"
+	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 	"github.com/filecoin-project/lotus/journal"
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/modules/testing"
 	"github.com/google/uuid"
 	"github.com/mitchellh/go-homedir"
@@ -577,7 +580,12 @@ var genesisCarCmd = &cli.Command{
 		jrnl := journal.NilJournal()
 		bstor := blockstore.WrapIDStore(blockstore.NewMemorySync())
 		sbldr := vm.Syscalls(ffiwrapper.ProofVerifier)
-		_, err := testing.MakeGenesis(ofile, c.Args().First())(bstor, sbldr, jrnl)()
+		if len(build.BuiltinActorsV8Bundle()) > 0 {
+			if err := actors.LoadManifestFromBundle(context.TODO(), bstor, actors.Version8, build.BuiltinActorsV8Bundle()); err != nil {
+				return xerrors.Errorf("error loading actor manifest: %w", err)
+			}
+		}
+		_, err := testing.MakeGenesis(ofile, c.Args().First())(bstor, sbldr, jrnl, dtypes.BuiltinActorsLoaded{})()
 		return err
 	},
 }

extern/filecoin-ffi

@@ -1 +1 @@
-Subproject commit 0c6d698610a98c9efcefc58d5277ee8813d80033
+Subproject commit 4ff4fec93f1b40e1351af978b8df5de1156f389d

node/builder.go

@@ -384,6 +384,9 @@ func Test() Option {
 		Unset(new(*peermgr.PeerMgr)),
 		Override(new(beacon.Schedule), testing.RandomBeacon),
 		Override(new(*storageadapter.DealPublisher), storageadapter.NewDealPublisher(nil, storageadapter.PublishMsgConfig{})),
+		// use the testing bundles
+		Unset(new(dtypes.BuiltinActorsLoaded)),
+		Override(new(dtypes.BuiltinActorsLoaded), modules.LoadBuiltinActorsTesting),
 	)
 }
 

node/modules/builtin_actors.go

@@ -1,6 +1,10 @@
 package modules
 
 import (
+	"fmt"
+	"io"
+	"os"
+
 	"go.uber.org/fx"
 	"golang.org/x/xerrors"
 
@@ -39,3 +43,38 @@ func LoadBultinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtypes.Univer
 
 	return result, nil
 }
+
+// for itests
+func LoadBuiltinActorsTesting(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtypes.UniversalBlockstore) (result dtypes.BuiltinActorsLoaded, err error) {
+	ctx := helpers.LifecycleCtx(mctx, lc)
+
+	base := os.Getenv("LOTUS_SRC_DIR")
+	if base == "" {
+		base = "."
+	}
+
+	for _, ver := range []actors.Version{actors.Version8} {
+		path := fmt.Sprintf("%s/build/builtin-actors/v%d/builtin-actors-testing.car", base, ver)
+
+		file, err := os.Open(path)
+		if err != nil {
+			return result, xerrors.Errorf("error opening v%d bundle: %w", ver, err)
+		}
+
+		bundle, err := io.ReadAll(file)
+		if err != nil {
+			return result, xerrors.Errorf("error reading v%d bundle: %w", ver, err)
+		}
+
+		if err := actors.LoadBundle(ctx, bs, actors.Version8, bundle); err != nil {
+			return result, xerrors.Errorf("error loading v%d bundle: %w", ver, err)
+		}
+	}
+
+	cborStore := cbor.NewCborStore(bs)
+	if err := actors.LoadManifests(ctx, cborStore); err != nil {
+		return result, xerrors.Errorf("error loading actor manifests: %w", err)
+	}
+
+	return result, nil
+}

node/modules/testing/genesis.go

@@ -30,8 +30,8 @@ import (
 
 var glog = logging.Logger("genesis")
 
-func MakeGenesisMem(out io.Writer, template genesis.Template) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis {
-	return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis {
+func MakeGenesisMem(out io.Writer, template genesis.Template) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal, _ dtypes.BuiltinActorsLoaded) modules.Genesis {
+	return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal, _ dtypes.BuiltinActorsLoaded) modules.Genesis {
 		return func() (*types.BlockHeader, error) {
 			glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network")
 			b, err := genesis2.MakeGenesisBlock(context.TODO(), j, bs, syscalls, template)
@@ -51,8 +51,8 @@ func MakeGenesisMem(out io.Writer, template genesis.Template) func(bs dtypes.Cha
 	}
 }
 
-func MakeGenesis(outFile, genesisTemplate string) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis {
-	return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis {
+func MakeGenesis(outFile, genesisTemplate string) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal, _ dtypes.BuiltinActorsLoaded) modules.Genesis {
+	return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal, _ dtypes.BuiltinActorsLoaded) modules.Genesis {
 		return func() (*types.BlockHeader, error) {
 			glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network")
 			genesisTemplate, err := homedir.Expand(genesisTemplate)