diff --git a/.circleci/config.yml b/.circleci/config.yml index ca94d2a77..3fd7b24a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -86,22 +86,10 @@ commands: git fetch --all fetch_builtin_actors: steps: - - restore_cache: - name: restore builtin actors bundle cache - keys: - - 'builtin-actors-v8.car' - paths: - - build/builtin-actors/ - run: name: fetch builtin actor bundles command: | build/builtin-actors/fetch-bundles.sh - - save_cache: - name: save builtin actors bundle cache - key: 'builtin-actors-v8.car' - paths: - - build/builtin-actors/ - packer_build: description: "Run a packer build" parameters: @@ -214,6 +202,7 @@ jobs: environment: TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >> SKIP_CONFORMANCE: "1" + LOTUS_SRC_DIR: /home/circleci/project command: | mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts @@ -1125,4 +1114,4 @@ workflows: only: - master jobs: - - publish-packer-snap \ No newline at end of file + - publish-packer-snap diff --git a/.circleci/template.yml b/.circleci/template.yml index 0543b3676..2e4831dd7 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -86,22 +86,10 @@ commands: git fetch --all fetch_builtin_actors: steps: - - restore_cache: - name: restore builtin actors bundle cache - keys: - - 'builtin-actors-v8.car' - paths: - - build/builtin-actors/ - run: name: fetch builtin actor bundles command: | build/builtin-actors/fetch-bundles.sh - - save_cache: - name: save builtin actors bundle cache - key: 'builtin-actors-v8.car' - paths: - - build/builtin-actors/ - packer_build: description: "Run a packer build" parameters: @@ -214,6 +202,7 @@ jobs: environment: TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >> SKIP_CONFORMANCE: "1" + LOTUS_SRC_DIR: /home/circleci/project command: | mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts @@ -930,4 +919,4 @@ workflows: only: - master jobs: - - publish-packer-snap \ No newline at end of file + - publish-packer-snap diff --git a/.gitignore b/.gitignore index b30b96310..00107fdcc 100644 --- a/.gitignore +++ b/.gitignore @@ -49,4 +49,4 @@ bin/tmp/* .idea scratchpad -build/builtin-actors/*.car +build/builtin-actors/v* diff --git a/build/builtin-actors/bundles.env b/build/builtin-actors/bundles.env new file mode 100644 index 000000000..78405d953 --- /dev/null +++ b/build/builtin-actors/bundles.env @@ -0,0 +1,2 @@ +actors7_release="" +actors8_release=a9635268e3b359bd diff --git a/build/builtin-actors/fetch-bundles.sh b/build/builtin-actors/fetch-bundles.sh index dd30d6a1f..6345480a5 100755 --- a/build/builtin-actors/fetch-bundles.sh +++ b/build/builtin-actors/fetch-bundles.sh @@ -3,50 +3,83 @@ set -e cd "$(dirname "$0")" -# gateway to use -dweb="dweb.link" - -actors7_cid="" -actors7_hash="" -actors8_cid="bafybeictmywrut5tprz5fnoti6adfwuvixvrfardhqwldxosmdsfavc56e" -actors8_hash="687b38f59b0c32800f55a8f1f303de214ec173c90e653984d67f393bc41c1416" +. bundles.env die() { echo "$1" exit 1 } -check() { - file=$1 - hash=$2 - if [ -e "$file" ]; then - echo "$hash $file" | shasum -a 256 --check - else - return 1 - fi -} - fetch() { - output=$1 - cid=$2 - hash=$3 - if (check "$output" "$hash"); then - return 0 - else - echo "fetching $cid to $output" - curl --retry 3 -k "https://$dweb/ipfs/$cid" -o "$output" - check "$output" "$hash" || die "hash mismatch" + ver=$1 + rel=$2 + + if [ ! -e $ver ]; then + mkdir $ver fi + + if [ -e $ver/release ]; then + cur=$(cat $ver/release) + if [ $cur == $rel ]; then + return 0 + fi + fi + + for net in mainnet caterpillarnet butterflynet calibrationnet devnet testing; do + fetch_bundle $ver $rel $net + done + + # remember the current release so that we don't have to hit github unless we have modified it + echo $rel > $ver/release } -if [ -n "$actors7_cid" ]; then - fetch builtin-actors-v7.car "$actors7_cid" "$actors7_hash" +fetch_bundle() { + ver=$1 + rel=$2 + net=$3 + + target=builtin-actors-$net.car + hash=builtin-actors-$net.sha256 + + pushd $ver + + # fetch the hash first and check if it matches what we (may) already have + curl -L --retry 3 https://github.com/filecoin-project/builtin-actors/releases/download/$rel/$hash -o $hash || die "error fetching hash for $ver/$net" + if [ -e $target ]; then + if (shasum -a 256 --check $hash); then + popd + return 0 + fi + fi + + # we don't have the (correct) bundle, fetch it + curl -L --retry 3 https://github.com/filecoin-project/builtin-actors/releases/download/$rel/$target -o $target || die "error fetching bundle for $ver/$net" + # verify + shasum -a 256 --check $hash || die "hash mismatch" + # all good + popd +} + +touch_bundles() { + ver=$1 + + if [ ! -e $ver ]; then + mkdir $ver + fi + + for net in mainnet caterpillarnet butterflynet calibrationnet devnet testing; do + touch $ver/builtin-actors-$net.car + done +} + +if [ -n "$actors7_release" ]; then + fetch v7 "$actors7_release" else - touch builtin-actors-v7.car + touch_bundles v7 fi -if [ -n "$actors8_cid" ]; then - fetch builtin-actors-v8.car "$actors8_cid" "$actors8_hash" +if [ -n "$actors8_release" ]; then + fetch v8 "$actors8_release" else - touch builtin-actors-v8.car + touch_bundles v8 fi diff --git a/build/builtin_actors_2k.go b/build/builtin_actors_2k.go new file mode 100644 index 000000000..1dfee49dc --- /dev/null +++ b/build/builtin_actors_2k.go @@ -0,0 +1,22 @@ +//go:build debug || 2k || testground +// +build debug 2k testground + +package build + +import ( + _ "embed" +) + +//go:embed builtin-actors/v8/builtin-actors-devnet.car +var actorsv8 []byte + +func BuiltinActorsV8Bundle() []byte { + return actorsv8 +} + +//go:embed builtin-actors/v7/builtin-actors-devnet.car +var actorsv7 []byte + +func BuiltinActorsV7Bundle() []byte { + return actorsv7 +} diff --git a/build/builtin_actors_butterfly.go b/build/builtin_actors_butterfly.go new file mode 100644 index 000000000..d3d76e8f9 --- /dev/null +++ b/build/builtin_actors_butterfly.go @@ -0,0 +1,22 @@ +//go:build butterflynet +// +build butterflynet + +package build + +import ( + _ "embed" +) + +//go:embed builtin-actors/v8/builtin-actors-butterflynet.car +var actorsv8 []byte + +func BuiltinActorsV8Bundle() []byte { + return actorsv8 +} + +//go:embed builtin-actors/v7/builtin-actors-butterflynet.car +var actorsv7 []byte + +func BuiltinActorsV7Bundle() []byte { + return actorsv7 +} diff --git a/build/builtin_actors.go b/build/builtin_actors_calibnet.go similarity index 54% rename from build/builtin_actors.go rename to build/builtin_actors_calibnet.go index 56bc5169a..d73760a77 100644 --- a/build/builtin_actors.go +++ b/build/builtin_actors_calibnet.go @@ -1,17 +1,20 @@ +//go:build calibnet +// +build calibnet + package build import ( _ "embed" ) -//go:embed builtin-actors/builtin-actors-v8.car +//go:embed builtin-actors/v8/builtin-actors-calibrationnet.car var actorsv8 []byte func BuiltinActorsV8Bundle() []byte { return actorsv8 } -//go:embed builtin-actors/builtin-actors-v7.car +//go:embed builtin-actors/v7/builtin-actors-calibrationnet.car var actorsv7 []byte func BuiltinActorsV7Bundle() []byte { diff --git a/build/builtin_actors_interop.go b/build/builtin_actors_interop.go new file mode 100644 index 000000000..8203e4b3d --- /dev/null +++ b/build/builtin_actors_interop.go @@ -0,0 +1,22 @@ +//go:build interopnet +// +build interopnet + +package build + +import ( + _ "embed" +) + +//go:embed builtin-actors/v8/builtin-actors-caterpillarnet.car +var actorsv8 []byte + +func BuiltinActorsV8Bundle() []byte { + return actorsv8 +} + +//go:embed builtin-actors/v7/builtin-actors-caterpillarnet.car +var actorsv7 []byte + +func BuiltinActorsV7Bundle() []byte { + return actorsv7 +} diff --git a/build/builtin_actors_mainnet.go b/build/builtin_actors_mainnet.go new file mode 100644 index 000000000..400ac7827 --- /dev/null +++ b/build/builtin_actors_mainnet.go @@ -0,0 +1,22 @@ +//go:build !debug && !2k && !testground && !calibnet && !nerpanet && !butterflynet && !interopnet +// +build !debug,!2k,!testground,!calibnet,!nerpanet,!butterflynet,!interopnet + +package build + +import ( + _ "embed" +) + +//go:embed builtin-actors/v8/builtin-actors-mainnet.car +var actorsv8 []byte + +func BuiltinActorsV8Bundle() []byte { + return actorsv8 +} + +//go:embed builtin-actors/v7/builtin-actors-mainnet.car +var actorsv7 []byte + +func BuiltinActorsV7Bundle() []byte { + return actorsv7 +} diff --git a/cmd/lotus-seed/genesis.go b/cmd/lotus-seed/genesis.go index 89feec33a..34dd15b89 100644 --- a/cmd/lotus-seed/genesis.go +++ b/cmd/lotus-seed/genesis.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/csv" "encoding/json" "fmt" @@ -12,9 +13,11 @@ import ( "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/journal" + "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/testing" "github.com/google/uuid" "github.com/mitchellh/go-homedir" @@ -577,7 +580,12 @@ var genesisCarCmd = &cli.Command{ jrnl := journal.NilJournal() bstor := blockstore.WrapIDStore(blockstore.NewMemorySync()) sbldr := vm.Syscalls(ffiwrapper.ProofVerifier) - _, err := testing.MakeGenesis(ofile, c.Args().First())(bstor, sbldr, jrnl)() + if len(build.BuiltinActorsV8Bundle()) > 0 { + if err := actors.LoadManifestFromBundle(context.TODO(), bstor, actors.Version8, build.BuiltinActorsV8Bundle()); err != nil { + return xerrors.Errorf("error loading actor manifest: %w", err) + } + } + _, err := testing.MakeGenesis(ofile, c.Args().First())(bstor, sbldr, jrnl, dtypes.BuiltinActorsLoaded{})() return err }, } diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 0c6d69861..4ff4fec93 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 0c6d698610a98c9efcefc58d5277ee8813d80033 +Subproject commit 4ff4fec93f1b40e1351af978b8df5de1156f389d diff --git a/node/builder.go b/node/builder.go index f0106ad97..6f9805760 100644 --- a/node/builder.go +++ b/node/builder.go @@ -384,6 +384,9 @@ func Test() Option { Unset(new(*peermgr.PeerMgr)), Override(new(beacon.Schedule), testing.RandomBeacon), Override(new(*storageadapter.DealPublisher), storageadapter.NewDealPublisher(nil, storageadapter.PublishMsgConfig{})), + // use the testing bundles + Unset(new(dtypes.BuiltinActorsLoaded)), + Override(new(dtypes.BuiltinActorsLoaded), modules.LoadBuiltinActorsTesting), ) } diff --git a/node/modules/builtin_actors.go b/node/modules/builtin_actors.go index a669123f2..c14b60ce8 100644 --- a/node/modules/builtin_actors.go +++ b/node/modules/builtin_actors.go @@ -1,6 +1,10 @@ package modules import ( + "fmt" + "io" + "os" + "go.uber.org/fx" "golang.org/x/xerrors" @@ -39,3 +43,38 @@ func LoadBultinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtypes.Univer return result, nil } + +// for itests +func LoadBuiltinActorsTesting(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtypes.UniversalBlockstore) (result dtypes.BuiltinActorsLoaded, err error) { + ctx := helpers.LifecycleCtx(mctx, lc) + + base := os.Getenv("LOTUS_SRC_DIR") + if base == "" { + base = "." + } + + for _, ver := range []actors.Version{actors.Version8} { + path := fmt.Sprintf("%s/build/builtin-actors/v%d/builtin-actors-testing.car", base, ver) + + file, err := os.Open(path) + if err != nil { + return result, xerrors.Errorf("error opening v%d bundle: %w", ver, err) + } + + bundle, err := io.ReadAll(file) + if err != nil { + return result, xerrors.Errorf("error reading v%d bundle: %w", ver, err) + } + + if err := actors.LoadBundle(ctx, bs, actors.Version8, bundle); err != nil { + return result, xerrors.Errorf("error loading v%d bundle: %w", ver, err) + } + } + + cborStore := cbor.NewCborStore(bs) + if err := actors.LoadManifests(ctx, cborStore); err != nil { + return result, xerrors.Errorf("error loading actor manifests: %w", err) + } + + return result, nil +} diff --git a/node/modules/testing/genesis.go b/node/modules/testing/genesis.go index a3d25e36a..68d792d83 100644 --- a/node/modules/testing/genesis.go +++ b/node/modules/testing/genesis.go @@ -30,8 +30,8 @@ import ( var glog = logging.Logger("genesis") -func MakeGenesisMem(out io.Writer, template genesis.Template) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { - return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { +func MakeGenesisMem(out io.Writer, template genesis.Template) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal, _ dtypes.BuiltinActorsLoaded) modules.Genesis { + return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal, _ dtypes.BuiltinActorsLoaded) modules.Genesis { return func() (*types.BlockHeader, error) { glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network") b, err := genesis2.MakeGenesisBlock(context.TODO(), j, bs, syscalls, template) @@ -51,8 +51,8 @@ func MakeGenesisMem(out io.Writer, template genesis.Template) func(bs dtypes.Cha } } -func MakeGenesis(outFile, genesisTemplate string) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { - return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { +func MakeGenesis(outFile, genesisTemplate string) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal, _ dtypes.BuiltinActorsLoaded) modules.Genesis { + return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal, _ dtypes.BuiltinActorsLoaded) modules.Genesis { return func() (*types.BlockHeader, error) { glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network") genesisTemplate, err := homedir.Expand(genesisTemplate)