diff --git a/Makefile b/Makefile index abeb5d492..5aab3c170 100644 --- a/Makefile +++ b/Makefile @@ -295,6 +295,12 @@ actors-gen: $(GOCC) run ./chain/actors/agen $(GOCC) fmt ./... +bundle-gen: + $(GOCC) run ./gen/bundle + $(GOCC) fmt ./build/... +.PHONY: bundle-gen + + api-gen: $(GOCC) run ./gen/api goimports -w api @@ -341,7 +347,7 @@ docsgen-openrpc-worker: docsgen-openrpc-bin .PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin -gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci +gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci bundle-gen @echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO MAKE docsgen-cli" .PHONY: gen diff --git a/build/README-bundle.md b/build/README-bundle.md deleted file mode 100644 index 4dc96c687..000000000 --- a/build/README-bundle.md +++ /dev/null @@ -1,59 +0,0 @@ -# Builtin Actor Bundles - -With NV16, builtin actor bundles must be loaded into lotus for the FVM to operate. - -The bundles are specified in build/bundles.toml using the following syntax: -```toml -[[bundles]] -version = X # actors version -release = tag # release tag -``` - -This will add a bundle for version `X`, using the github release `tag` -to fetch the bundles at first startup. - -If you don't want to fetch the bundle from github, you can specify an explicit path to the bundle (which must be appropriate for your network, typically mainnet): -```toml -[[bundles]] -version = X # actors version -release = tag # release tag -path = /path/to/builtin-actors.car -``` - -For development bundles, you can also specify `development = true` so that the bundle is not -recorded in the datastore and reloaded every time the daemon starts up: -```toml -[[bundles]] -version = X # actors version -release = tag # release gag -path = /path/to/builtin-actors.car -development = true -``` - -## Additional Options for Bundles - -- You can also specify a URL, together with a sha256 checksum to avoid downloading from - github. -- You can also specify an environment variable (`LOTUS_BUILTIN_ACTORS_VX_BUNDLE`), to provide the path dynamically at runtime. - -The precedence for bundle fetching/loading is as folllows: -- Check the environment variable `LOTUS_BUILTIN_ACTORS_VX_BUNDLE` for version X bundle; use it if set. -- Check the Path; use the bundle specified by it. -- Check the URL; use the bundle specified by it, and verify the checksum which must be present. -- Otherwise, use the release tag and download from github. - -## Local Storage - -Bundles downloaded from github will be stored in -`$LOTUS_PATH/builtin-actors/vXXX/YYY/builtin-actors-ZZZ.car``, where -`XXX` is the actors version, `YYY` is the release tag, and `ZZZ` is -the network bundle name. - -The sha256 sum of the bundle will be stored next to it, in -`$LOTUS_PATH/builtin-actors/vXXX/YYY/builtin-actors-ZZZ.sha256` - -On startup, if a bundle is recorded as loaded the manifest CID will be -checked for presence in the blockstore. If the manifest is missing, -then the bundle will be reloaded from the local file (if it exists) or -refetched from github. The sha256 sum is always checked before -loading the bundle. diff --git a/build/actors/README.md b/build/actors/README.md new file mode 100644 index 000000000..097959f4f --- /dev/null +++ b/build/actors/README.md @@ -0,0 +1,18 @@ +# Bundles + +This directory includes the actors bundles for each release. Each actor bundle is a zstd compressed +tarfile containing one bundle per network type. These tarfiles are subsequently embedded in the +lotus binary. + +## Updating + +To update, run the `./pack.sh` script. For example, the following will pack the [builtin actors release](https://github.com/filecoin-project/builtin-actors/releases) `dev/20220602` into the `v8` tarfile. + +```bash +./pack.sh v8 dev/20220602 +``` + +This will: + +1. Download the actors bundles and pack them into the appropriate tarfile (`$VERSION.tar.zst`). +2. Run `make bundle-gen` in the top-level directory to regenerate the bundle metadata file for _all_ network versions (all `*.tar.zst` files in this directory). diff --git a/build/actors/pack.sh b/build/actors/pack.sh new file mode 100755 index 000000000..e76c06295 --- /dev/null +++ b/build/actors/pack.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -e + +if [[ $# -ne 2 ]]; then + echo "expected two arguments, an actors version (e.g., v8) and an actors release" + exit 1 +fi + +VERSION="$1" # actors version +RELEASE="$2" # actors release name +NETWORKS=(devnet mainnet caterpillarnet butterflynet testing testing-fake-proofs) + +echo "Downloading bundles for actors version ${VERSION}, release ${RELEASE}" + +TARGET_FILE="$(pwd)/${VERSION}.tar.zst" +WORKDIR=$(mktemp --tmpdir -d "actor-bundles-${VERSION}.XXXXXXXXXX") +trap 'rm -rf -- "$WORKDIR"' EXIT + +pushd "${WORKDIR}" +encoded_release="$(jq -rn --arg release "$RELEASE" '$release | @uri')" +for network in "${NETWORKS[@]}"; do + wget "https://github.com/filecoin-project/builtin-actors/releases/download/${encoded_release}/builtin-actors-${network}"{.car,.sha256} +done + +echo "Checking the checksums..." + +sha256sum -c -- *.sha256 + + +echo "Packing..." + +rm -f -- "$TARGET_FILE" +tar -cf "$TARGET_FILE" -I "zstd -19" -- *.car +popd + +echo "Generating metadata..." + +make -C ../../ bundle-gen diff --git a/build/actors/v8.tar.zst b/build/actors/v8.tar.zst new file mode 100644 index 000000000..a889d133c Binary files /dev/null and b/build/actors/v8.tar.zst differ diff --git a/build/builtin_actors.go b/build/builtin_actors.go index c6ed38c5b..39f414c89 100644 --- a/build/builtin_actors.go +++ b/build/builtin_actors.go @@ -1,27 +1,263 @@ package build import ( - "bytes" + "archive/tar" + "context" + "embed" + "fmt" + "io" + "os" + "path" + "sort" + "strconv" + "strings" + "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/adt" - "github.com/BurntSushi/toml" + "github.com/DataDog/zstd" + "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" + "github.com/ipld/go-car" + "golang.org/x/xerrors" ) -var BuiltinActorReleases map[actors.Version]Bundle +//go:embed actors/*.tar.zst +var embeddedBuiltinActorReleases embed.FS func init() { - BuiltinActorReleases = make(map[actors.Version]Bundle) - - spec := BundleSpec{} - - r := bytes.NewReader(BuiltinActorBundles) - _, err := toml.DecodeReader(r, &spec) - if err != nil { + if BundleOverrides == nil { + BundleOverrides = make(map[actors.Version]string) + } + for _, av := range actors.Versions { + path := os.Getenv(fmt.Sprintf("LOTUS_BUILTIN_ACTORS_V%d_BUNDLE", av)) + if path == "" { + continue + } + BundleOverrides[actors.Version(av)] = path + } + if err := loadManifests(NetworkBundle); err != nil { panic(err) } +} - for _, b := range spec.Bundles { - BuiltinActorReleases[b.Version] = b +// UseNetworkBundle switches to a different network bundle, by name. +func UseNetworkBundle(netw string) error { + if NetworkBundle == netw { + return nil + } + if err := loadManifests(netw); err != nil { + return err + } + NetworkBundle = netw + return nil +} + +func loadManifests(netw string) error { + overridden := make(map[actors.Version]struct{}) + var newMetadata []*BuiltinActorsMetadata + // First, prefer overrides. + for av, path := range BundleOverrides { + root, actorCids, err := readBundleManifestFromFile(path) + if err != nil { + return err + } + newMetadata = append(newMetadata, &BuiltinActorsMetadata{ + Network: netw, + Version: av, + ManifestCid: root, + Actors: actorCids, + }) + overridden[av] = struct{}{} + } + + // Then load embedded bundle metadata. + for _, meta := range EmbeddedBuiltinActorsMetadata { + if meta.Network != netw { + continue + } + if _, ok := overridden[meta.Version]; ok { + continue + } + newMetadata = append(newMetadata, meta) + } + + actors.ClearManifests() + + for _, meta := range newMetadata { + actors.RegisterManifest(meta.Version, meta.ManifestCid, meta.Actors) + } + + return nil +} + +type BuiltinActorsMetadata struct { + Network string + Version actors.Version + ManifestCid cid.Cid + Actors map[string]cid.Cid +} + +// ReadEmbeddedBuiltinActorsMetadata reads the metadata from the embedded built-in actor bundles. +// There should be no need to call this method as the result is cached in the +// `EmbeddedBuiltinActorsMetadata` variable on `make gen`. +func ReadEmbeddedBuiltinActorsMetadata() ([]*BuiltinActorsMetadata, error) { + files, err := embeddedBuiltinActorReleases.ReadDir("actors") + if err != nil { + return nil, xerrors.Errorf("failed to read embedded bundle directory: %s", err) + } + var bundles []*BuiltinActorsMetadata + for _, dirent := range files { + name := dirent.Name() + b, err := readEmbeddedBuiltinActorsMetadata(name) + if err != nil { + return nil, err + } + bundles = append(bundles, b...) + } + // Sort by network, then by bundle. + sort.Slice(bundles, func(i, j int) bool { + if bundles[i].Network == bundles[j].Network { + return bundles[i].Version < bundles[j].Version + } + return bundles[i].Network < bundles[j].Network + }) + return bundles, nil +} + +func readEmbeddedBuiltinActorsMetadata(bundle string) ([]*BuiltinActorsMetadata, error) { + const ( + archiveExt = ".tar.zst" + bundleExt = ".car" + bundlePrefix = "builtin-actors-" + ) + + if !strings.HasPrefix(bundle, "v") { + return nil, xerrors.Errorf("bundle bundle '%q' doesn't start with a 'v'", bundle) + } + if !strings.HasSuffix(bundle, archiveExt) { + return nil, xerrors.Errorf("bundle bundle '%q' doesn't end with '%s'", bundle, archiveExt) + } + version, err := strconv.ParseInt(bundle[1:len(bundle)-len(archiveExt)], 10, 0) + if err != nil { + return nil, xerrors.Errorf("failed to parse actors version from bundle '%q': %s", bundle, err) + } + fi, err := embeddedBuiltinActorReleases.Open(fmt.Sprintf("actors/%s", bundle)) + if err != nil { + return nil, err + } + defer fi.Close() //nolint + + uncompressed := zstd.NewReader(fi) + defer uncompressed.Close() //nolint + + var bundles []*BuiltinActorsMetadata + + tarReader := tar.NewReader(uncompressed) + for { + header, err := tarReader.Next() + switch err { + case io.EOF: + return bundles, nil + case nil: + default: + return nil, err + } + + // Read the network name from the bundle name. + name := path.Base(header.Name) + if !strings.HasSuffix(name, bundleExt) { + return nil, xerrors.Errorf("expected bundle to end with .car: %s", name) + } + if !strings.HasPrefix(name, bundlePrefix) { + return nil, xerrors.Errorf("expected bundle to end with .car: %s", name) + } + name = name[len(bundlePrefix) : len(name)-len(bundleExt)] + + // Load the bundle. + root, actorCids, err := readBundleManifest(tarReader) + if err != nil { + return nil, xerrors.Errorf("error loading builtin actors bundle: %w", err) + } + bundles = append(bundles, &BuiltinActorsMetadata{ + Network: name, + Version: actors.Version(version), + ManifestCid: root, + Actors: actorCids, + }) + } +} + +func readBundleManifestFromFile(path string) (cid.Cid, map[string]cid.Cid, error) { + fi, err := os.Open(path) + if err != nil { + return cid.Undef, nil, err + } + defer fi.Close() //nolint + + return readBundleManifest(fi) +} + +func readBundleManifest(r io.Reader) (cid.Cid, map[string]cid.Cid, error) { + // Load the bundle. + bs := blockstore.NewMemory() + hdr, err := car.LoadCar(context.Background(), bs, r) + if err != nil { + return cid.Undef, nil, xerrors.Errorf("error loading builtin actors bundle: %w", err) + } + + if len(hdr.Roots) != 1 { + return cid.Undef, nil, xerrors.Errorf("expected one root when loading actors bundle, got %d", len(hdr.Roots)) + } + root := hdr.Roots[0] + actorCids, err := actors.ReadManifest(context.Background(), adt.WrapStore(context.Background(), cbor.NewCborStore(bs)), root) + if err != nil { + return cid.Undef, nil, err + } + + // Make sure we have all the actors. + for name, c := range actorCids { + if has, err := bs.Has(context.Background(), c); err != nil { + return cid.Undef, nil, xerrors.Errorf("got an error when checking that the bundle has the actor %q: %w", name, err) + } else if !has { + return cid.Undef, nil, xerrors.Errorf("actor %q missing from bundle", name) + } + } + + return root, actorCids, nil +} + +// GetEmbeddedBuiltinActorsBundle returns the builtin-actors bundle for the given actors version. +func GetEmbeddedBuiltinActorsBundle(version actors.Version) ([]byte, bool) { + fi, err := embeddedBuiltinActorReleases.Open(fmt.Sprintf("actors/v%d.tar.zst", version)) + if err != nil { + return nil, false + } + defer fi.Close() //nolint + + uncompressed := zstd.NewReader(fi) + defer uncompressed.Close() //nolint + + tarReader := tar.NewReader(uncompressed) + targetFileName := fmt.Sprintf("builtin-actors-%s.car", NetworkBundle) + for { + header, err := tarReader.Next() + switch err { + case io.EOF: + return nil, false + case nil: + default: + panic(err) + } + if header.Name != targetFileName { + continue + } + + car, err := io.ReadAll(tarReader) + if err != nil { + panic(err) + } + return car, true } } diff --git a/build/builtin_actors_gen.go b/build/builtin_actors_gen.go new file mode 100644 index 000000000..adf969acb --- /dev/null +++ b/build/builtin_actors_gen.go @@ -0,0 +1,111 @@ +// WARNING: This file has automatically been generated + +package build + +import ( + "github.com/ipfs/go-cid" +) + +var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMetadata{{ + Network: "butterflynet", + Version: 8, + ManifestCid: MustParseCid("bafy2bzacedktlamzlgecxpnsnods5iyh5uspbg2ukqrrn6h6c2h5iozmtb2d6"), + Actors: map[string]cid.Cid{ + "account": MustParseCid("bafk2bzacebbdtqcmjtyqzheoycfn3kitaqivhywxjcvrl5dyor4qh4rh23zfu"), + "cron": MustParseCid("bafk2bzacedy7j3r4wfjxm4vhlc7xdxkuuihbmcvq6kldtm65ioll2du6jwgnq"), + "init": MustParseCid("bafk2bzaceauk2zhkie7btkmjmcuv2z5f53lv3adqlq4ozp7dln64lkath6iia"), + "multisig": MustParseCid("bafk2bzacedaxdz26xg44m5chunjjpjayzjhyfofnzaotmbuay7p4sso6n4vz4"), + "paymentchannel": MustParseCid("bafk2bzacebklebmukj53bkeu2zaeawv2aolzd7qepchpybimfewii43ti3np6"), + "reward": MustParseCid("bafk2bzaceanwnmfpzgyrp6upiutqaumkugzjncrq3grettmq7wgg3rejfcmoi"), + "storagemarket": MustParseCid("bafk2bzacecqpkkhltbzju4oe4kszp7pgffopifbcy5i4xuxsexsoyp7psvgyu"), + "storageminer": MustParseCid("bafk2bzacebewpe2ufauqsey7p7wuxihi6eqmszf4nyjwcq2rcw2ld53k7muke"), + "storagepower": MustParseCid("bafk2bzacec5rymvcahknqtm677udm6webygsgarzfm75udpac4zqzs6n4bmpu"), + "system": MustParseCid("bafk2bzacec5mttzs3bddj6c2fbfxyzkeqgy5fcn3cihflq3jju37uzoatarsm"), + "verifiedregistry": MustParseCid("bafk2bzacecbhego5vgecly7rw2igkjggzplpv4vnclpyti4x42gaejokm6vmc"), + }, +}, { + Network: "caterpillarnet", + Version: 8, + ManifestCid: MustParseCid("bafy2bzacecooicf3mbtuqjmdfdtbw35ivp2xyjktlxwfebwvulaahamrqw3k4"), + Actors: map[string]cid.Cid{ + "account": MustParseCid("bafk2bzacebbr5xcsjhc4heykayi4nqyit2bnl4xv6degmhrlbcrkuah7xcsam"), + "cron": MustParseCid("bafk2bzacedrsf2aplg4wgjzwzzyxyhdsxlpp2ydxgnvfvq5a4ym6xvcdhh222"), + "init": MustParseCid("bafk2bzacedpwzwvfyypfo2b2pg2k2vrru6x2haa6ozlb2lf22pnacw7r76c5c"), + "multisig": MustParseCid("bafk2bzacea4iz3rmpft4v3uy2dxymoez6hfnlql3v5x4g3pk3rmgfjis34pzi"), + "paymentchannel": MustParseCid("bafk2bzacedx4behmy7x2a4xn5oh6pgl7vubc3by5zaakio3bbkmop3ju4ncbm"), + "reward": MustParseCid("bafk2bzaceazq62qx7bjot3jfmrawg5mgmjsrblzp3x6yug7uf22glhwjou7uq"), + "storagemarket": MustParseCid("bafk2bzaceaevfwztizbgtg6ybshcfaenyoverce22hhhwuv6y5gogf52tqx4g"), + "storageminer": MustParseCid("bafk2bzacebjyrh6vs5vjrh5wxrrgtdbfzaib6vc2ubmr3pgv66mbebhkd3m4k"), + "storagepower": MustParseCid("bafk2bzacedblxcejkcuuxbtmyiq5klkq5zcokor2fm2aeaq7wfou4eln637yc"), + "system": MustParseCid("bafk2bzaceamrktidazdlpsjm7quo7lynurnknqtfhd3cam3p3b2ec2ua5zlk4"), + "verifiedregistry": MustParseCid("bafk2bzacecyadbss4wve276yr4yc2adhow56apq3rglmb2fusgorpxpkmavz6"), + }, +}, { + Network: "devnet", + Version: 8, + ManifestCid: MustParseCid("bafy2bzaceb6v3llydxlcaofdhubn24gkwuooymar56iun5ceidvvm5zm2q42a"), + Actors: map[string]cid.Cid{ + "account": MustParseCid("bafk2bzacedeoab5nfo3obkkedm6sbjefdxhdfh7vpdxoqx6vba25ur2irnkpy"), + "cron": MustParseCid("bafk2bzacedbz5ox2ua6b2tee2armmwmq32ge6xnyu2n2vbbmxo5r5p73aiwfw"), + "init": MustParseCid("bafk2bzaceaf7zqjo5hjyozvzudllvoub6ejcqs7vpquie7wdp6fkt27swqpqu"), + "multisig": MustParseCid("bafk2bzacebkm2rputkjqhqy3djtut7xoq6nvbnqken3oaelmv4oxljxzhvuzg"), + "paymentchannel": MustParseCid("bafk2bzaceawo3dveeocpgkstqirdjob7xlrmy4qy2lcgq5gpnyxvest2g6joa"), + "reward": MustParseCid("bafk2bzaceainy5y3xu6ryjh3hfh7p2ab5n7uh26mprzwnajgsmu7eajxlaxe6"), + "storagemarket": MustParseCid("bafk2bzacecg45uhzcfrgi6iblseykqk5e7zi4snzy55hceqbanmq4p4zj3y4o"), + "storageminer": MustParseCid("bafk2bzacebqjo3ziocbaikgvysc2zdyxe3iqn3vsi5v7lipgcvswxbss5zcrw"), + "storagepower": MustParseCid("bafk2bzaceax24kwja27fheihx4hh6wwgiouffz4np2u3ghbpm372oait6zdh2"), + "system": MustParseCid("bafk2bzacecfiu6omvipao6qvngnoisjuoq3im4a2czojbkqzyru37puiy4iz6"), + "verifiedregistry": MustParseCid("bafk2bzaceacw5575umfg2kon6yzacthrvppky3leugsjc5vmixbfp3ob5evw4"), + }, +}, { + Network: "mainnet", + Version: 8, + ManifestCid: MustParseCid("bafy2bzacebkful67ipjqhpp4gwqhotjebrruowmpnhs55aljykmdwp2eee6am"), + Actors: map[string]cid.Cid{ + "account": MustParseCid("bafk2bzaceab5hj4qbccnlbcq2dlzurghekipwl6hvm7wmfoxj2nhnmqhjqete"), + "cron": MustParseCid("bafk2bzacecs34u2mrisxcqanpu3lgv2pyxvwosopn6cdcxg7pcxefr47mkzda"), + "init": MustParseCid("bafk2bzacec24c457opefiv7zo4wqi3f5w6k4ptfbkyhtmbwilzkindmfby5g2"), + "multisig": MustParseCid("bafk2bzacec7jef7mpiiv7wjnp6ncgkwufpbmqz5z6c2vfd2no4tqgf77qgifi"), + "paymentchannel": MustParseCid("bafk2bzacedqx4wnzlhvalcrxjlxwer2n54woz7ggearqxuk7lx3gasg6dx52s"), + "reward": MustParseCid("bafk2bzacedu4fgsp72huqi7tbsyguoia7g3oyuwh74omkx44a65pwrm3rjth2"), + "storagemarket": MustParseCid("bafk2bzacedylxxikx3rzl3dfswlisafr42qmn2mid3acajjzadwyheofi4ngw"), + "storageminer": MustParseCid("bafk2bzacecpxjo5xfofk363mjem6tdnmuhobqhlhitnf4etevuxzbeoincl3k"), + "storagepower": MustParseCid("bafk2bzacecm37s2363hn2hbscuqf2ensbqq3nsfnwbplstpggbpijdblins2m"), + "system": MustParseCid("bafk2bzacedhasoz5w7hthbxbeifmg7mrlgpwhmqcif2d63uopna4xzk37uafy"), + "verifiedregistry": MustParseCid("bafk2bzacecvwi3ndugiyfi2hlpo6kixhlvhsnuy64m33gfrr5bhxfjorpzef6"), + }, +}, { + Network: "testing", + Version: 8, + ManifestCid: MustParseCid("bafy2bzacedajylykoksplon3gzx27wli5bs72lzd5nih5c375ytqa4rjxnesa"), + Actors: map[string]cid.Cid{ + "account": MustParseCid("bafk2bzacea2lrlc3dpgfay3v3h4hfhm36il3eeegdbstvwuawf2dm6z4hkzsy"), + "cron": MustParseCid("bafk2bzaceao6lmovdtxyiyhtlvx6hqsm2oal2vrlnabgtmozqsl2pet2gpef6"), + "init": MustParseCid("bafk2bzacebbhy4izopnuj5moauauvlxw4oeozvylnejxollmwdfcmbnnkj77s"), + "multisig": MustParseCid("bafk2bzacecblegf5dogxfb7edzsdttollesh2qup7bxrqjem5sdssiuoiqxja"), + "paymentchannel": MustParseCid("bafk2bzaceaxvwc4qiwss3juhrdsy4pb4bitl7ahllxnup4k55q37eori6ttho"), + "reward": MustParseCid("bafk2bzacedd7lgezplkws6zpokyyjcmwzwof7ruxr3frpguhxkglqg52i3lf4"), + "storagemarket": MustParseCid("bafk2bzaceauec7dpcnjrgsmy2rektiraq7lfvr4t4azyukjuluxmufglca7wc"), + "storageminer": MustParseCid("bafk2bzacebrhhpno62lv6g7i5ccuvj57kdozbubqvq6mtjbrt2slihxddl3s4"), + "storagepower": MustParseCid("bafk2bzacecoh3c7fm7wdvhij46fzetwwkxks4ftu5jgovhitoxts2xild3v3u"), + "system": MustParseCid("bafk2bzacea4zg4zf77wne3rewpb5m4adl5bc2dogiph7j3q4rhrbpknwvq7qc"), + "verifiedregistry": MustParseCid("bafk2bzaceakcp7ur3wsnxi4cfh7huybjg5otlazmztrhzikxwxvrjv4uljcoq"), + }, +}, { + Network: "testing-fake-proofs", + Version: 8, + ManifestCid: MustParseCid("bafy2bzaced3kmznycbhfjkcdzymlywevk5jamtujr6eadgun6vrebmi5np22m"), + Actors: map[string]cid.Cid{ + "account": MustParseCid("bafk2bzacea2lrlc3dpgfay3v3h4hfhm36il3eeegdbstvwuawf2dm6z4hkzsy"), + "cron": MustParseCid("bafk2bzaceao6lmovdtxyiyhtlvx6hqsm2oal2vrlnabgtmozqsl2pet2gpef6"), + "init": MustParseCid("bafk2bzaceaqllna5m7kwvsahn6ijnhv4fkkyjthxrb2bwdacufvpbi3a6qygo"), + "multisig": MustParseCid("bafk2bzaceduvgbzhawh5gi23lsd7be2sefnldqhbe4clsh4bqvafyo4sk5g3y"), + "paymentchannel": MustParseCid("bafk2bzacedaxkwccobed76fsmyevttptfw74pumb637himbwnce4jtaxtxazs"), + "reward": MustParseCid("bafk2bzacedd7lgezplkws6zpokyyjcmwzwof7ruxr3frpguhxkglqg52i3lf4"), + "storagemarket": MustParseCid("bafk2bzacea3zogutvlgo7vea3widjsjz6z23mgftvlhcrnuwuptuh2cgv75ww"), + "storageminer": MustParseCid("bafk2bzacec2bz7ts7lzvty2qzovj3r3hutwgs3j7yo3ep7v67gsytft7spexa"), + "storagepower": MustParseCid("bafk2bzaceaiip5k5yendslu7ldy7rnmtabzlez6sbgdcvqfjivxr2gvxb7cy4"), + "system": MustParseCid("bafk2bzacea4zg4zf77wne3rewpb5m4adl5bc2dogiph7j3q4rhrbpknwvq7qc"), + "verifiedregistry": MustParseCid("bafk2bzaceakcp7ur3wsnxi4cfh7huybjg5otlazmztrhzikxwxvrjv4uljcoq"), + }, +}} diff --git a/build/builtin_actors_test.go b/build/builtin_actors_test.go new file mode 100644 index 000000000..389de5cbf --- /dev/null +++ b/build/builtin_actors_test.go @@ -0,0 +1,33 @@ +package build_test + +import ( + "testing" + + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/stretchr/testify/require" +) + +// Test that the embedded metadata is correct. +func TestEmbeddedMetadata(t *testing.T) { + metadata, err := build.ReadEmbeddedBuiltinActorsMetadata() + require.NoError(t, err) + + require.Equal(t, metadata, build.EmbeddedBuiltinActorsMetadata) +} + +// Test that we're registering the manifest correctly. +func TestRegistration(t *testing.T) { + manifestCid, found := actors.GetManifest(actors.Version8) + require.True(t, found) + require.True(t, manifestCid.Defined()) + + for _, key := range actors.GetBuiltinActorsKeys() { + actorCid, found := actors.GetActorCodeID(actors.Version8, key) + require.True(t, found) + name, version, found := actors.GetActorMetaByCode(actorCid) + require.True(t, found) + require.Equal(t, actors.Version8, version) + require.Equal(t, key, name) + } +} diff --git a/build/bundle.go b/build/bundle.go deleted file mode 100644 index 56a9f5bfa..000000000 --- a/build/bundle.go +++ /dev/null @@ -1,36 +0,0 @@ -package build - -import ( - _ "embed" - - "github.com/filecoin-project/lotus/chain/actors" -) - -//go:embed bundles.toml -var BuiltinActorBundles []byte - -type BundleSpec struct { - Bundles []Bundle -} - -type Bundle struct { - // Version is the actors version in this bundle - Version actors.Version - // Release is the release id - Release string - // Path is the (optional) bundle path; takes precedence over url - Path map[string]string - // URL is the (optional) bundle URL; takes precedence over github release - URL map[string]BundleURL - // Devlopment indicates whether this is a development version; when set, in conjunction with path, - // it will always load the bundle to the blockstore, without recording the manifest CID in the - // datastore. - Development bool -} - -type BundleURL struct { - // URL is the url of the bundle - URL string - // Checksum is the sha256 checksum of the bundle - Checksum string -} diff --git a/build/bundles.toml b/build/bundles.toml deleted file mode 100644 index c99637d53..000000000 --- a/build/bundles.toml +++ /dev/null @@ -1,3 +0,0 @@ -[[bundles]] -version = 8 -release = "dev/20220527" \ No newline at end of file diff --git a/build/params_2k.go b/build/params_2k.go index ec903ec6c..c5417a11e 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -11,6 +11,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/policy" ) @@ -19,6 +20,7 @@ const BootstrappersFile = "" const GenesisFile = "" var NetworkBundle = "devnet" +var BundleOverrides map[actors.Version]string const GenesisNetworkVersion = network.Version16 @@ -53,8 +55,6 @@ var UpgradeChocolateHeight = abi.ChainEpoch(-17) var UpgradeOhSnapHeight = abi.ChainEpoch(-18) var UpgradeSkyrHeight = abi.ChainEpoch(-19) -var ActorsCIDs = map[actors.Version]cid.Cid{} - var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, } diff --git a/build/params_butterfly.go b/build/params_butterfly.go index a98ac676f..f99b6d7e3 100644 --- a/build/params_butterfly.go +++ b/build/params_butterfly.go @@ -20,6 +20,7 @@ var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ const GenesisNetworkVersion = network.Version14 var NetworkBundle = "butterflynet" +var BundleOverrides map[actors.Version]string const BootstrappersFile = "butterflynet.pi" const GenesisFile = "butterflynet.car" @@ -50,10 +51,6 @@ const UpgradeOhSnapHeight = 240 // 2022-05-31T14:32:00Z const UpgradeSkyrHeight = abi.ChainEpoch(333258) -var ActorsCIDs = map[actors.Version]cid.Cid{ - actors.Version8: MustParseCid("bafy2bzacedy4qgxbr6pbyfgcp7s7bdkc2whi5errnw67al5e2tk75j46iucv6"), -} - var SupportedProofTypes = []abi.RegisteredSealProof{ abi.RegisteredSealProof_StackedDrg512MiBV1, abi.RegisteredSealProof_StackedDrg32GiBV1, diff --git a/build/params_calibnet.go b/build/params_calibnet.go index 8683470f7..c940d58e4 100644 --- a/build/params_calibnet.go +++ b/build/params_calibnet.go @@ -20,6 +20,7 @@ var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ const GenesisNetworkVersion = network.Version0 var NetworkBundle = "calibrationnet" +var BundleOverrides map[actors.Version]string const BootstrappersFile = "calibnet.pi" const GenesisFile = "calibnet.car" @@ -62,8 +63,6 @@ const UpgradeOhSnapHeight = 682006 var UpgradeSkyrHeight = abi.ChainEpoch(99999999999999) -var ActorsCIDs = map[actors.Version]cid.Cid{} - var SupportedProofTypes = []abi.RegisteredSealProof{ abi.RegisteredSealProof_StackedDrg32GiBV1, abi.RegisteredSealProof_StackedDrg64GiBV1, diff --git a/build/params_interop.go b/build/params_interop.go index 38cba440c..bc5098f22 100644 --- a/build/params_interop.go +++ b/build/params_interop.go @@ -7,17 +7,17 @@ import ( "os" "strconv" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/policy" builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/ipfs/go-cid" ) var NetworkBundle = "caterpillarnet" +var BundleOverrides map[actors.Version]string const BootstrappersFile = "interopnet.pi" const GenesisFile = "interopnet.car" @@ -53,10 +53,6 @@ var UpgradeChocolateHeight = abi.ChainEpoch(-17) var UpgradeOhSnapHeight = abi.ChainEpoch(-18) var UpgradeSkyrHeight = abi.ChainEpoch(100) -var ActorsCIDs = map[actors.Version]cid.Cid{ - actors.Version8: MustParseCid("bafy2bzacebjfypksxfieex7d6rh27lu2dk2m7chnok5mtzigj3txvb7leqbys"), -} - var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, } diff --git a/build/params_mainnet.go b/build/params_mainnet.go index 5d52e2bd9..4ac898f48 100644 --- a/build/params_mainnet.go +++ b/build/params_mainnet.go @@ -7,10 +7,8 @@ import ( "math" "os" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/ipfs/go-cid" - "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" @@ -24,6 +22,9 @@ var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ var NetworkBundle = "mainnet" +// NOTE: DO NOT change this unless you REALLY know what you're doing. This is consensus critical. +var BundleOverrides map[actors.Version]string + const GenesisNetworkVersion = network.Version0 const BootstrappersFile = "mainnet.pi" @@ -77,8 +78,6 @@ const UpgradeOhSnapHeight = 1594680 var UpgradeSkyrHeight = abi.ChainEpoch(99999999999999) -var ActorsCIDs = map[actors.Version]cid.Cid{} - var SupportedProofTypes = []abi.RegisteredSealProof{ abi.RegisteredSealProof_StackedDrg32GiBV1, abi.RegisteredSealProof_StackedDrg64GiBV1, diff --git a/build/params_testground.go b/build/params_testground.go index 5e2dde3b0..3123a3bd0 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -115,6 +115,7 @@ var ( GenesisNetworkVersion = network.Version0 NetworkBundle = "devnet" + BundleOverrides map[actors.Version]string NewestNetworkVersion = network.Version15 ActorUpgradeNetworkVersion = network.Version15 @@ -128,5 +129,3 @@ var ( ) const BootstrapPeerThreshold = 1 - -var ActorsCIDs = map[actors.Version]cid.Cid{} diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index 6d707ba03..d62fae462 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -1,6 +1,8 @@ package builtin import ( + "fmt" + "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" "golang.org/x/xerrors" @@ -65,9 +67,8 @@ func QAPowerForWeight(size abi.SectorSize, duration abi.ChainEpoch, dealWeight, } func ActorNameByCode(c cid.Cid) string { - name, _, ok := actors.GetActorMetaByCode(c) - if ok { - return name + if name, version, ok := actors.GetActorMetaByCode(c); ok { + return fmt.Sprintf("fil/%d/%s", version, name) } switch { diff --git a/chain/actors/builtin/builtin.go.template b/chain/actors/builtin/builtin.go.template index bfa5d9ec2..ef4dcef39 100644 --- a/chain/actors/builtin/builtin.go.template +++ b/chain/actors/builtin/builtin.go.template @@ -1,6 +1,8 @@ package builtin import ( + "fmt" + "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" "golang.org/x/xerrors" @@ -53,9 +55,8 @@ func QAPowerForWeight(size abi.SectorSize, duration abi.ChainEpoch, dealWeight, } func ActorNameByCode(c cid.Cid) string { - name, _, ok := actors.GetActorMetaByCode(c) - if ok { - return name + if name, version, ok := actors.GetActorMetaByCode(c); ok { + return fmt.Sprintf("fil/%d/%s", version, name) } switch { diff --git a/chain/actors/manifest.go b/chain/actors/manifest.go index b78c9a83c..a00c50217 100644 --- a/chain/actors/manifest.go +++ b/chain/actors/manifest.go @@ -14,9 +14,9 @@ import ( cbor "github.com/ipfs/go-ipld-cbor" ) -var manifestCids map[Version]cid.Cid -var manifests map[Version]*manifest.Manifest -var actorMeta map[cid.Cid]actorEntry +var manifestCids map[Version]cid.Cid = make(map[Version]cid.Cid) +var manifests map[Version]map[string]cid.Cid = make(map[Version]map[string]cid.Cid) +var actorMeta map[cid.Cid]actorEntry = make(map[cid.Cid]actorEntry) const ( AccountKey = "account" @@ -57,17 +57,30 @@ type actorEntry struct { version Version } -func AddManifest(av Version, manifestCid cid.Cid) { +// ClearManifest clears all known manifests. This is usually used in tests that need to switch networks. +func ClearManifests() { manifestMx.Lock() defer manifestMx.Unlock() - if manifestCids == nil { - manifestCids = make(map[Version]cid.Cid) - } - - manifestCids[av] = manifestCid + manifestCids = make(map[Version]cid.Cid) + manifests = make(map[Version]map[string]cid.Cid) + actorMeta = make(map[cid.Cid]actorEntry) } +// RegisterManifest registers an actors manifest with lotus. +func RegisterManifest(av Version, manifestCid cid.Cid, entries map[string]cid.Cid) { + manifestMx.Lock() + defer manifestMx.Unlock() + + manifestCids[av] = manifestCid + manifests[av] = entries + + for name, c := range entries { + actorMeta[c] = actorEntry{name: name, version: av} + } +} + +// GetManifest gets a loaded manifest. func GetManifest(av Version) (cid.Cid, bool) { manifestMx.RLock() defer manifestMx.RUnlock() @@ -76,53 +89,37 @@ func GetManifest(av Version) (cid.Cid, bool) { return c, ok } -func LoadManifests(ctx context.Context, store cbor.IpldStore) error { - manifestMx.Lock() - defer manifestMx.Unlock() - - return loadManifests(ctx, store) -} - -func loadManifests(ctx context.Context, store cbor.IpldStore) error { +// ReadManifest reads a manifest from a blockstore. It does not "add" it. +func ReadManifest(ctx context.Context, store cbor.IpldStore, mfCid cid.Cid) (map[string]cid.Cid, error) { adtStore := adt.WrapStore(ctx, store) - manifests = make(map[Version]*manifest.Manifest) - actorMeta = make(map[cid.Cid]actorEntry) + var mf manifest.Manifest + if err := adtStore.Get(ctx, mfCid, &mf); err != nil { + return nil, xerrors.Errorf("error reading manifest (cid: %s): %w", mfCid, err) + } - for av, mfCid := range manifestCids { - mf := &manifest.Manifest{} - if err := adtStore.Get(ctx, mfCid, mf); err != nil { - return xerrors.Errorf("error reading manifest for network version %d (cid: %s): %w", av, mfCid, err) - } + if err := mf.Load(ctx, adtStore); err != nil { + return nil, xerrors.Errorf("error loading manifest (cid: %s): %w", mfCid, err) + } - if err := mf.Load(ctx, adtStore); err != nil { - return xerrors.Errorf("error loading manifest for network version %d: %w", av, err) - } - - manifests[av] = mf - - var actorKeys = GetBuiltinActorsKeys() - for _, name := range actorKeys { - c, ok := mf.Get(name) - if ok { - actorMeta[c] = actorEntry{name: name, version: av} - } + actorKeys := GetBuiltinActorsKeys() // TODO: we should be able to enumerate manifests directly. + metadata := make(map[string]cid.Cid, len(actorKeys)) + for _, name := range actorKeys { + if c, ok := mf.Get(name); ok { + metadata[name] = c } } - return nil + return metadata, nil } +// GetActorCodeID looks up a builtin actor's code CID by actor version and canonical actor name name. func GetActorCodeID(av Version, name string) (cid.Cid, bool) { manifestMx.RLock() defer manifestMx.RUnlock() - mf, ok := manifests[av] - if ok { - return mf.Get(name) - } - - return cid.Undef, false + c, ok := manifests[av][name] + return c, ok } func GetActorMetaByCode(c cid.Cid) (string, Version, bool) { diff --git a/chain/consensus/filcns/upgrades.go b/chain/consensus/filcns/upgrades.go index 1e9f17d71..300ab1eb9 100644 --- a/chain/consensus/filcns/upgrades.go +++ b/chain/consensus/filcns/upgrades.go @@ -1371,9 +1371,7 @@ func upgradeActorsV8Common( store := store.ActorStore(ctx, buf) // ensure that the manifest is loaded in the blockstore - if err := bundle.FetchAndLoadBundles(ctx, buf, map[actors.Version]build.Bundle{ - actors.Version8: build.BuiltinActorReleases[actors.Version8], - }); err != nil { + if err := bundle.LoadBundles(ctx, buf, actors.Version8); err != nil { return cid.Undef, xerrors.Errorf("failed to load manifest bundle: %w", err) } @@ -1395,12 +1393,6 @@ func upgradeActorsV8Common( return cid.Undef, xerrors.Errorf("no manifest CID for v8 upgrade") } - if val, ok := build.ActorsCIDs[actors.Version8]; ok { - if val != manifest { - return cid.Undef, xerrors.Errorf("actors V8 manifest CID %s did not match CID given in params file: %s", manifest, val) - } - } - // Perform the migration newHamtRoot, err := nv16.MigrateStateTree(ctx, store, manifest, stateRoot.Actors, epoch, config, migrationLogger{}, cache) if err != nil { diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go index fa4a0dedd..138437b78 100644 --- a/chain/gen/genesis/genesis.go +++ b/chain/gen/genesis/genesis.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/filecoin-project/lotus/chain/consensus/filcns" + "github.com/filecoin-project/lotus/node/bundle" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" @@ -155,6 +156,10 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge return nil, nil, xerrors.Errorf("getting network version: %w", err) } + if err := bundle.LoadBundles(ctx, bs, av); err != nil { + return nil, nil, xerrors.Errorf("loading actors for genesis block: %w", err) + } + // Create system actor sysact, err := SetupSystemActor(ctx, bs, av) diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index ffc3ab138..01ff14480 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -164,9 +164,10 @@ func (ar *ActorRegistry) Register(av actors.Version, pred ActorPredicate, vmacto Ret: et.Out(0), } } - ar.Methods[a.Code()] = methods if realCode.Defined() { ar.Methods[realCode] = methods + } else { + ar.Methods[a.Code()] = methods } } } diff --git a/cli/init.go b/cli/init.go deleted file mode 100644 index 25d6a1116..000000000 --- a/cli/init.go +++ /dev/null @@ -1,21 +0,0 @@ -package cli - -import ( - "context" - - "github.com/filecoin-project/lotus/blockstore" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/node/bundle" -) - -func init() { - // preload manifest so that we have the correct code CID inventory for cli since that doesn't - // go through CI - // TODO loading the bundle in every cli invocation adds some latency; we should figure out a way - // to load actor CIDs without incurring this hit. - bs := blockstore.NewMemory() - - if err := bundle.FetchAndLoadBundles(context.Background(), bs, build.BuiltinActorReleases); err != nil { - panic(err) - } -} diff --git a/cmd/lotus-miner/allinfo_test.go b/cmd/lotus-miner/allinfo_test.go index f64a4ab72..9a5359213 100644 --- a/cmd/lotus-miner/allinfo_test.go +++ b/cmd/lotus-miner/allinfo_test.go @@ -12,7 +12,6 @@ import ( "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/node/repo" ) @@ -25,12 +24,6 @@ func TestMinerAllInfo(t *testing.T) { kit.QuietMiningLogs() - oldDelay := policy.GetPreCommitChallengeDelay() - policy.SetPreCommitChallengeDelay(5) - t.Cleanup(func() { - policy.SetPreCommitChallengeDelay(oldDelay) - }) - client, miner, ens := kit.EnsembleMinimal(t) ens.InterconnectAll().BeginMining(time.Second) diff --git a/cmd/lotus-miner/init.go b/cmd/lotus-miner/init.go index 6dedb2656..b2c71e18c 100644 --- a/cmd/lotus-miner/init.go +++ b/cmd/lotus-miner/init.go @@ -45,7 +45,6 @@ import ( lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/v0api" "github.com/filecoin-project/lotus/api/v1api" - "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" @@ -59,7 +58,6 @@ import ( "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/journal/fsjournal" storageminer "github.com/filecoin-project/lotus/miner" - "github.com/filecoin-project/lotus/node/bundle" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" @@ -218,13 +216,6 @@ var initCmd = &cli.Command{ return err } - // load bundles - bs := blockstore.NewMemory() - - if err := bundle.FetchAndLoadBundles(ctx, bs, build.BuiltinActorReleases); err != nil { - return err - } - var localPaths []stores.LocalPath if pssb := cctx.StringSlice("pre-sealed-sectors"); len(pssb) != 0 { diff --git a/cmd/lotus-seed/genesis.go b/cmd/lotus-seed/genesis.go index 61a4d2813..27bf25466 100644 --- a/cmd/lotus-seed/genesis.go +++ b/cmd/lotus-seed/genesis.go @@ -15,8 +15,6 @@ import ( "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/journal" - "github.com/filecoin-project/lotus/node/bundle" - "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/testing" "github.com/google/uuid" "github.com/mitchellh/go-homedir" @@ -580,12 +578,7 @@ var genesisCarCmd = &cli.Command{ bstor := blockstore.WrapIDStore(blockstore.NewMemorySync()) sbldr := vm.Syscalls(ffiwrapper.ProofVerifier) - // load appropriate bundles - if err := bundle.FetchAndLoadBundles(c.Context, bstor, build.BuiltinActorReleases); err != nil { - return err - } - - _, err := testing.MakeGenesis(ofile, c.Args().First())(bstor, sbldr, jrnl, dtypes.BuiltinActorsLoaded{})() + _, err := testing.MakeGenesis(ofile, c.Args().First())(bstor, sbldr, jrnl)() return err }, } diff --git a/gen/bundle/bundle.go b/gen/bundle/bundle.go new file mode 100644 index 000000000..c7655157e --- /dev/null +++ b/gen/bundle/bundle.go @@ -0,0 +1,50 @@ +package main + +import ( + "os" + "text/template" + + "github.com/filecoin-project/lotus/build" +) + +var tmpl *template.Template = template.Must(template.New("actor-metadata").Parse(` +// WARNING: This file has automatically been generated + +package build + +import ( + "github.com/ipfs/go-cid" +) + +var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMetadata{ +{{- range . }} { + Network: {{printf "%q" .Network}}, + Version: {{.Version}}, + ManifestCid: MustParseCid({{printf "%q" .ManifestCid}}), + Actors: map[string]cid.Cid { + {{- range $name, $cid := .Actors }} + {{printf "%q" $name}}: MustParseCid({{printf "%q" $cid}}), + {{- end }} + }, +}, +{{- end -}} +} +`)) + +func main() { + metadata, err := build.ReadEmbeddedBuiltinActorsMetadata() + if err != nil { + panic(err) + } + + fi, err := os.Create("./build/builtin_actors_gen.go") + if err != nil { + panic(err) + } + defer fi.Close() //nolint + + err = tmpl.Execute(fi, metadata) + if err != nil { + panic(err) + } +} diff --git a/go.mod b/go.mod index 624e2467b..f3ea49b1d 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ retract v1.14.0 // Accidentally force-pushed tag, use v1.14.1+ instead. require ( contrib.go.opencensus.io/exporter/prometheus v0.4.0 github.com/BurntSushi/toml v0.4.1 + github.com/DataDog/zstd v1.4.1 github.com/GeertJohan/go.rice v1.0.2 github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee github.com/Kubuxu/imtui v0.0.0-20210401140320-41663d68d0fa @@ -169,7 +170,6 @@ require ( ) require ( - github.com/DataDog/zstd v1.4.1 // indirect github.com/GeertJohan/go.incremental v1.0.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect diff --git a/itests/deals_512mb_test.go b/itests/deals_512mb_test.go index 967e33da4..8c3aea655 100644 --- a/itests/deals_512mb_test.go +++ b/itests/deals_512mb_test.go @@ -22,8 +22,6 @@ func TestStorageDealMissingBlock(t *testing.T) { //stm: @CLIENT_STORAGE_DEALS_LIST_IMPORTS_001 ctx := context.Background() - // enable 512MiB proofs so we can conduct larger transfers. - kit.EnableLargeSectors(t) kit.QuietMiningLogs() client, miner, ens := kit.EnsembleMinimal(t, diff --git a/itests/deals_anycid_test.go b/itests/deals_anycid_test.go index f504ead86..03317596b 100644 --- a/itests/deals_anycid_test.go +++ b/itests/deals_anycid_test.go @@ -18,7 +18,6 @@ import ( "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/modules" @@ -31,14 +30,7 @@ func TestDealRetrieveByAnyCid(t *testing.T) { } ctx := context.Background() - oldDelay := policy.GetPreCommitChallengeDelay() - policy.SetPreCommitChallengeDelay(10) - t.Cleanup(func() { - policy.SetPreCommitChallengeDelay(oldDelay) - }) - // Allow 8MB sectors - policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg8MiBV1) kit.QuietMiningLogs() // For these tests where the block time is artificially short, just use diff --git a/itests/deals_concurrent_test.go b/itests/deals_concurrent_test.go index 53aa0c782..3248a1d20 100644 --- a/itests/deals_concurrent_test.go +++ b/itests/deals_concurrent_test.go @@ -16,7 +16,6 @@ import ( provider "github.com/filecoin-project/index-provider" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/modules" @@ -39,12 +38,6 @@ func TestDealWithMarketAndMinerNode(t *testing.T) { kit.QuietMiningLogs() - oldDelay := policy.GetPreCommitChallengeDelay() - policy.SetPreCommitChallengeDelay(10) - t.Cleanup(func() { - policy.SetPreCommitChallengeDelay(oldDelay) - }) - // For these tests where the block time is artificially short, just use // a deal start epoch that is guaranteed to be far enough in the future // so that the deal starts sealing in time @@ -91,12 +84,6 @@ func TestDealCyclesConcurrent(t *testing.T) { t.Skip("skipping test in short mode") } - oldDelay := policy.GetPreCommitChallengeDelay() - policy.SetPreCommitChallengeDelay(10) - t.Cleanup(func() { - policy.SetPreCommitChallengeDelay(oldDelay) - }) - kit.QuietMiningLogs() // For these tests where the block time is artificially short, just use @@ -142,12 +129,6 @@ func TestSimultanenousTransferLimit(t *testing.T) { kit.QuietMiningLogs() - oldDelay := policy.GetPreCommitChallengeDelay() - policy.SetPreCommitChallengeDelay(10) - t.Cleanup(func() { - policy.SetPreCommitChallengeDelay(oldDelay) - }) - // For these tests where the block time is artificially short, just use // a deal start epoch that is guaranteed to be far enough in the future // so that the deal starts sealing in time diff --git a/itests/deals_max_staging_deals_test.go b/itests/deals_max_staging_deals_test.go index 6a4234e02..fd18c4de0 100644 --- a/itests/deals_max_staging_deals_test.go +++ b/itests/deals_max_staging_deals_test.go @@ -22,8 +22,6 @@ func TestMaxStagingDeals(t *testing.T) { //stm: @CLIENT_STORAGE_DEALS_LIST_IMPORTS_001 ctx := context.Background() - // enable 512MiB proofs so we can conduct larger transfers. - kit.EnableLargeSectors(t) kit.QuietMiningLogs() client, miner, ens := kit.EnsembleMinimal(t, diff --git a/itests/deals_padding_test.go b/itests/deals_padding_test.go index 9ac736f8c..2b0dd0d17 100644 --- a/itests/deals_padding_test.go +++ b/itests/deals_padding_test.go @@ -9,7 +9,6 @@ import ( commcid "github.com/filecoin-project/go-fil-commcid" commp "github.com/filecoin-project/go-fil-commp-hashhash" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/itests/kit" "github.com/stretchr/testify/require" ) @@ -26,7 +25,6 @@ func TestDealPadding(t *testing.T) { var blockTime = 250 * time.Millisecond startEpoch := abi.ChainEpoch(2 << 12) - policy.SetPreCommitChallengeDelay(10) client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.WithAllSubsystems()) // no mock proofs. ens.InterconnectAll().BeginMining(blockTime) diff --git a/itests/deals_partial_retrieval_dm-level_test.go b/itests/deals_partial_retrieval_dm-level_test.go index 7cb1d9b77..aa92cf731 100644 --- a/itests/deals_partial_retrieval_dm-level_test.go +++ b/itests/deals_partial_retrieval_dm-level_test.go @@ -13,7 +13,6 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" api0 "github.com/filecoin-project/lotus/api/v0api" - "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/itests/kit" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" @@ -46,7 +45,6 @@ func TestDMLevelPartialRetrieval(t *testing.T) { //stm: @CLIENT_RETRIEVAL_RETRIEVE_001, @CLIENT_RETRIEVAL_FIND_001 ctx := context.Background() - policy.SetPreCommitChallengeDelay(10) kit.QuietMiningLogs() client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.MockProofs()) dh := kit.NewDealHarness(t, client, miner, miner) diff --git a/itests/deals_partial_retrieval_test.go b/itests/deals_partial_retrieval_test.go index 5e7f073f0..e1be43637 100644 --- a/itests/deals_partial_retrieval_test.go +++ b/itests/deals_partial_retrieval_test.go @@ -16,7 +16,6 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/itests/kit" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" @@ -48,8 +47,6 @@ func TestPartialRetrieval(t *testing.T) { //stm: @CLIENT_RETRIEVAL_RETRIEVE_001 ctx := context.Background() - policy.SetPreCommitChallengeDelay(10) - kit.EnableLargeSectors(t) kit.QuietMiningLogs() client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.MockProofs(), kit.SectorSize(512<<20)) dh := kit.NewDealHarness(t, client, miner, miner) diff --git a/itests/deals_retry_deal_no_funds_test.go b/itests/deals_retry_deal_no_funds_test.go index 3abd82053..cf52d6f55 100644 --- a/itests/deals_retry_deal_no_funds_test.go +++ b/itests/deals_retry_deal_no_funds_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/itests/kit" @@ -35,14 +34,7 @@ func TestDealsRetryLackOfFunds(t *testing.T) { //stm: @CHAIN_INCOMING_HANDLE_INCOMING_BLOCKS_001, @CHAIN_INCOMING_VALIDATE_BLOCK_PUBSUB_001, @CHAIN_INCOMING_VALIDATE_MESSAGE_PUBSUB_001 //stm: @CLIENT_STORAGE_DEALS_LIST_IMPORTS_001 ctx := context.Background() - oldDelay := policy.GetPreCommitChallengeDelay() - policy.SetPreCommitChallengeDelay(10) - t.Cleanup(func() { - policy.SetPreCommitChallengeDelay(oldDelay) - }) - - policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg8MiBV1) kit.QuietMiningLogs() // Allow 8MB sectors @@ -119,14 +111,6 @@ func TestDealsRetryLackOfFunds_blockInPublishDeal(t *testing.T) { //stm: @CHAIN_SYNCER_NEW_PEER_HEAD_001, @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_STOP_001 //stm: @CLIENT_STORAGE_DEALS_LIST_IMPORTS_001 ctx := context.Background() - oldDelay := policy.GetPreCommitChallengeDelay() - policy.SetPreCommitChallengeDelay(10) - - t.Cleanup(func() { - policy.SetPreCommitChallengeDelay(oldDelay) - }) - - policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg8MiBV1) kit.QuietMiningLogs() // Allow 8MB sectors @@ -200,14 +184,6 @@ func TestDealsRetryLackOfFunds_belowLimit(t *testing.T) { //stm: @CHAIN_SYNCER_NEW_PEER_HEAD_001, @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_STOP_001 //stm: @CLIENT_STORAGE_DEALS_LIST_IMPORTS_001 ctx := context.Background() - oldDelay := policy.GetPreCommitChallengeDelay() - policy.SetPreCommitChallengeDelay(10) - - t.Cleanup(func() { - policy.SetPreCommitChallengeDelay(oldDelay) - }) - - policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg8MiBV1) kit.QuietMiningLogs() // Allow 8MB sectors diff --git a/itests/deals_test.go b/itests/deals_test.go index c7a2b2ad1..9810d417a 100644 --- a/itests/deals_test.go +++ b/itests/deals_test.go @@ -5,7 +5,6 @@ import ( "testing" "time" - "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/itests/kit" ) @@ -22,12 +21,6 @@ func TestDealsWithSealingAndRPC(t *testing.T) { kit.QuietMiningLogs() - oldDelay := policy.GetPreCommitChallengeDelay() - policy.SetPreCommitChallengeDelay(10) - t.Cleanup(func() { - policy.SetPreCommitChallengeDelay(oldDelay) - }) - client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.WithAllSubsystems()) // no mock proofs. ens.InterconnectAll().BeginMining(250 * time.Millisecond) dh := kit.NewDealHarness(t, client, miner, miner) diff --git a/itests/kit/ensemble.go b/itests/kit/ensemble.go index ee6482cf7..8e0a3ef6c 100644 --- a/itests/kit/ensemble.go +++ b/itests/kit/ensemble.go @@ -160,6 +160,14 @@ func NewEnsemble(t *testing.T, opts ...EnsembleOpt) *Ensemble { }) } + // Ensure we're using the right actors. This really shouldn't be some global thing, but it's + // the best we can do for now. + if n.options.mockProofs { + require.NoError(t, build.UseNetworkBundle("testing-fake-proofs")) + } else { + require.NoError(t, build.UseNetworkBundle("testing")) + } + return n } diff --git a/itests/kit/init.go b/itests/kit/init.go index c455cf8c6..33a9ace7d 100644 --- a/itests/kit/init.go +++ b/itests/kit/init.go @@ -15,10 +15,27 @@ import ( func init() { _ = logging.SetLogLevel("*", "INFO") + // These values mimic the values set in the builtin-actors when configured to use the "testing" network. Specifically: + // - All proof types. + // - 2k minimum power. + // - "small" verified deals. + // - short precommit + // + // See: + // - https://github.com/filecoin-project/builtin-actors/blob/0502c0722225ee58d1e6641431b4f9356cb2d18e/actors/runtime/src/runtime/policy.rs#L235 + // - https://github.com/filecoin-project/builtin-actors/blob/0502c0722225ee58d1e6641431b4f9356cb2d18e/actors/runtime/build.rs#L17-L45 policy.SetProviderCollateralSupplyTarget(big.Zero(), big.NewInt(1)) policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) - policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) + policy.SetPreCommitChallengeDelay(10) + policy.SetSupportedProofTypes( + abi.RegisteredSealProof_StackedDrg2KiBV1, + abi.RegisteredSealProof_StackedDrg8MiBV1, + abi.RegisteredSealProof_StackedDrg512MiBV1, + abi.RegisteredSealProof_StackedDrg32GiBV1, + abi.RegisteredSealProof_StackedDrg64GiBV1, + ) + policy.SetProviderCollateralSupplyTarget(big.NewInt(0), big.NewInt(100)) build.InsecurePoStValidation = true diff --git a/itests/kit/sectors.go b/itests/kit/sectors.go deleted file mode 100644 index d9c52b9ca..000000000 --- a/itests/kit/sectors.go +++ /dev/null @@ -1,23 +0,0 @@ -package kit - -import ( - "testing" - - "github.com/filecoin-project/go-state-types/abi" - - "github.com/filecoin-project/lotus/chain/actors/policy" -) - -// EnableLargeSectors enables 512MiB sectors. This is useful in combination with -// mock proofs, for testing larger transfers. -func EnableLargeSectors(t *testing.T) { - policy.SetSupportedProofTypes( - abi.RegisteredSealProof_StackedDrg2KiBV1, - abi.RegisteredSealProof_StackedDrg512MiBV1, // <== here - ) - t.Cleanup(func() { // reset when done. - policy.SetSupportedProofTypes( - abi.RegisteredSealProof_StackedDrg2KiBV1, - ) - }) -} diff --git a/lotuspond/front/src/chain/methodgen.go b/lotuspond/front/src/chain/methodgen.go index 057ec7d01..da174bfaa 100644 --- a/lotuspond/front/src/chain/methodgen.go +++ b/lotuspond/front/src/chain/methodgen.go @@ -5,10 +5,9 @@ import ( "io/ioutil" "os" - "github.com/multiformats/go-multihash" - "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/consensus/filcns" ) @@ -46,12 +45,7 @@ func main() { out := map[string][]string{} for c, methods := range filcns.NewActorRegistry().Methods { - cmh, err := multihash.Decode(c.Hash()) - if err != nil { - panic(err) - } - - name := string(cmh.Digest) + name := builtin.ActorNameByCode(c) remaining := len(methods) // iterate over actor methods in order. diff --git a/node/builder.go b/node/builder.go index 6f9805760..f0106ad97 100644 --- a/node/builder.go +++ b/node/builder.go @@ -384,9 +384,6 @@ func Test() Option { Unset(new(*peermgr.PeerMgr)), Override(new(beacon.Schedule), testing.RandomBeacon), Override(new(*storageadapter.DealPublisher), storageadapter.NewDealPublisher(nil, storageadapter.PublishMsgConfig{})), - // use the testing bundles - Unset(new(dtypes.BuiltinActorsLoaded)), - Override(new(dtypes.BuiltinActorsLoaded), modules.LoadBuiltinActorsTesting), ) } diff --git a/node/builder_chain.go b/node/builder_chain.go index 650b79794..d2c116b96 100644 --- a/node/builder_chain.go +++ b/node/builder_chain.go @@ -48,13 +48,6 @@ var ChainNode = Options( // Full node or lite node // TODO: Fix offline mode - // FVM: builtin actor bundle loading - // Note: this has to load before the upgrade schedule, so that we can patch in the - // right manifest cid. - // This restriction will be lifted once we have the final actors v8 bundle and we know - // the manifest cid. - Override(new(dtypes.BuiltinActorsLoaded), modules.LoadBuiltinActors), - // Consensus settings Override(new(dtypes.DrandSchedule), modules.BuiltinDrandConfig), Override(new(stmgr.UpgradeSchedule), modules.UpgradeSchedule), diff --git a/node/builder_miner.go b/node/builder_miner.go index 11cd34af5..2223d14ce 100644 --- a/node/builder_miner.go +++ b/node/builder_miner.go @@ -51,10 +51,6 @@ var MinerNode = Options( // Mining / proving Override(new(*storage.AddressSelector), modules.AddressSelector(nil)), - - // builtin actors manifest - Override(new(dtypes.BuiltinActorsLoaded), modules.LoadBuiltinActors), - Override(new(dtypes.UniversalBlockstore), modules.MemoryBlockstore), ) func ConfigStorageMiner(c interface{}) Option { diff --git a/node/bundle/bundle.go b/node/bundle/bundle.go index a3a14f468..b08ff691c 100644 --- a/node/bundle/bundle.go +++ b/node/bundle/bundle.go @@ -2,215 +2,86 @@ package bundle import ( "bytes" - "crypto/sha256" - "encoding/hex" - "fmt" + "context" "io" - "net" - "net/http" "os" - "path/filepath" - "strings" - "time" + + "github.com/ipld/go-car" "golang.org/x/xerrors" - logging "github.com/ipfs/go-log/v2" + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors" + + cid "github.com/ipfs/go-cid" ) -var log = logging.Logger("bundle-fetcher") +func LoadBundleFromFile(ctx context.Context, bs blockstore.Blockstore, path string) (cid.Cid, error) { + f, err := os.Open(path) + if err != nil { + return cid.Undef, xerrors.Errorf("error opening bundle %q for builtin-actors: %w", path, err) + } + defer f.Close() //nolint -type BundleFetcher struct { - path string + return LoadBundle(ctx, bs, f) } -func NewBundleFetcher(basepath string) (*BundleFetcher, error) { - path := filepath.Join(basepath, "builtin-actors") - if err := os.MkdirAll(path, 0755); err != nil { - return nil, xerrors.Errorf("error making bundle directory %s: %w", path, err) +func LoadBundle(ctx context.Context, bs blockstore.Blockstore, r io.Reader) (cid.Cid, error) { + hdr, err := car.LoadCar(ctx, bs, r) + if err != nil { + return cid.Undef, xerrors.Errorf("error loading builtin actors bundle: %w", err) } - return &BundleFetcher{path: path}, nil + if len(hdr.Roots) != 1 { + return cid.Undef, xerrors.Errorf("expected one root when loading actors bundle, got %d", len(hdr.Roots)) + } + return hdr.Roots[0], nil } -func (b *BundleFetcher) FetchFromRelease(version int, release, netw string) (path string, err error) { - bundleName := fmt.Sprintf("builtin-actors-%s", netw) - bundleFile := fmt.Sprintf("%s.car", bundleName) - bundleHash := fmt.Sprintf("%s.sha256", bundleName) - bundleBasePath := filepath.Join(b.path, fmt.Sprintf("v%d", version), release) - - if err := os.MkdirAll(bundleBasePath, 0755); err != nil { - return "", xerrors.Errorf("error making bundle directory %s: %w", bundleBasePath, err) - } - - // check if it exists; if it does, check the hash - bundleFilePath := filepath.Join(bundleBasePath, bundleFile) - if _, err := os.Stat(bundleFilePath); err == nil { - err := b.checkRelease(bundleBasePath, bundleFile, bundleHash) - if err == nil { - return bundleFilePath, nil - } - - log.Warnf("invalid bundle %s: %s; refetching", bundleName, err) - } - - log.Infof("fetching bundle %s", bundleFile) - if err := b.fetchFromRelease(release, bundleBasePath, bundleFile, bundleHash); err != nil { - log.Errorf("error fetching bundle %s: %s", bundleName, err) - return "", xerrors.Errorf("error fetching bundle: %w", err) - } - - if err := b.checkRelease(bundleBasePath, bundleFile, bundleHash); err != nil { - log.Errorf("error checking bundle %s: %s", bundleName, err) - return "", xerrors.Errorf("error checking bundle: %s", err) - } - - return bundleFilePath, nil -} - -func (b *BundleFetcher) FetchFromURL(version int, release, netw, url, cksum string) (path string, err error) { - bundleName := fmt.Sprintf("builtin-actors-%s", netw) - bundleFile := fmt.Sprintf("%s.car", bundleName) - bundleBasePath := filepath.Join(b.path, fmt.Sprintf("v%d", version), release) - - if err := os.MkdirAll(bundleBasePath, 0755); err != nil { - return "", xerrors.Errorf("error making bundle directory %s: %w", bundleBasePath, err) - } - - // check if it exists; if it does, check the hash - bundleFilePath := filepath.Join(bundleBasePath, bundleFile) - if _, err := os.Stat(bundleFilePath); err == nil { - err := b.checkHash(bundleBasePath, bundleFile, cksum) - if err == nil { - return bundleFilePath, nil - } - - log.Warnf("invalid bundle %s: %s; refetching", bundleName, err) - } - - log.Infof("fetching bundle %s", bundleFile) - if err := b.fetchFromURL(bundleBasePath, bundleFile, url); err != nil { - log.Errorf("error fetching bundle %s: %s", bundleName, err) - return "", xerrors.Errorf("error fetching bundle: %w", err) - } - - if err := b.checkHash(bundleBasePath, bundleFile, cksum); err != nil { - log.Errorf("error checking bundle %s: %s", bundleName, err) - return "", xerrors.Errorf("error checking bundle: %s", err) - } - - return bundleFilePath, nil -} - -func (b *BundleFetcher) fetchURL(url, path string) error { - log.Infof("fetching URL: %s", url) - - for i := 0; i < 3; i++ { - resp, err := http.Get(url) //nolint - if err != nil { - if isTemporary(err) { - log.Warnf("temporary error fetching %s: %s; retrying in 1s", url, err) - time.Sleep(time.Second) - continue - } - return xerrors.Errorf("error fetching %s: %w", url, err) - } - defer resp.Body.Close() //nolint - - if resp.StatusCode != http.StatusOK { - log.Warnf("unexpected response fetching %s: %s (%d); retrying in 1s", url, resp.Status, resp.StatusCode) - time.Sleep(time.Second) +// LoadBundles loads the bundles for the specified actor versions into the passed blockstore, if and +// only if the bundle's manifest is not already present in the blockstore. +func LoadBundles(ctx context.Context, bs blockstore.Blockstore, versions ...actors.Version) error { + for _, av := range versions { + // No bundles before version 8. + if av < actors.Version8 { continue } - f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + manifestCid, ok := actors.GetManifest(av) + if !ok { + // All manifests are registered on start, so this must succeed. + return xerrors.Errorf("unknown actor version v%d", av) + } + + if haveManifest, err := bs.Has(ctx, manifestCid); err != nil { + return xerrors.Errorf("blockstore error when loading manifest %s: %w", manifestCid, err) + } else if haveManifest { + // We already have the manifest, and therefore everything under it. + continue + } + + var ( + root cid.Cid + err error + ) + if path, ok := build.BundleOverrides[av]; ok { + root, err = LoadBundleFromFile(ctx, bs, path) + } else if embedded, ok := build.GetEmbeddedBuiltinActorsBundle(av); ok { + root, err = LoadBundle(ctx, bs, bytes.NewReader(embedded)) + } else { + err = xerrors.Errorf("bundle for actors version v%d not found", av) + } + if err != nil { - return xerrors.Errorf("error opening %s for writing: %w", path, err) - } - defer f.Close() //nolint - - if _, err := io.Copy(f, resp.Body); err != nil { - return xerrors.Errorf("error writing %s: %w", path, err) + return err } - return nil - } + if root != manifestCid { + return xerrors.Errorf("expected manifest for actors version %d does not match actual: %s != %s", av, manifestCid, root) + } - return xerrors.Errorf("all attempts to fetch %s failed", url) -} - -func (b *BundleFetcher) fetchFromRelease(release, bundleBasePath, bundleFile, bundleHash string) error { - bundleHashUrl := fmt.Sprintf("https://github.com/filecoin-project/builtin-actors/releases/download/%s/%s", - release, bundleHash) - bundleHashPath := filepath.Join(bundleBasePath, bundleHash) - if err := b.fetchURL(bundleHashUrl, bundleHashPath); err != nil { - return err - } - - bundleFileUrl := fmt.Sprintf("https://github.com/filecoin-project/builtin-actors/releases/download/%s/%s", - release, bundleFile) - bundleFilePath := filepath.Join(bundleBasePath, bundleFile) - if err := b.fetchURL(bundleFileUrl, bundleFilePath); err != nil { - return err } return nil } - -func (b *BundleFetcher) fetchFromURL(bundleBasePath, bundleFile, url string) error { - bundleFilePath := filepath.Join(bundleBasePath, bundleFile) - return b.fetchURL(url, bundleFilePath) -} - -func (b *BundleFetcher) checkRelease(bundleBasePath, bundleFile, bundleHash string) error { - bundleHashPath := filepath.Join(bundleBasePath, bundleHash) - f, err := os.Open(bundleHashPath) - if err != nil { - return xerrors.Errorf("error opening %s: %w", bundleHashPath, err) - } - defer f.Close() //nolint - - bs, err := io.ReadAll(f) - if err != nil { - return xerrors.Errorf("error reading %s: %w", bundleHashPath, err) - } - - parts := strings.Split(string(bs), " ") - hashHex := parts[0] - - return b.checkHash(bundleBasePath, bundleFile, hashHex) -} - -func (b *BundleFetcher) checkHash(bundleBasePath, bundleFile, cksum string) error { - expectedDigest, err := hex.DecodeString(cksum) - if err != nil { - return xerrors.Errorf("error decoding digest from %s: %w", cksum, err) - } - - bundleFilePath := filepath.Join(bundleBasePath, bundleFile) - f, err := os.Open(bundleFilePath) - if err != nil { - return xerrors.Errorf("error opening %s: %w", bundleFilePath, err) - } - defer f.Close() //nolint - - h256 := sha256.New() - if _, err := io.Copy(h256, f); err != nil { - return xerrors.Errorf("error computing digest for %s: %w", bundleFilePath, err) - } - digest := h256.Sum(nil) - - if !bytes.Equal(digest, expectedDigest) { - return xerrors.Errorf("hash mismatch") - } - - return nil -} - -func isTemporary(err error) bool { - if ne, ok := err.(net.Error); ok { - return ne.Temporary() - } - - return false -} diff --git a/node/bundle/manifest.go b/node/bundle/manifest.go deleted file mode 100644 index b292b94eb..000000000 --- a/node/bundle/manifest.go +++ /dev/null @@ -1,127 +0,0 @@ -package bundle - -import ( - "context" - "fmt" - "os" - - "github.com/ipld/go-car" - - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/blockstore" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" - - cid "github.com/ipfs/go-cid" - cbor "github.com/ipfs/go-ipld-cbor" - - "github.com/mitchellh/go-homedir" -) - -func FetchAndLoadBundleFromRelease(ctx context.Context, basePath string, bs blockstore.Blockstore, av actors.Version, rel, netw string) (cid.Cid, error) { - fetcher, err := NewBundleFetcher(basePath) - if err != nil { - return cid.Undef, xerrors.Errorf("error creating fetcher for builtin-actors version %d: %w", av, err) - } - - path, err := fetcher.FetchFromRelease(int(av), rel, netw) - if err != nil { - return cid.Undef, xerrors.Errorf("error fetching bundle for builtin-actors version %d: %w", av, err) - } - - return LoadBundle(ctx, bs, path, av) -} - -func FetchAndLoadBundleFromURL(ctx context.Context, basePath string, bs blockstore.Blockstore, av actors.Version, rel, netw, url, cksum string) (cid.Cid, error) { - fetcher, err := NewBundleFetcher(basePath) - if err != nil { - return cid.Undef, xerrors.Errorf("error creating fetcher for builtin-actors version %d: %w", av, err) - } - - path, err := fetcher.FetchFromURL(int(av), rel, netw, url, cksum) - if err != nil { - return cid.Undef, xerrors.Errorf("error fetching bundle for builtin-actors version %d: %w", av, err) - } - - return LoadBundle(ctx, bs, path, av) -} - -func LoadBundle(ctx context.Context, bs blockstore.Blockstore, path string, av actors.Version) (cid.Cid, error) { - f, err := os.Open(path) - if err != nil { - return cid.Undef, xerrors.Errorf("error opening bundle for builtin-actors version %d: %w", av, err) - } - defer f.Close() //nolint - - hdr, err := car.LoadCar(ctx, bs, f) - if err != nil { - return cid.Undef, xerrors.Errorf("error loading builtin actors v%d bundle: %w", av, err) - } - - // TODO: check that this only has one root? - manifestCid := hdr.Roots[0] - - if val, ok := build.ActorsCIDs[av]; ok { - if val != manifestCid { - return cid.Undef, xerrors.Errorf("actors V%d manifest CID %s did not match CID given in params file: %s", av, manifestCid, val) - } - } - actors.AddManifest(av, manifestCid) - - mfCid, ok := actors.GetManifest(av) - if !ok { - return cid.Undef, xerrors.Errorf("missing manifest CID for builtin-actors vrsion %d", av) - } - - return mfCid, nil -} - -// utility for blanket loading outside DI -func FetchAndLoadBundles(ctx context.Context, bs blockstore.Blockstore, bar map[actors.Version]build.Bundle) error { - netw := build.NetworkBundle - - path := os.Getenv("LOTUS_PATH") - if path == "" { - var err error - path, err = homedir.Expand("~/.lotus") - if err != nil { - return err - } - } - - for av, bd := range bar { - envvar := fmt.Sprintf("LOTUS_BUILTIN_ACTORS_V%d_BUNDLE", av) - switch { - case os.Getenv(envvar) != "": - // this is a local bundle, specified by an env var to load from the filesystem - path := os.Getenv(envvar) - - if _, err := LoadBundle(ctx, bs, path, av); err != nil { - return err - } - - case bd.Path[netw] != "": - if _, err := LoadBundle(ctx, bs, bd.Path[netw], av); err != nil { - return err - } - - case bd.URL[netw].URL != "": - if _, err := FetchAndLoadBundleFromURL(ctx, path, bs, av, bd.Release, netw, bd.URL[netw].URL, bd.URL[netw].Checksum); err != nil { - return err - } - - case bd.Release != "": - if _, err := FetchAndLoadBundleFromRelease(ctx, path, bs, av, bd.Release, netw); err != nil { - return err - } - } - } - - cborStore := cbor.NewCborStore(bs) - if err := actors.LoadManifests(ctx, cborStore); err != nil { - return err - } - - return nil -} diff --git a/node/modules/builtin_actors.go b/node/modules/builtin_actors.go deleted file mode 100644 index 474619c05..000000000 --- a/node/modules/builtin_actors.go +++ /dev/null @@ -1,177 +0,0 @@ -package modules - -import ( - "fmt" - "os" - "sync" - - "go.uber.org/fx" - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/node/bundle" - "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/node/modules/helpers" - "github.com/filecoin-project/lotus/node/repo" - - cid "github.com/ipfs/go-cid" - dstore "github.com/ipfs/go-datastore" - cbor "github.com/ipfs/go-ipld-cbor" -) - -func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo, bs dtypes.UniversalBlockstore, ds dtypes.MetadataDS) (result dtypes.BuiltinActorsLoaded, err error) { - ctx := helpers.LifecycleCtx(mctx, lc) - - // We can't put it as a dep in inputs causes a stack overflow in DI from circular dependency - // So we pass it through ldflags instead - netw := build.NetworkBundle - - for av, bd := range build.BuiltinActorReleases { - // first check to see if we know this release - key := dstore.NewKey(fmt.Sprintf("/builtin-actors/v%d/%s/%s", av, bd.Release, netw)) - - data, err := ds.Get(ctx, key) - switch err { - case nil: - // ok, we do, this should be the manifest cid - mfCid, err := cid.Cast(data) - if err != nil { - return result, xerrors.Errorf("error parsing cid for %s: %w", key, err) - } - - // check the blockstore for existence of the manifest - has, err := bs.Has(ctx, mfCid) - if err != nil { - return result, xerrors.Errorf("error checking blockstore for manifest cid %s: %w", mfCid, err) - } - - if has { - // it's there, no need to reload the bundle to the blockstore; just add it to the manifest list. - if val, ok := build.ActorsCIDs[av]; ok { - if val != mfCid { - return result, xerrors.Errorf("actors V%d manifest CID %s did not match CID given in params file: %s", av, mfCid, val) - } - } - actors.AddManifest(av, mfCid) - continue - } - - // we have a release key but don't have the manifest in the blockstore; maybe the user - // nuked his blockstore to restart from a snapshot. So fallthrough to refetch (if necessary) - // and reload the bundle. - - case dstore.ErrNotFound: - // we don't have a release key, we need to load the bundle - - default: - return result, xerrors.Errorf("error loading %s from datastore: %w", key, err) - } - - // we haven't recorded it in the datastore, so we need to load it - envvar := fmt.Sprintf("LOTUS_BUILTIN_ACTORS_V%d_BUNDLE", av) - var mfCid cid.Cid - switch { - case os.Getenv(envvar) != "": - path := os.Getenv(envvar) - - mfCid, err = bundle.LoadBundle(ctx, bs, path, av) - if err != nil { - return result, err - } - - case bd.Path[netw] != "": - // this is a local bundle, load it directly from the filessystem - mfCid, err = bundle.LoadBundle(ctx, bs, bd.Path[netw], av) - if err != nil { - return result, err - } - - case bd.URL[netw].URL != "": - // fetch it from the specified URL - mfCid, err = bundle.FetchAndLoadBundleFromURL(ctx, r.Path(), bs, av, bd.Release, netw, bd.URL[netw].URL, bd.URL[netw].Checksum) - if err != nil { - return result, err - } - - case bd.Release != "": - // fetch it and add it to the blockstore - mfCid, err = bundle.FetchAndLoadBundleFromRelease(ctx, r.Path(), bs, av, bd.Release, netw) - if err != nil { - return result, err - } - - default: - return result, xerrors.Errorf("no release or path specified for version %d bundle", av) - } - - if bd.Development || bd.Release == "" { - // don't store the release key so that we always load development bundles - continue - } - - // add the release key with the manifest to avoid reloading it in next restart. - if err := ds.Put(ctx, key, mfCid.Bytes()); err != nil { - return result, xerrors.Errorf("error storing manifest CID for builtin-actors vrsion %d to the datastore: %w", av, err) - } - } - - // we've loaded all the bundles, now load the manifests to get actor code CIDs. - cborStore := cbor.NewCborStore(bs) - if err := actors.LoadManifests(ctx, cborStore); err != nil { - return result, xerrors.Errorf("error loading actor manifests: %w", err) - } - - return result, nil -} - -// for itests -var testingBundleMx sync.Mutex - -func LoadBuiltinActorsTesting(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtypes.UniversalBlockstore) (result dtypes.BuiltinActorsLoaded, err error) { - ctx := helpers.LifecycleCtx(mctx, lc) - - var netw string - if build.InsecurePoStValidation { - netw = "testing-fake-proofs" - } else { - netw = "testing" - } - - testingBundleMx.Lock() - defer testingBundleMx.Unlock() - - // don't clobber the bundle during migration - build.NetworkBundle = netw - - const basePath = "/tmp/lotus-testing" - for av, bd := range build.BuiltinActorReleases { - switch { - case bd.Path[netw] != "": - if _, err := bundle.LoadBundle(ctx, bs, bd.Path[netw], av); err != nil { - return result, xerrors.Errorf("error loading testing bundle for builtin-actors version %d/%s: %w", av, netw, err) - } - - case bd.URL[netw].URL != "": - // fetch it from the specified URL - if _, err := bundle.FetchAndLoadBundleFromURL(ctx, basePath, bs, av, bd.Release, netw, bd.URL[netw].URL, bd.URL[netw].Checksum); err != nil { - return result, err - } - - case bd.Release != "": - if _, err := bundle.FetchAndLoadBundleFromRelease(ctx, basePath, bs, av, bd.Release, netw); err != nil { - return result, xerrors.Errorf("error loading testing bundle for builtin-actors version %d/%s: %w", av, netw, err) - } - - default: - return result, xerrors.Errorf("no path or release specified for version %d testing bundle", av) - } - } - - cborStore := cbor.NewCborStore(bs) - if err := actors.LoadManifests(ctx, cborStore); err != nil { - return result, xerrors.Errorf("error loading actor manifests: %w", err) - } - - return result, nil -} diff --git a/node/modules/chain.go b/node/modules/chain.go index ceb18f316..a01feddb2 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -178,6 +178,6 @@ func NewSlashFilter(ds dtypes.MetadataDS) *slashfilter.SlashFilter { return slashfilter.New(ds) } -func UpgradeSchedule(_ dtypes.BuiltinActorsLoaded) stmgr.UpgradeSchedule { +func UpgradeSchedule() stmgr.UpgradeSchedule { return filcns.DefaultUpgradeSchedule() } diff --git a/node/modules/dtypes/chain.go b/node/modules/dtypes/chain.go index da18b8664..0c924c12a 100644 --- a/node/modules/dtypes/chain.go +++ b/node/modules/dtypes/chain.go @@ -2,4 +2,3 @@ package dtypes type NetworkName string type AfterGenesisSet struct{} -type BuiltinActorsLoaded struct{} diff --git a/node/modules/genesis.go b/node/modules/genesis.go index 5aea2a5b4..03b4e2907 100644 --- a/node/modules/genesis.go +++ b/node/modules/genesis.go @@ -22,8 +22,8 @@ func ErrorGenesis() Genesis { } } -func LoadGenesis(genBytes []byte) func(fx.Lifecycle, helpers.MetricsCtx, dtypes.ChainBlockstore, dtypes.BuiltinActorsLoaded) Genesis { - return func(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtypes.ChainBlockstore, _ dtypes.BuiltinActorsLoaded) Genesis { +func LoadGenesis(genBytes []byte) func(fx.Lifecycle, helpers.MetricsCtx, dtypes.ChainBlockstore) Genesis { + return func(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtypes.ChainBlockstore) Genesis { return func() (header *types.BlockHeader, e error) { ctx := helpers.LifecycleCtx(mctx, lc) c, err := car.LoadCar(ctx, bs, bytes.NewReader(genBytes)) @@ -49,7 +49,7 @@ func LoadGenesis(genBytes []byte) func(fx.Lifecycle, helpers.MetricsCtx, dtypes. func DoSetGenesis(_ dtypes.AfterGenesisSet) {} -func SetGenesis(lc fx.Lifecycle, mctx helpers.MetricsCtx, cs *store.ChainStore, g Genesis, _ dtypes.BuiltinActorsLoaded) (dtypes.AfterGenesisSet, error) { +func SetGenesis(lc fx.Lifecycle, mctx helpers.MetricsCtx, cs *store.ChainStore, g Genesis) (dtypes.AfterGenesisSet, error) { ctx := helpers.LifecycleCtx(mctx, lc) genFromRepo, err := cs.GetGenesis(ctx) if err == nil { diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index ae3628764..09b8b6f31 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -219,8 +219,6 @@ type StorageMinerParams struct { Journal journal.Journal AddrSel *storage.AddressSelector Maddr dtypes.MinerAddress - - ManifestLoaded dtypes.BuiltinActorsLoaded } func StorageMiner(fc config.MinerFeeConfig) func(params StorageMinerParams) (*storage.Miner, error) { diff --git a/node/modules/testing/genesis.go b/node/modules/testing/genesis.go index 68d792d83..a3d25e36a 100644 --- a/node/modules/testing/genesis.go +++ b/node/modules/testing/genesis.go @@ -30,8 +30,8 @@ import ( var glog = logging.Logger("genesis") -func MakeGenesisMem(out io.Writer, template genesis.Template) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal, _ dtypes.BuiltinActorsLoaded) modules.Genesis { - return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal, _ dtypes.BuiltinActorsLoaded) modules.Genesis { +func MakeGenesisMem(out io.Writer, template genesis.Template) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { + return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { return func() (*types.BlockHeader, error) { glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network") b, err := genesis2.MakeGenesisBlock(context.TODO(), j, bs, syscalls, template) @@ -51,8 +51,8 @@ func MakeGenesisMem(out io.Writer, template genesis.Template) func(bs dtypes.Cha } } -func MakeGenesis(outFile, genesisTemplate string) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal, _ dtypes.BuiltinActorsLoaded) modules.Genesis { - return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal, _ dtypes.BuiltinActorsLoaded) modules.Genesis { +func MakeGenesis(outFile, genesisTemplate string) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { + return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { return func() (*types.BlockHeader, error) { glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network") genesisTemplate, err := homedir.Expand(genesisTemplate)