From b4be759b2c453dd409295d77095a301296a07057 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 16 May 2022 15:11:10 +0300 Subject: [PATCH 01/19] don't store dev bundle release keys in the datastore This is a nicety for development, so that we always load a development bundle to avoid having to give them distinct names etc. Just call you release "dev" or "dev.xxx..." and put the bundle in `.lotus/builtin-actors/v8/dev/builting-actors-${network}.{car,sha256sum}` and it will be unconditionally loaded. --- node/modules/builtin_actors.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/node/modules/builtin_actors.go b/node/modules/builtin_actors.go index 05b6e9e32..991372526 100644 --- a/node/modules/builtin_actors.go +++ b/node/modules/builtin_actors.go @@ -2,6 +2,7 @@ package modules import ( "fmt" + "strings" "sync" "go.uber.org/fx" @@ -71,6 +72,11 @@ func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRe return result, err } + if rel == "dev" || strings.HasPrefix(rel, "dev.") { + // don't store the release key so that we always load development bundles + continue + } + // add the release key with the manifest to avoid reloading it in next restart. if err := ds.Put(ctx, key, mfCid.Bytes()); err != nil { return result, xerrors.Errorf("error storing manifest CID for builtin-actors vrsion %d to the datastore: %w", av, err) From 46a85fc11e9d5e4ad0f64c06eb7123aaa972b334 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 16 May 2022 21:09:09 +0300 Subject: [PATCH 02/19] niceties for development bundle loading --- build/builtin_actors.go | 6 ++--- build/bundle.go | 7 ++++++ node/bundle/manifest.go | 18 ++++++++++---- node/modules/builtin_actors.go | 44 ++++++++++++++++++++++++---------- 4 files changed, 55 insertions(+), 20 deletions(-) diff --git a/build/builtin_actors.go b/build/builtin_actors.go index e65ebabf9..c6ed38c5b 100644 --- a/build/builtin_actors.go +++ b/build/builtin_actors.go @@ -8,10 +8,10 @@ import ( "github.com/BurntSushi/toml" ) -var BuiltinActorReleases map[actors.Version]string +var BuiltinActorReleases map[actors.Version]Bundle func init() { - BuiltinActorReleases = make(map[actors.Version]string) + BuiltinActorReleases = make(map[actors.Version]Bundle) spec := BundleSpec{} @@ -22,6 +22,6 @@ func init() { } for _, b := range spec.Bundles { - BuiltinActorReleases[b.Version] = b.Release + BuiltinActorReleases[b.Version] = b } } diff --git a/build/bundle.go b/build/bundle.go index 1a6fbcd93..3afe41daa 100644 --- a/build/bundle.go +++ b/build/bundle.go @@ -16,6 +16,13 @@ type BundleSpec struct { } type Bundle struct { + // actors version in this bundle Version actors.Version + // release id Release string + // bundle path: optional, uses the appropriate release bundle if unset + Path string + // development version; when set, in conjunction with path, it will always + // load the bundle to the blockstore + Development bool } diff --git a/node/bundle/manifest.go b/node/bundle/manifest.go index 45c632bdb..4bd55075e 100644 --- a/node/bundle/manifest.go +++ b/node/bundle/manifest.go @@ -28,6 +28,10 @@ func FetchAndLoadBundle(ctx context.Context, basePath string, bs blockstore.Bloc return cid.Undef, xerrors.Errorf("error fetching bundle for builtin-actors version %d: %w", av, err) } + return LoadBundle(ctx, bs, path, av) +} + +func LoadBundle(ctx context.Context, bs blockstore.Blockstore, path string, av actors.Version) (cid.Cid, error) { f, err := os.Open(path) if err != nil { return cid.Undef, xerrors.Errorf("error opening bundle for builtin-actors vresion %d: %w", av, err) @@ -52,7 +56,7 @@ func FetchAndLoadBundle(ctx context.Context, basePath string, bs blockstore.Bloc } // utility for blanket loading outside DI -func FetchAndLoadBundles(ctx context.Context, bs blockstore.Blockstore, bar map[actors.Version]string) error { +func FetchAndLoadBundles(ctx context.Context, bs blockstore.Blockstore, bar map[actors.Version]build.Bundle) error { netw := build.NetworkBundle if netw == "" { netw = "mainnet" @@ -67,9 +71,15 @@ func FetchAndLoadBundles(ctx context.Context, bs blockstore.Blockstore, bar map[ } } - for av, rel := range bar { - if _, err := FetchAndLoadBundle(ctx, path, bs, av, rel, netw); err != nil { - return err + for av, bd := range bar { + if bd.Path != "" { + if _, err := LoadBundle(ctx, bs, bd.Path, av); err != nil { + return err + } + } else { + if _, err := FetchAndLoadBundle(ctx, path, bs, av, bd.Release, netw); err != nil { + return err + } } } diff --git a/node/modules/builtin_actors.go b/node/modules/builtin_actors.go index 991372526..5f496b00f 100644 --- a/node/modules/builtin_actors.go +++ b/node/modules/builtin_actors.go @@ -2,7 +2,7 @@ package modules import ( "fmt" - "strings" + "path/filepath" "sync" "go.uber.org/fx" @@ -30,9 +30,9 @@ func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRe netw = "mainnet" } - for av, rel := range build.BuiltinActorReleases { + for av, bd := range build.BuiltinActorReleases { // first check to see if we know this release - key := dstore.NewKey(fmt.Sprintf("/builtin-actors/v%d/%s", av, rel)) + key := dstore.NewKey(fmt.Sprintf("/builtin-actors/v%d/%s", av, bd.Release)) data, err := ds.Get(ctx, key) switch err { @@ -66,13 +66,23 @@ func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRe return result, xerrors.Errorf("error loading %s from datastore: %w", key, err) } - // ok, we don't have it -- fetch it and add it to the blockstore - mfCid, err := bundle.FetchAndLoadBundle(ctx, r.Path(), bs, av, rel, netw) - if err != nil { - return result, err + // we haven't recorded it in the daatastore, so we need to load it + var mfCid cid.Cid + if bd.Path != "" { + // this is a local bundle, load it directly from the filessystem + mfCid, err = bundle.LoadBundle(ctx, bs, bd.Path, av) + if err != nil { + return result, err + } + } else { + // fetch it and add it to the blockstore + mfCid, err = bundle.FetchAndLoadBundle(ctx, r.Path(), bs, av, bd.Release, netw) + if err != nil { + return result, err + } } - if rel == "dev" || strings.HasPrefix(rel, "dev.") { + if bd.Development { // don't store the release key so that we always load development bundles continue } @@ -108,11 +118,19 @@ func LoadBuiltinActorsTesting(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtype testingBundleMx.Lock() defer testingBundleMx.Unlock() - for av, rel := range build.BuiltinActorReleases { - const basePath = "/tmp/lotus-testing" - - if _, err := bundle.FetchAndLoadBundle(ctx, basePath, bs, av, rel, netw); err != nil { - return result, xerrors.Errorf("error loading bundle for builtin-actors vresion %d: %w", av, err) + for av, bd := range build.BuiltinActorReleases { + if bd.Path != "" { + // we need the appopriate bundle for tests; it should live next to the main bundle, with the + // appropriate network name + path := filepath.Join(filepath.Dir(bd.Path), fmt.Sprintf("builtin-actors-%s.car", netw)) + if _, err := bundle.LoadBundle(ctx, bs, path, av); err != nil { + return result, xerrors.Errorf("error loading testing bundle for builtin-actors version %d/%s: %w", av, netw, err) + } + } else { + const basePath = "/tmp/lotus-testing" + if _, err := bundle.FetchAndLoadBundle(ctx, basePath, bs, av, bd.Release, netw); err != nil { + return result, xerrors.Errorf("error loading testing bundle for builtin-actors version %d/%s: %w", av, netw, err) + } } } From 30283e60f5545cae7274c58f0e3861f203961666 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 16 May 2022 21:13:58 +0300 Subject: [PATCH 03/19] add basic readme for bundle specifications --- build/README-bundle.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 build/README-bundle.md diff --git a/build/README-bundle.md b/build/README-bundle.md new file mode 100644 index 000000000..31ca6cb93 --- /dev/null +++ b/build/README-bundle.md @@ -0,0 +1,31 @@ +# Builtin Actor Bundles + +With NV16, builtin actor bundles must be loaded into lotus for the FVM to operate. + +The bundles are specified in build/bundles.toml using the foloowing syntax: +```toml +[[bundles]] +version = X # actors version +release = tag # release gag +``` + +This will add a bundle for version `X`, using the github release `tag` +to fetch the bundles at first startup. + +If you don't want to fetch the bundle from github, you can specify an explicit path to the bundle (which must be appropriate for your network, typically mainnet): +```toml +[[bundles]] +version = X # actors version +release = tag # release gag +path = /path/to/builtin-actors.car +``` + +For development bundles, you can also specify `development = true` so that the bundle is not +recorded in the datastore and reloaded every time the daemon starts up: +```toml +[[bundles]] +version = X # actors version +release = tag # release gag +path = /path/to/builtin-actors.car +development = true +``` From 221e7952abf264a95166a94e50ea40f34f0dff28 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 16 May 2022 21:38:09 +0300 Subject: [PATCH 04/19] fix typo Co-authored-by: raulk --- node/modules/builtin_actors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/modules/builtin_actors.go b/node/modules/builtin_actors.go index 5f496b00f..a252afd85 100644 --- a/node/modules/builtin_actors.go +++ b/node/modules/builtin_actors.go @@ -66,7 +66,7 @@ func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRe return result, xerrors.Errorf("error loading %s from datastore: %w", key, err) } - // we haven't recorded it in the daatastore, so we need to load it + // we haven't recorded it in the datastore, so we need to load it var mfCid cid.Cid if bd.Path != "" { // this is a local bundle, load it directly from the filessystem From 5eccdaaab706d8d6fe2a27c625e11ba3c042c3fd Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 16 May 2022 21:38:43 +0300 Subject: [PATCH 05/19] fix typo Co-authored-by: raulk --- node/modules/builtin_actors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/modules/builtin_actors.go b/node/modules/builtin_actors.go index a252afd85..67cd9871d 100644 --- a/node/modules/builtin_actors.go +++ b/node/modules/builtin_actors.go @@ -120,7 +120,7 @@ func LoadBuiltinActorsTesting(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtype for av, bd := range build.BuiltinActorReleases { if bd.Path != "" { - // we need the appopriate bundle for tests; it should live next to the main bundle, with the + // we need the appropriate bundle for tests; it should live next to the main bundle, with the // appropriate network name path := filepath.Join(filepath.Dir(bd.Path), fmt.Sprintf("builtin-actors-%s.car", netw)) if _, err := bundle.LoadBundle(ctx, bs, path, av); err != nil { From 06f6b2885ae0fc2bd2d8127fe0c4db5786742fc1 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 16 May 2022 21:39:08 +0300 Subject: [PATCH 06/19] fix typo Co-authored-by: raulk --- build/README-bundle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/README-bundle.md b/build/README-bundle.md index 31ca6cb93..397a80262 100644 --- a/build/README-bundle.md +++ b/build/README-bundle.md @@ -2,7 +2,7 @@ With NV16, builtin actor bundles must be loaded into lotus for the FVM to operate. -The bundles are specified in build/bundles.toml using the foloowing syntax: +The bundles are specified in build/bundles.toml using the following syntax: ```toml [[bundles]] version = X # actors version From 60bf39756906504472d5a6b2ace6601e11920a44 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 16 May 2022 21:39:22 +0300 Subject: [PATCH 07/19] fix typo Co-authored-by: raulk --- build/README-bundle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/README-bundle.md b/build/README-bundle.md index 397a80262..6a9d1c94c 100644 --- a/build/README-bundle.md +++ b/build/README-bundle.md @@ -16,7 +16,7 @@ If you don't want to fetch the bundle from github, you can specify an explicit p ```toml [[bundles]] version = X # actors version -release = tag # release gag +release = tag # release tag path = /path/to/builtin-actors.car ``` From eee73eed59ce49e15708849957b8005f3b3a5bb4 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 16 May 2022 21:44:31 +0300 Subject: [PATCH 08/19] use switch for release/path logic --- node/modules/builtin_actors.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/node/modules/builtin_actors.go b/node/modules/builtin_actors.go index 67cd9871d..a21820585 100644 --- a/node/modules/builtin_actors.go +++ b/node/modules/builtin_actors.go @@ -68,21 +68,26 @@ func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRe // we haven't recorded it in the datastore, so we need to load it var mfCid cid.Cid - if bd.Path != "" { + switch { + case bd.Path != "": // this is a local bundle, load it directly from the filessystem mfCid, err = bundle.LoadBundle(ctx, bs, bd.Path, av) if err != nil { return result, err } - } else { + + case bd.Release != "": // fetch it and add it to the blockstore mfCid, err = bundle.FetchAndLoadBundle(ctx, r.Path(), bs, av, bd.Release, netw) if err != nil { return result, err } + + default: + return result, xerrors.Errorf("no release or path specified for version %d bundle", av) } - if bd.Development { + if bd.Development || bd.Release == "" { // don't store the release key so that we always load development bundles continue } @@ -119,18 +124,23 @@ func LoadBuiltinActorsTesting(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtype defer testingBundleMx.Unlock() for av, bd := range build.BuiltinActorReleases { - if bd.Path != "" { + switch { + case bd.Path != "": // we need the appropriate bundle for tests; it should live next to the main bundle, with the // appropriate network name path := filepath.Join(filepath.Dir(bd.Path), fmt.Sprintf("builtin-actors-%s.car", netw)) if _, err := bundle.LoadBundle(ctx, bs, path, av); err != nil { return result, xerrors.Errorf("error loading testing bundle for builtin-actors version %d/%s: %w", av, netw, err) } - } else { + + case bd.Release != "": const basePath = "/tmp/lotus-testing" if _, err := bundle.FetchAndLoadBundle(ctx, basePath, bs, av, bd.Release, netw); err != nil { return result, xerrors.Errorf("error loading testing bundle for builtin-actors version %d/%s: %w", av, netw, err) } + + default: + return result, xerrors.Errorf("no path or release specified for version %d bundle", av) } } From 6360107d53c8d018f753877be995372ec03a2a99 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 16 May 2022 21:48:12 +0300 Subject: [PATCH 09/19] godocs for bundle struct --- build/bundle.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build/bundle.go b/build/bundle.go index 3afe41daa..4564e21e0 100644 --- a/build/bundle.go +++ b/build/bundle.go @@ -16,13 +16,13 @@ type BundleSpec struct { } type Bundle struct { - // actors version in this bundle + // Version is the actors version in this bundle Version actors.Version - // release id + // Release is the release id Release string - // bundle path: optional, uses the appropriate release bundle if unset + // Path is the (optional) bundle path; uses the appropriate release bundle if unset Path string - // development version; when set, in conjunction with path, it will always - // load the bundle to the blockstore + // Devlopment indicates whether this is a development version; when set, in conjunction with path, + // it will always load the bundle to the blockstore Development bool } From 31301e2b9d99b529e9addcee9cb536b08b897e97 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 16 May 2022 21:50:50 +0300 Subject: [PATCH 10/19] storage blah --- build/README-bundle.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/build/README-bundle.md b/build/README-bundle.md index 6a9d1c94c..16d8442da 100644 --- a/build/README-bundle.md +++ b/build/README-bundle.md @@ -29,3 +29,13 @@ release = tag # release gag path = /path/to/builtin-actors.car development = true ``` + +## Local Storage + +Bundles downloaded from github will be stored in +`$LOTUS_PATH/builtin-actors/vXXX/YYY/builtin-actors-ZZZ.car``, where +`XXX` is the actors version, `YYY` is the release tag, and `ZZZ` is +the network bundle name. + +The sha256 sum of the bundle will be stored next to it, in +`$LOTUS_PATH/builtin-actors/vXXX/YYY/builtin-actors-ZZZ.sha256` From 5c0c4d64e271799073baa15d985b6208c350154e Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 16 May 2022 21:52:34 +0300 Subject: [PATCH 11/19] more storage blah --- build/README-bundle.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build/README-bundle.md b/build/README-bundle.md index 16d8442da..7f39e8d7e 100644 --- a/build/README-bundle.md +++ b/build/README-bundle.md @@ -39,3 +39,9 @@ the network bundle name. The sha256 sum of the bundle will be stored next to it, in `$LOTUS_PATH/builtin-actors/vXXX/YYY/builtin-actors-ZZZ.sha256` + +On startup, if a bundle is recorded as loaded the manifest CID will be +checked for presence in the blockstore. If the manifest is missing, +then the bundle will be reloaded from the local file (if it exists) or +referched from github. The sha256 sum is always checked before +loading the bundle. From 314b4174462d441be9d48a50b35b52f644368386 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 16 May 2022 22:02:37 +0300 Subject: [PATCH 12/19] use network bundle names consistent with the network name --- Makefile | 4 ++-- build/bundle.go | 15 +++++++++++++++ node/bundle/manifest.go | 5 +---- node/modules/builtin_actors.go | 5 +---- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 2f729447d..141a90dda 100644 --- a/Makefile +++ b/Makefile @@ -75,7 +75,7 @@ debug: build-devnets 2k: build-devnets calibnet: GOFLAGS+=-tags=calibnet -calibnet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=calibrationnet +calibnet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=calibnet calibnet: build-devnets butterflynet: GOFLAGS+=-tags=butterflynet @@ -83,7 +83,7 @@ butterflynet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.Netwo butterflynet: build-devnets interopnet: GOFLAGS+=-tags=interopnet -interopnet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=caterpillarnet +interopnet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=interopnet interopnet: build-devnets lotus: $(BUILD_DEPS) diff --git a/build/bundle.go b/build/bundle.go index 4564e21e0..c0859da26 100644 --- a/build/bundle.go +++ b/build/bundle.go @@ -8,6 +8,21 @@ import ( var NetworkBundle string +func GetNetworkBundle() string { + switch NetworkBundle { + case "devnet": + return "devnet" + case "calibnet", "calibrationnet": + return "calibrationnet" + case "butterflynet": + return "butterflynet" + case "interopnet", "caterpillarnet": + return "caterpillarnet" + default: + return "mainnet" + } +} + //go:embed bundles.toml var BuiltinActorBundles []byte diff --git a/node/bundle/manifest.go b/node/bundle/manifest.go index 4bd55075e..a21b72ce3 100644 --- a/node/bundle/manifest.go +++ b/node/bundle/manifest.go @@ -57,10 +57,7 @@ func LoadBundle(ctx context.Context, bs blockstore.Blockstore, path string, av a // utility for blanket loading outside DI func FetchAndLoadBundles(ctx context.Context, bs blockstore.Blockstore, bar map[actors.Version]build.Bundle) error { - netw := build.NetworkBundle - if netw == "" { - netw = "mainnet" - } + netw := build.GetNetworkBundle() path := os.Getenv("LOTUS_PATH") if path == "" { diff --git a/node/modules/builtin_actors.go b/node/modules/builtin_actors.go index a21820585..388800926 100644 --- a/node/modules/builtin_actors.go +++ b/node/modules/builtin_actors.go @@ -25,10 +25,7 @@ func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRe // We can't put it as a dep in inputs causes a stack overflow in DI from circular dependency // So we pass it through ldflags instead - netw := build.NetworkBundle - if netw == "" { - netw = "mainnet" - } + netw := build.GetNetworkBundle() for av, bd := range build.BuiltinActorReleases { // first check to see if we know this release From 5981495740fd2ae780aa9801495b43983eca0633 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 16 May 2022 22:42:21 +0300 Subject: [PATCH 13/19] Update build/README-bundle.md Co-authored-by: Aayush Rajasekaran --- build/README-bundle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/README-bundle.md b/build/README-bundle.md index 7f39e8d7e..2e33df0c8 100644 --- a/build/README-bundle.md +++ b/build/README-bundle.md @@ -43,5 +43,5 @@ The sha256 sum of the bundle will be stored next to it, in On startup, if a bundle is recorded as loaded the manifest CID will be checked for presence in the blockstore. If the manifest is missing, then the bundle will be reloaded from the local file (if it exists) or -referched from github. The sha256 sum is always checked before +refetched from github. The sha256 sum is always checked before loading the bundle. From e9ece8eb3f992f3dda864788bbea3ec7cd19d089 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 16 May 2022 22:47:19 +0300 Subject: [PATCH 14/19] improve comment --- build/bundle.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/bundle.go b/build/bundle.go index 4564e21e0..ff2631bb0 100644 --- a/build/bundle.go +++ b/build/bundle.go @@ -23,6 +23,7 @@ type Bundle struct { // Path is the (optional) bundle path; uses the appropriate release bundle if unset Path string // Devlopment indicates whether this is a development version; when set, in conjunction with path, - // it will always load the bundle to the blockstore + // it will always load the bundle to the blockstore, without recording the manifest CID in the + // datastore. Development bool } From daf452a180ce2e216ff9e81c448bc32f4c8ac43d Mon Sep 17 00:00:00 2001 From: vyzo Date: Tue, 17 May 2022 20:38:32 +0300 Subject: [PATCH 15/19] flexible bundle loading --- build/README-bundle.md | 14 +++++++- build/bundle.go | 8 ++++- node/bundle/bundle.go | 62 +++++++++++++++++++++++++++++----- node/bundle/manifest.go | 42 ++++++++++++++++++++--- node/modules/builtin_actors.go | 26 ++++++++++++-- 5 files changed, 133 insertions(+), 19 deletions(-) diff --git a/build/README-bundle.md b/build/README-bundle.md index 2e33df0c8..2bbe652eb 100644 --- a/build/README-bundle.md +++ b/build/README-bundle.md @@ -6,7 +6,7 @@ The bundles are specified in build/bundles.toml using the following syntax: ```toml [[bundles]] version = X # actors version -release = tag # release gag +release = tag # release tag ``` This will add a bundle for version `X`, using the github release `tag` @@ -30,6 +30,18 @@ path = /path/to/builtin-actors.car development = true ``` +## Additional Options for Bundles + +- You can also specify a URL, together with a sha256 checksum to avoid downloading from + github. +- You can also specify an EnvVar, to provide the path dynamically at runtime. + +The precedence for bundle fetching/loading is as folllows: +- Check the EnvVar; use the bundle specified by it. +- Check the Path; use the bundle specified by it. +- Check the URL; use the bundle specified by it, and verify the checksum which must be present. +- Otherwise, use the release tag and download from github. + ## Local Storage Bundles downloaded from github will be stored in diff --git a/build/bundle.go b/build/bundle.go index ff2631bb0..e404abf09 100644 --- a/build/bundle.go +++ b/build/bundle.go @@ -20,8 +20,14 @@ type Bundle struct { Version actors.Version // Release is the release id Release string - // Path is the (optional) bundle path; uses the appropriate release bundle if unset + // EnvVar is the (optional) env var specifying the bundle path; takes precdence over path + EnvVar string + // Path is the (optional) bundle path; takes precedence over url Path string + // URL is the (optional) bundle URL; takes precdence over github release + URL string + // CHecksum is the bundle sha256 checksume in hex, when specifying a URL. + Checksum string // Devlopment indicates whether this is a development version; when set, in conjunction with path, // it will always load the bundle to the blockstore, without recording the manifest CID in the // datastore. diff --git a/node/bundle/bundle.go b/node/bundle/bundle.go index 99f214383..a3a14f468 100644 --- a/node/bundle/bundle.go +++ b/node/bundle/bundle.go @@ -33,7 +33,7 @@ func NewBundleFetcher(basepath string) (*BundleFetcher, error) { return &BundleFetcher{path: path}, nil } -func (b *BundleFetcher) Fetch(version int, release, netw string) (path string, err error) { +func (b *BundleFetcher) FetchFromRelease(version int, release, netw string) (path string, err error) { bundleName := fmt.Sprintf("builtin-actors-%s", netw) bundleFile := fmt.Sprintf("%s.car", bundleName) bundleHash := fmt.Sprintf("%s.sha256", bundleName) @@ -46,7 +46,7 @@ func (b *BundleFetcher) Fetch(version int, release, netw string) (path string, e // check if it exists; if it does, check the hash bundleFilePath := filepath.Join(bundleBasePath, bundleFile) if _, err := os.Stat(bundleFilePath); err == nil { - err := b.check(bundleBasePath, bundleFile, bundleHash) + err := b.checkRelease(bundleBasePath, bundleFile, bundleHash) if err == nil { return bundleFilePath, nil } @@ -55,12 +55,46 @@ func (b *BundleFetcher) Fetch(version int, release, netw string) (path string, e } log.Infof("fetching bundle %s", bundleFile) - if err := b.fetch(release, bundleBasePath, bundleFile, bundleHash); err != nil { + if err := b.fetchFromRelease(release, bundleBasePath, bundleFile, bundleHash); err != nil { log.Errorf("error fetching bundle %s: %s", bundleName, err) return "", xerrors.Errorf("error fetching bundle: %w", err) } - if err := b.check(bundleBasePath, bundleFile, bundleHash); err != nil { + if err := b.checkRelease(bundleBasePath, bundleFile, bundleHash); err != nil { + log.Errorf("error checking bundle %s: %s", bundleName, err) + return "", xerrors.Errorf("error checking bundle: %s", err) + } + + return bundleFilePath, nil +} + +func (b *BundleFetcher) FetchFromURL(version int, release, netw, url, cksum string) (path string, err error) { + bundleName := fmt.Sprintf("builtin-actors-%s", netw) + bundleFile := fmt.Sprintf("%s.car", bundleName) + bundleBasePath := filepath.Join(b.path, fmt.Sprintf("v%d", version), release) + + if err := os.MkdirAll(bundleBasePath, 0755); err != nil { + return "", xerrors.Errorf("error making bundle directory %s: %w", bundleBasePath, err) + } + + // check if it exists; if it does, check the hash + bundleFilePath := filepath.Join(bundleBasePath, bundleFile) + if _, err := os.Stat(bundleFilePath); err == nil { + err := b.checkHash(bundleBasePath, bundleFile, cksum) + if err == nil { + return bundleFilePath, nil + } + + log.Warnf("invalid bundle %s: %s; refetching", bundleName, err) + } + + log.Infof("fetching bundle %s", bundleFile) + if err := b.fetchFromURL(bundleBasePath, bundleFile, url); err != nil { + log.Errorf("error fetching bundle %s: %s", bundleName, err) + return "", xerrors.Errorf("error fetching bundle: %w", err) + } + + if err := b.checkHash(bundleBasePath, bundleFile, cksum); err != nil { log.Errorf("error checking bundle %s: %s", bundleName, err) return "", xerrors.Errorf("error checking bundle: %s", err) } @@ -105,7 +139,7 @@ func (b *BundleFetcher) fetchURL(url, path string) error { return xerrors.Errorf("all attempts to fetch %s failed", url) } -func (b *BundleFetcher) fetch(release, bundleBasePath, bundleFile, bundleHash string) error { +func (b *BundleFetcher) fetchFromRelease(release, bundleBasePath, bundleFile, bundleHash string) error { bundleHashUrl := fmt.Sprintf("https://github.com/filecoin-project/builtin-actors/releases/download/%s/%s", release, bundleHash) bundleHashPath := filepath.Join(bundleBasePath, bundleHash) @@ -123,7 +157,12 @@ func (b *BundleFetcher) fetch(release, bundleBasePath, bundleFile, bundleHash st return nil } -func (b *BundleFetcher) check(bundleBasePath, bundleFile, bundleHash string) error { +func (b *BundleFetcher) fetchFromURL(bundleBasePath, bundleFile, url string) error { + bundleFilePath := filepath.Join(bundleBasePath, bundleFile) + return b.fetchURL(url, bundleFilePath) +} + +func (b *BundleFetcher) checkRelease(bundleBasePath, bundleFile, bundleHash string) error { bundleHashPath := filepath.Join(bundleBasePath, bundleHash) f, err := os.Open(bundleHashPath) if err != nil { @@ -138,13 +177,18 @@ func (b *BundleFetcher) check(bundleBasePath, bundleFile, bundleHash string) err parts := strings.Split(string(bs), " ") hashHex := parts[0] - expectedDigest, err := hex.DecodeString(hashHex) + + return b.checkHash(bundleBasePath, bundleFile, hashHex) +} + +func (b *BundleFetcher) checkHash(bundleBasePath, bundleFile, cksum string) error { + expectedDigest, err := hex.DecodeString(cksum) if err != nil { - return xerrors.Errorf("error decoding digest from %s: %w", bundleHashPath, err) + return xerrors.Errorf("error decoding digest from %s: %w", cksum, err) } bundleFilePath := filepath.Join(bundleBasePath, bundleFile) - f, err = os.Open(bundleFilePath) + f, err := os.Open(bundleFilePath) if err != nil { return xerrors.Errorf("error opening %s: %w", bundleFilePath, err) } diff --git a/node/bundle/manifest.go b/node/bundle/manifest.go index 4bd55075e..0e4819ce2 100644 --- a/node/bundle/manifest.go +++ b/node/bundle/manifest.go @@ -17,13 +17,27 @@ import ( "github.com/mitchellh/go-homedir" ) -func FetchAndLoadBundle(ctx context.Context, basePath string, bs blockstore.Blockstore, av actors.Version, rel, netw string) (cid.Cid, error) { +func FetchAndLoadBundleFromRelease(ctx context.Context, basePath string, bs blockstore.Blockstore, av actors.Version, rel, netw string) (cid.Cid, error) { fetcher, err := NewBundleFetcher(basePath) if err != nil { return cid.Undef, xerrors.Errorf("error creating fetcher for builtin-actors version %d: %w", av, err) } - path, err := fetcher.Fetch(int(av), rel, netw) + path, err := fetcher.FetchFromRelease(int(av), rel, netw) + if err != nil { + return cid.Undef, xerrors.Errorf("error fetching bundle for builtin-actors version %d: %w", av, err) + } + + return LoadBundle(ctx, bs, path, av) +} + +func FetchAndLoadBundleFromURL(ctx context.Context, basePath string, bs blockstore.Blockstore, av actors.Version, rel, netw, url, cksum string) (cid.Cid, error) { + fetcher, err := NewBundleFetcher(basePath) + if err != nil { + return cid.Undef, xerrors.Errorf("error creating fetcher for builtin-actors version %d: %w", av, err) + } + + path, err := fetcher.FetchFromURL(int(av), rel, netw, url, cksum) if err != nil { return cid.Undef, xerrors.Errorf("error fetching bundle for builtin-actors version %d: %w", av, err) } @@ -72,12 +86,30 @@ func FetchAndLoadBundles(ctx context.Context, bs blockstore.Blockstore, bar map[ } for av, bd := range bar { - if bd.Path != "" { + switch { + case bd.EnvVar != "": + // this is a local bundle, specified by an env var to load from the filesystem + path := os.Getenv(bd.EnvVar) + if path == "" { + return xerrors.Errorf("bundle envvar is empty: %s", bd.EnvVar) + } + + if _, err := LoadBundle(ctx, bs, path, av); err != nil { + return err + } + + case bd.Path != "": if _, err := LoadBundle(ctx, bs, bd.Path, av); err != nil { return err } - } else { - if _, err := FetchAndLoadBundle(ctx, path, bs, av, bd.Release, netw); err != nil { + + case bd.URL != "": + if _, err := FetchAndLoadBundleFromURL(ctx, path, bs, av, bd.Release, netw, bd.URL, bd.Checksum); err != nil { + return err + } + + case bd.Release != "": + if _, err := FetchAndLoadBundleFromRelease(ctx, path, bs, av, bd.Release, netw); err != nil { return err } } diff --git a/node/modules/builtin_actors.go b/node/modules/builtin_actors.go index a21820585..c7cacaa56 100644 --- a/node/modules/builtin_actors.go +++ b/node/modules/builtin_actors.go @@ -2,6 +2,7 @@ package modules import ( "fmt" + "os" "path/filepath" "sync" @@ -69,6 +70,18 @@ func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRe // we haven't recorded it in the datastore, so we need to load it var mfCid cid.Cid switch { + case bd.EnvVar != "": + // this is a local bundle, specified by an env var to load from the filesystem + path := os.Getenv(bd.EnvVar) + if path == "" { + return result, xerrors.Errorf("bundle envvar is empty: %s", bd.EnvVar) + } + + mfCid, err = bundle.LoadBundle(ctx, bs, path, av) + if err != nil { + return result, err + } + case bd.Path != "": // this is a local bundle, load it directly from the filessystem mfCid, err = bundle.LoadBundle(ctx, bs, bd.Path, av) @@ -76,9 +89,16 @@ func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRe return result, err } + case bd.URL != "": + // fetch it from the specified URL + mfCid, err = bundle.FetchAndLoadBundleFromURL(ctx, r.Path(), bs, av, bd.Release, netw, bd.URL, bd.Checksum) + if err != nil { + return result, err + } + case bd.Release != "": // fetch it and add it to the blockstore - mfCid, err = bundle.FetchAndLoadBundle(ctx, r.Path(), bs, av, bd.Release, netw) + mfCid, err = bundle.FetchAndLoadBundleFromRelease(ctx, r.Path(), bs, av, bd.Release, netw) if err != nil { return result, err } @@ -135,12 +155,12 @@ func LoadBuiltinActorsTesting(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtype case bd.Release != "": const basePath = "/tmp/lotus-testing" - if _, err := bundle.FetchAndLoadBundle(ctx, basePath, bs, av, bd.Release, netw); err != nil { + if _, err := bundle.FetchAndLoadBundleFromRelease(ctx, basePath, bs, av, bd.Release, netw); err != nil { return result, xerrors.Errorf("error loading testing bundle for builtin-actors version %d/%s: %w", av, netw, err) } default: - return result, xerrors.Errorf("no path or release specified for version %d bundle", av) + return result, xerrors.Errorf("no path or release specified for version %d testing bundle", av) } } From d949b6c8efaef71bf0bb32293de81178de13ccf1 Mon Sep 17 00:00:00 2001 From: vyzo Date: Tue, 17 May 2022 21:14:49 +0300 Subject: [PATCH 16/19] fix envvar handling --- build/README-bundle.md | 4 ++-- build/bundle.go | 2 -- node/bundle/manifest.go | 9 ++++----- node/modules/builtin_actors.go | 9 +++------ 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/build/README-bundle.md b/build/README-bundle.md index 2bbe652eb..4dc96c687 100644 --- a/build/README-bundle.md +++ b/build/README-bundle.md @@ -34,10 +34,10 @@ development = true - You can also specify a URL, together with a sha256 checksum to avoid downloading from github. -- You can also specify an EnvVar, to provide the path dynamically at runtime. +- You can also specify an environment variable (`LOTUS_BUILTIN_ACTORS_VX_BUNDLE`), to provide the path dynamically at runtime. The precedence for bundle fetching/loading is as folllows: -- Check the EnvVar; use the bundle specified by it. +- Check the environment variable `LOTUS_BUILTIN_ACTORS_VX_BUNDLE` for version X bundle; use it if set. - Check the Path; use the bundle specified by it. - Check the URL; use the bundle specified by it, and verify the checksum which must be present. - Otherwise, use the release tag and download from github. diff --git a/build/bundle.go b/build/bundle.go index e404abf09..c1c5df611 100644 --- a/build/bundle.go +++ b/build/bundle.go @@ -20,8 +20,6 @@ type Bundle struct { Version actors.Version // Release is the release id Release string - // EnvVar is the (optional) env var specifying the bundle path; takes precdence over path - EnvVar string // Path is the (optional) bundle path; takes precedence over url Path string // URL is the (optional) bundle URL; takes precdence over github release diff --git a/node/bundle/manifest.go b/node/bundle/manifest.go index 0e4819ce2..c1a5b0953 100644 --- a/node/bundle/manifest.go +++ b/node/bundle/manifest.go @@ -2,6 +2,7 @@ package bundle import ( "context" + "fmt" "io" "os" @@ -86,13 +87,11 @@ func FetchAndLoadBundles(ctx context.Context, bs blockstore.Blockstore, bar map[ } for av, bd := range bar { + envvar := fmt.Sprintf("LOGUS_BUILTIN_ACTORS_V%d_BUNDLE", av) switch { - case bd.EnvVar != "": + case os.Getenv(envvar) != "": // this is a local bundle, specified by an env var to load from the filesystem - path := os.Getenv(bd.EnvVar) - if path == "" { - return xerrors.Errorf("bundle envvar is empty: %s", bd.EnvVar) - } + path := os.Getenv(envvar) if _, err := LoadBundle(ctx, bs, path, av); err != nil { return err diff --git a/node/modules/builtin_actors.go b/node/modules/builtin_actors.go index c7cacaa56..3697747d8 100644 --- a/node/modules/builtin_actors.go +++ b/node/modules/builtin_actors.go @@ -68,14 +68,11 @@ func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRe } // we haven't recorded it in the datastore, so we need to load it + envvar := fmt.Sprintf("LOGUS_BUILTIN_ACTORS_V%d_BUNDLE", av) var mfCid cid.Cid switch { - case bd.EnvVar != "": - // this is a local bundle, specified by an env var to load from the filesystem - path := os.Getenv(bd.EnvVar) - if path == "" { - return result, xerrors.Errorf("bundle envvar is empty: %s", bd.EnvVar) - } + case os.Getenv(envvar) != "": + path := os.Getenv(envvar) mfCid, err = bundle.LoadBundle(ctx, bs, path, av) if err != nil { From ccb2e44e36c289bdb316f4321f0b41d9de904e48 Mon Sep 17 00:00:00 2001 From: vyzo Date: Tue, 17 May 2022 21:45:17 +0300 Subject: [PATCH 17/19] make path/url be a map of network name to car uri --- build/bundle.go | 13 +++++++++---- node/bundle/manifest.go | 8 ++++---- node/modules/builtin_actors.go | 24 +++++++++++++----------- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/build/bundle.go b/build/bundle.go index c1c5df611..548ded286 100644 --- a/build/bundle.go +++ b/build/bundle.go @@ -21,13 +21,18 @@ type Bundle struct { // Release is the release id Release string // Path is the (optional) bundle path; takes precedence over url - Path string + Path map[string]string // URL is the (optional) bundle URL; takes precdence over github release - URL string - // CHecksum is the bundle sha256 checksume in hex, when specifying a URL. - Checksum string + URL map[string]BundleURL // Devlopment indicates whether this is a development version; when set, in conjunction with path, // it will always load the bundle to the blockstore, without recording the manifest CID in the // datastore. Development bool } + +type BundleURL struct { + // URL is the url of the bundle + URL string + // Checksum is the sha256 checksum of the bundle + Checksum string +} diff --git a/node/bundle/manifest.go b/node/bundle/manifest.go index c1a5b0953..b35a70f20 100644 --- a/node/bundle/manifest.go +++ b/node/bundle/manifest.go @@ -97,13 +97,13 @@ func FetchAndLoadBundles(ctx context.Context, bs blockstore.Blockstore, bar map[ return err } - case bd.Path != "": - if _, err := LoadBundle(ctx, bs, bd.Path, av); err != nil { + case bd.Path[netw] != "": + if _, err := LoadBundle(ctx, bs, bd.Path[netw], av); err != nil { return err } - case bd.URL != "": - if _, err := FetchAndLoadBundleFromURL(ctx, path, bs, av, bd.Release, netw, bd.URL, bd.Checksum); err != nil { + case bd.URL[netw].URL != "": + if _, err := FetchAndLoadBundleFromURL(ctx, path, bs, av, bd.Release, netw, bd.URL[netw].URL, bd.URL[netw].Checksum); err != nil { return err } diff --git a/node/modules/builtin_actors.go b/node/modules/builtin_actors.go index 3697747d8..58ae035d9 100644 --- a/node/modules/builtin_actors.go +++ b/node/modules/builtin_actors.go @@ -3,7 +3,6 @@ package modules import ( "fmt" "os" - "path/filepath" "sync" "go.uber.org/fx" @@ -79,16 +78,16 @@ func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRe return result, err } - case bd.Path != "": + case bd.Path[netw] != "": // this is a local bundle, load it directly from the filessystem - mfCid, err = bundle.LoadBundle(ctx, bs, bd.Path, av) + mfCid, err = bundle.LoadBundle(ctx, bs, bd.Path[netw], av) if err != nil { return result, err } - case bd.URL != "": + case bd.URL[netw].URL != "": // fetch it from the specified URL - mfCid, err = bundle.FetchAndLoadBundleFromURL(ctx, r.Path(), bs, av, bd.Release, netw, bd.URL, bd.Checksum) + mfCid, err = bundle.FetchAndLoadBundleFromURL(ctx, r.Path(), bs, av, bd.Release, netw, bd.URL[netw].URL, bd.URL[netw].Checksum) if err != nil { return result, err } @@ -140,18 +139,21 @@ func LoadBuiltinActorsTesting(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtype testingBundleMx.Lock() defer testingBundleMx.Unlock() + const basePath = "/tmp/lotus-testing" for av, bd := range build.BuiltinActorReleases { switch { - case bd.Path != "": - // we need the appropriate bundle for tests; it should live next to the main bundle, with the - // appropriate network name - path := filepath.Join(filepath.Dir(bd.Path), fmt.Sprintf("builtin-actors-%s.car", netw)) - if _, err := bundle.LoadBundle(ctx, bs, path, av); err != nil { + case bd.Path[netw] != "": + if _, err := bundle.LoadBundle(ctx, bs, bd.Path[netw], av); err != nil { return result, xerrors.Errorf("error loading testing bundle for builtin-actors version %d/%s: %w", av, netw, err) } + case bd.URL[netw].URL != "": + // fetch it from the specified URL + if _, err := bundle.FetchAndLoadBundleFromURL(ctx, basePath, bs, av, bd.Release, netw, bd.URL[netw].URL, bd.URL[netw].Checksum); err != nil { + return result, err + } + case bd.Release != "": - const basePath = "/tmp/lotus-testing" if _, err := bundle.FetchAndLoadBundleFromRelease(ctx, basePath, bs, av, bd.Release, netw); err != nil { return result, xerrors.Errorf("error loading testing bundle for builtin-actors version %d/%s: %w", av, netw, err) } From 31c846011677ee65f431f38073c6fa14cb8dedee Mon Sep 17 00:00:00 2001 From: vyzo Date: Tue, 17 May 2022 21:47:01 +0300 Subject: [PATCH 18/19] fix typo --- node/bundle/manifest.go | 2 +- node/modules/builtin_actors.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/node/bundle/manifest.go b/node/bundle/manifest.go index b35a70f20..d874d743d 100644 --- a/node/bundle/manifest.go +++ b/node/bundle/manifest.go @@ -87,7 +87,7 @@ func FetchAndLoadBundles(ctx context.Context, bs blockstore.Blockstore, bar map[ } for av, bd := range bar { - envvar := fmt.Sprintf("LOGUS_BUILTIN_ACTORS_V%d_BUNDLE", av) + envvar := fmt.Sprintf("LOTUS_BUILTIN_ACTORS_V%d_BUNDLE", av) switch { case os.Getenv(envvar) != "": // this is a local bundle, specified by an env var to load from the filesystem diff --git a/node/modules/builtin_actors.go b/node/modules/builtin_actors.go index 58ae035d9..407e3fc3e 100644 --- a/node/modules/builtin_actors.go +++ b/node/modules/builtin_actors.go @@ -67,7 +67,7 @@ func LoadBuiltinActors(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRe } // we haven't recorded it in the datastore, so we need to load it - envvar := fmt.Sprintf("LOGUS_BUILTIN_ACTORS_V%d_BUNDLE", av) + envvar := fmt.Sprintf("LOTUS_BUILTIN_ACTORS_V%d_BUNDLE", av) var mfCid cid.Cid switch { case os.Getenv(envvar) != "": From dbbbab4f8a0fe881e3616baac433f8112f248cad Mon Sep 17 00:00:00 2001 From: vyzo Date: Tue, 17 May 2022 22:01:38 +0300 Subject: [PATCH 19/19] use dev bundle release --- build/bundles.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/bundles.toml b/build/bundles.toml index a3ef18414..347b81149 100644 --- a/build/bundles.toml +++ b/build/bundles.toml @@ -1,3 +1,3 @@ [[bundles]] version = 8 -release = "b71c2ec785aec23d" +release = "dev/20220517"