Merge pull request #11203 from filecoin-project/feat/lotus-provider

Feat/lotus provider

.gitignore

@@ -6,6 +6,7 @@
 /lotus-chainwatch
 /lotus-shed
 /lotus-sim
+/lotus-provider
 /lotus-townhall
 /lotus-fountain
 /lotus-stats

Dockerfile

@@ -109,6 +109,7 @@ COPY --from=lotus-builder /opt/filecoin/lotus-wallet   /usr/local/bin/
 COPY --from=lotus-builder /opt/filecoin/lotus-gateway  /usr/local/bin/
 COPY --from=lotus-builder /opt/filecoin/lotus-miner    /usr/local/bin/
 COPY --from=lotus-builder /opt/filecoin/lotus-worker   /usr/local/bin/
+COPY --from=lotus-builder /opt/filecoin/lotus-provider    /usr/local/bin/
 COPY --from=lotus-builder /opt/filecoin/lotus-stats    /usr/local/bin/
 COPY --from=lotus-builder /opt/filecoin/lotus-fountain /usr/local/bin/
 
@@ -117,11 +118,13 @@ RUN mkdir /var/lib/lotus
 RUN mkdir /var/lib/lotus-miner
 RUN mkdir /var/lib/lotus-worker
 RUN mkdir /var/lib/lotus-wallet
+RUN mkdir /var/lib/lotus-provider
 RUN chown fc: /var/tmp/filecoin-proof-parameters
 RUN chown fc: /var/lib/lotus
 RUN chown fc: /var/lib/lotus-miner
 RUN chown fc: /var/lib/lotus-worker
 RUN chown fc: /var/lib/lotus-wallet
+RUN chown fc: /var/lib/lotus-provider
 
 
 VOLUME /var/tmp/filecoin-proof-parameters
@@ -129,6 +132,7 @@ VOLUME /var/lib/lotus
 VOLUME /var/lib/lotus-miner
 VOLUME /var/lib/lotus-worker
 VOLUME /var/lib/lotus-wallet
+VOLUME /var/lib/lotus-provider
 
 EXPOSE 1234
 EXPOSE 2345

Makefile

@@ -97,6 +97,10 @@ lotus-miner: $(BUILD_DEPS)
 .PHONY: lotus-miner
 BINS+=lotus-miner
 
+lotus-provider: $(BUILD_DEPS)
+	rm -f lotus-provider
+	$(GOCC) build $(GOFLAGS) -o lotus-provider ./cmd/lotus-provider
+
 lotus-worker: $(BUILD_DEPS)
 	rm -f lotus-worker
 	$(GOCC) build $(GOFLAGS) -o lotus-worker ./cmd/lotus-worker
@@ -129,6 +133,9 @@ install-daemon:
 install-miner:
 	install -C ./lotus-miner /usr/local/bin/lotus-miner
 
+install-provider:
+	install -C ./lotus-provider /usr/local/bin/lotus-provider
+
 install-worker:
 	install -C ./lotus-worker /usr/local/bin/lotus-worker
 
@@ -144,6 +151,9 @@ uninstall-daemon:
 uninstall-miner:
 	rm -f /usr/local/bin/lotus-miner
 
+uninstall-provider:
+	rm -f /usr/local/bin/lotus-provider
+
 uninstall-worker:
 	rm -f /usr/local/bin/lotus-worker
 
@@ -241,6 +251,14 @@ install-miner-service: install-miner install-daemon-service
 	@echo
 	@echo "lotus-miner service installed. Don't forget to run 'sudo systemctl start lotus-miner' to start it and 'sudo systemctl enable lotus-miner' for it to be enabled on startup."
 
+install-provider-service: install-provider install-daemon-service
+	mkdir -p /etc/systemd/system
+	mkdir -p /var/log/lotus
+	install -C -m 0644 ./scripts/lotus-provider.service /etc/systemd/system/lotus-provider.service
+	systemctl daemon-reload
+	@echo
+	@echo "lotus-provider service installed. Don't forget to run 'sudo systemctl start lotus-provider' to start it and 'sudo systemctl enable lotus-provider' for it to be enabled on startup."
+
 install-main-services: install-miner-service
 
 install-all-services: install-main-services
@@ -259,6 +277,12 @@ clean-miner-service:
 	rm -f /etc/systemd/system/lotus-miner.service
 	systemctl daemon-reload
 
+clean-provider-service:
+	-systemctl stop lotus-provider
+	-systemctl disable lotus-provider
+	rm -f /etc/systemd/system/lotus-provider.service
+	systemctl daemon-reload
+
 clean-main-services: clean-daemon-service
 
 clean-all-services: clean-main-services
@@ -360,15 +384,16 @@ gen: actors-code-gen type-gen cfgdoc-gen docsgen api-gen circleci fiximports
 
 jen: gen
 
-snap: lotus lotus-miner lotus-worker
+snap: lotus lotus-miner lotus-worker lotus-provider
 	snapcraft
 	# snapcraft upload ./lotus_*.snap
 
 # separate from gen because it needs binaries
-docsgen-cli: lotus lotus-miner lotus-worker
+docsgen-cli: lotus lotus-miner lotus-worker lotus-provider
 	python3 ./scripts/generate-lotus-cli.py
 	./lotus config default > documentation/en/default-lotus-config.toml
 	./lotus-miner config default > documentation/en/default-lotus-miner-config.toml
+	./lotus-provider config default > documentation/en/default-lotus-provider-config.toml
 .PHONY: docsgen-cli
 
 print-%:

cli/helper.go

@@ -1,6 +1,7 @@
 package cli
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -8,7 +9,6 @@ import (
 	"syscall"
 
 	ufcli "github.com/urfave/cli/v2"
-	"golang.org/x/xerrors"
 )
 
 type PrintHelpErr struct {
@@ -52,7 +52,7 @@ func RunApp(app *ufcli.App) {
 			fmt.Fprintf(os.Stderr, "ERROR: %s\n\n", err) // nolint:errcheck
 		}
 		var phe *PrintHelpErr
-		if xerrors.As(err, &phe) {
+		if errors.As(err, &phe) {
 			_ = ufcli.ShowCommandHelp(phe.Ctx, phe.Ctx.Command.Name)
 		}
 		os.Exit(1)

cmd/lotus-provider/config.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+)
+
+var configCmd = &cli.Command{
+	Name:  "config",
+	Usage: "Manage node config",
+	Subcommands: []*cli.Command{
+		configDefaultCmd,
+		configSetCmd,
+		configGetCmd,
+	},
+}
+
+var configDefaultCmd = &cli.Command{
+	Name:  "default",
+	Usage: "Print default system config",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "no-comment",
+			Usage: "don't comment default values",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		fmt.Println("[config]\nstatus = Coming Soon")
+		// [overlay.sealer1.tasks]\nsealer_task_enable = true
+		return nil
+	},
+}
+
+var configSetCmd = &cli.Command{
+	Name:  "set",
+	Usage: "Set all config",
+	Action: func(cctx *cli.Context) error {
+		fmt.Println("Coming soon")
+		return nil
+	},
+}
+
+var configGetCmd = &cli.Command{
+	Name:  "get",
+	Usage: "Get all config",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "no-comment",
+			Usage: "don't comment default values",
+		},
+		&cli.BoolFlag{
+			Name:  "no-doc",
+			Usage: "don't add value documentation",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		fmt.Println("Coming soon")
+		return nil
+	},
+}

cmd/lotus-provider/main.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"context"
+
+	"github.com/fatih/color"
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/urfave/cli/v2"
+
+	"github.com/filecoin-project/lotus/build"
+	lcli "github.com/filecoin-project/lotus/cli"
+	cliutil "github.com/filecoin-project/lotus/cli/util"
+	"github.com/filecoin-project/lotus/lib/lotuslog"
+	"github.com/filecoin-project/lotus/lib/tracing"
+	"github.com/filecoin-project/lotus/node/repo"
+)
+
+var log = logging.Logger("main")
+
+func main() {
+
+	lotuslog.SetupLogLevels()
+
+	local := []*cli.Command{
+		//initCmd,
+		runCmd,
+		stopCmd,
+		configCmd,
+		//backupCmd,
+		//lcli.WithCategory("chain", actorCmd),
+		//lcli.WithCategory("storage", sectorsCmd),
+		//lcli.WithCategory("storage", provingCmd),
+		//lcli.WithCategory("storage", storageCmd),
+		//lcli.WithCategory("storage", sealingCmd),
+	}
+
+	jaeger := tracing.SetupJaegerTracing("lotus")
+	defer func() {
+		if jaeger != nil {
+			_ = jaeger.ForceFlush(context.Background())
+		}
+	}()
+
+	for _, cmd := range local {
+		cmd := cmd
+		originBefore := cmd.Before
+		cmd.Before = func(cctx *cli.Context) error {
+			if jaeger != nil {
+				_ = jaeger.Shutdown(cctx.Context)
+			}
+			jaeger = tracing.SetupJaegerTracing("lotus/" + cmd.Name)
+
+			if cctx.IsSet("color") {
+				color.NoColor = !cctx.Bool("color")
+			}
+
+			if originBefore != nil {
+				return originBefore(cctx)
+			}
+
+			return nil
+		}
+	}
+
+	app := &cli.App{
+		Name:                 "lotus-provider",
+		Usage:                "Filecoin decentralized storage network provider",
+		Version:              build.UserVersion(),
+		EnableBashCompletion: true,
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:    "listen",
+				Usage:   "host address and port the worker api will listen on",
+				Value:   "0.0.0.0:3456",
+				EnvVars: []string{"LOTUS_WORKER_LISTEN"},
+			},
+			&cli.BoolFlag{
+				// examined in the Before above
+				Name:        "color",
+				Usage:       "use color in display output",
+				DefaultText: "depends on output being a TTY",
+			},
+			&cli.StringFlag{
+				Name:    "panic-reports",
+				EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"},
+				Hidden:  true,
+				Value:   "~/.lotusprovider", // should follow --repo default
+			},
+			&cli.StringFlag{
+				Name:    "repo",
+				EnvVars: []string{"LOTUS_PATH"},
+				Hidden:  true,
+				Value:   "~/.lotus",
+			},
+			cliutil.FlagVeryVerbose,
+		},
+		Commands: append(local, lcli.CommonCommands...),
+		Before: func(c *cli.Context) error {
+			return nil
+		},
+		After: func(c *cli.Context) error {
+			if r := recover(); r != nil {
+				// Generate report in LOTUS_PATH and re-raise panic
+				build.GeneratePanicReport(c.String("panic-reports"), c.String(FlagProviderRepo), c.App.Name)
+				panic(r)
+			}
+			return nil
+		},
+	}
+	app.Setup()
+	app.Metadata["repoType"] = repo.Provider
+	lcli.RunApp(app)
+}
+
+const (
+	FlagProviderRepo = "provider-repo"
+)

cmd/lotus-provider/run.go

@@ -0,0 +1,248 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/gin-contrib/pprof"
+	"github.com/gin-gonic/gin"
+	"github.com/urfave/cli/v2"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/tag"
+
+	"github.com/filecoin-project/go-jsonrpc/auth"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/build"
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
+	"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
+	"github.com/filecoin-project/lotus/lib/ulimit"
+	"github.com/filecoin-project/lotus/metrics"
+	"github.com/filecoin-project/lotus/node"
+	"github.com/filecoin-project/lotus/node/config"
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
+	"github.com/filecoin-project/lotus/node/repo"
+	"github.com/filecoin-project/lotus/storage/paths"
+	"github.com/filecoin-project/lotus/storage/sealer/storiface"
+)
+
+var runCmd = &cli.Command{
+	Name:  "run",
+	Usage: "Start a lotus provider process",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "provider-api",
+			Usage: "Port (default 12300)",
+		},
+		&cli.BoolFlag{
+			Name:  "nosync",
+			Usage: "don't check full-node sync status",
+		},
+		&cli.BoolFlag{
+			Name:   "halt-after-init",
+			Usage:  "only run init, then return",
+			Hidden: true,
+		},
+		&cli.BoolFlag{
+			Name:  "manage-fdlimit",
+			Usage: "manage open file limit",
+			Value: true,
+		},
+		&cli.StringFlag{
+			Name:    "db-host",
+			EnvVars: []string{"LOTUS_DB_HOST"},
+			Usage:   "Command separated list of hostnames for yugabyte cluster",
+			Value:   "yugabyte",
+		},
+		&cli.StringFlag{
+			Name:    "db-name",
+			EnvVars: []string{"LOTUS_DB_NAME"},
+			Value:   "yugabyte",
+		},
+		&cli.StringFlag{
+			Name:    "db-user",
+			EnvVars: []string{"LOTUS_DB_USER"},
+			Value:   "yugabyte",
+		},
+		&cli.StringFlag{
+			Name:    "db-password",
+			EnvVars: []string{"LOTUS_DB_PASSWORD"},
+			Value:   "yugabyte",
+		},
+		&cli.StringFlag{
+			Name:    "db-port",
+			EnvVars: []string{"LOTUS_DB_PORT"},
+			Hidden:  true,
+			Value:   "5433",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Bool("enable-gpu-proving") {
+			err := os.Setenv("BELLMAN_NO_GPU", "true")
+			if err != nil {
+				return err
+			}
+		}
+
+		ctx, _ := tag.New(lcli.DaemonContext(cctx),
+			tag.Insert(metrics.Version, build.BuildVersion),
+			tag.Insert(metrics.Commit, build.CurrentCommit),
+			tag.Insert(metrics.NodeType, "provider"),
+		)
+		// Register all metric views
+		/*
+			if err := view.Register(
+				metrics.MinerNodeViews...,
+			); err != nil {
+				log.Fatalf("Cannot register the view: %v", err)
+			}
+		*/
+		// Set the metric to one so it is published to the exporter
+		stats.Record(ctx, metrics.LotusInfo.M(1))
+
+		if cctx.Bool("manage-fdlimit") {
+			if _, _, err := ulimit.ManageFdLimit(); err != nil {
+				log.Errorf("setting file descriptor limit: %s", err)
+			}
+		}
+
+		// Open repo
+
+		repoPath := cctx.String(FlagProviderRepo)
+		r, err := repo.NewFS(repoPath)
+		if err != nil {
+			return err
+		}
+
+		ok, err := r.Exists()
+		if err != nil {
+			return err
+		}
+		if !ok {
+			if err := r.Init(repo.Provider); err != nil {
+				return err
+			}
+
+			lr, err := r.Lock(repo.Provider)
+			if err != nil {
+				return err
+			}
+
+			var localPaths []storiface.LocalPath
+
+			if err := lr.SetStorage(func(sc *storiface.StorageConfig) {
+				sc.StoragePaths = append(sc.StoragePaths, localPaths...)
+			}); err != nil {
+				return fmt.Errorf("set storage config: %w", err)
+			}
+
+			{
+				// init datastore for r.Exists
+				_, err := lr.Datastore(context.Background(), "/metadata")
+				if err != nil {
+					return err
+				}
+			}
+			if err := lr.Close(); err != nil {
+				return fmt.Errorf("close repo: %w", err)
+			}
+		}
+
+		lr, err := r.Lock(repo.Provider)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if err := lr.Close(); err != nil {
+				log.Error("closing repo", err)
+			}
+		}()
+
+		db, err := harmonydb.NewFromConfig(config.HarmonyDB{
+			Username: cctx.String("db_user"),
+			Password: cctx.String("db_password"),
+			Hosts:    strings.Split(cctx.String("db_host"), ","),
+			Database: cctx.String("db_name"),
+			Port:     cctx.String("db_port"),
+		})
+		if err != nil {
+			return err
+		}
+
+		shutdownChan := make(chan struct{})
+
+		stop, err := node.New(ctx,
+			node.Override(new(dtypes.ShutdownChan), shutdownChan),
+			node.Provider(r),
+		)
+		if err != nil {
+			return fmt.Errorf("creating node: %w", err)
+		}
+
+		const unspecifiedAddress = "0.0.0.0"
+		address := cctx.String("listen")
+		addressSlice := strings.Split(address, ":")
+		if ip := net.ParseIP(addressSlice[0]); ip != nil {
+			if ip.String() == unspecifiedAddress {
+				rip, err := db.GetRoutableIP()
+				if err != nil {
+					return err
+				}
+				address = rip + ":" + addressSlice[1]
+			}
+		}
+		localStore, err := paths.NewLocal(ctx, lr, nil, []string{"http://" + address + "/remote"})
+		if err != nil {
+			return err
+		}
+
+		taskEngine, err := harmonytask.New(db, []harmonytask.TaskInterface{}, address)
+		if err != nil {
+			return err
+		}
+		handler := gin.New()
+
+		taskEngine.ApplyHttpHandlers(handler.Group("/"))
+		defer taskEngine.GracefullyTerminate(time.Hour)
+
+		fh := &paths.FetchHandler{Local: localStore, PfHandler: &paths.DefaultPartialFileHandler{}}
+		handler.NoRoute(gin.HandlerFunc(func(c *gin.Context) {
+			if !auth.HasPerm(c, nil, api.PermAdmin) {
+				c.JSON(401, struct{ Error string }{"unauthorized: missing admin permission"})
+				return
+			}
+
+			fh.ServeHTTP(c.Writer, c.Request)
+		}))
+		// local APIs
+		{
+			// debugging
+			handler.GET("/debug/metrics", gin.WrapH(metrics.Exporter()))
+			pprof.Register(handler)
+		}
+
+		// Serve the RPC.
+		endpoint, err := r.APIEndpoint()
+		if err != nil {
+			return fmt.Errorf("getting API endpoint: %w", err)
+		}
+		rpcStopper, err := node.ServeRPC(handler, "lotus-provider", endpoint)
+		if err != nil {
+			return fmt.Errorf("failed to start json-rpc endpoint: %s", err)
+		}
+
+		// Monitor for shutdown.
+		finishCh := node.MonitorShutdown(shutdownChan,
+			node.ShutdownHandler{Component: "rpc server", StopFunc: rpcStopper},
+			node.ShutdownHandler{Component: "provider", StopFunc: stop},
+		)
+
+		<-finishCh
+		return nil
+	},
+}

cmd/lotus-provider/stop.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	_ "net/http/pprof"
+
+	"github.com/urfave/cli/v2"
+
+	lcli "github.com/filecoin-project/lotus/cli"
+)
+
+var stopCmd = &cli.Command{
+	Name:  "stop",
+	Usage: "Stop a running lotus provider",
+	Flags: []cli.Flag{},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		err = api.Shutdown(lcli.ReqContext(cctx))
+		if err != nil {
+			return err
+		}
+
+		return nil
+	},
+}

go.mod

@@ -63,6 +63,8 @@ require (
 	github.com/gbrlsnchs/jwt/v3 v3.0.1
 	github.com/gdamore/tcell/v2 v2.2.0
 	github.com/georgysavva/scany/v2 v2.0.0
+	github.com/gin-contrib/pprof v1.4.0
+	github.com/gin-gonic/gin v1.9.1
 	github.com/go-openapi/spec v0.19.11
 	github.com/golang/mock v1.6.0
 	github.com/google/uuid v1.3.0
@@ -179,8 +181,10 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bep/debounce v1.2.1 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
+	github.com/bytedance/sonic v1.9.1 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
 	github.com/cilium/ebpf v0.9.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
@@ -204,8 +208,9 @@ require (
 	github.com/flynn/noise v1.0.0 // indirect
 	github.com/francoispqt/gojay v1.2.13 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/gdamore/encoding v1.0.0 // indirect
-	github.com/gin-gonic/gin v1.9.1 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
@@ -214,7 +219,11 @@ require (
 	github.com/go-openapi/jsonpointer v0.19.3 // indirect
 	github.com/go-openapi/jsonreference v0.19.4 // indirect
 	github.com/go-openapi/swag v0.19.11 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.14.0 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/glog v1.1.0 // indirect
@@ -259,10 +268,12 @@ require (
 	github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kilic/bls12-381 v0.1.0 // indirect
 	github.com/klauspost/compress v1.16.5 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/koron/go-ssdp v0.0.4 // indirect
+	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/libp2p/go-cidranger v1.1.0 // indirect
 	github.com/libp2p/go-flow-metrics v0.1.0 // indirect
 	github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect
@@ -283,6 +294,8 @@ require (
 	github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
 	github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
 	github.com/minio/sha256-simd v1.0.1 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/multiformats/go-base36 v0.2.0 // indirect
 	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
@@ -293,6 +306,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.9.7 // indirect
 	github.com/opencontainers/runtime-spec v1.0.2 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
 	github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -312,6 +326,7 @@ require (
 	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/tidwall/gjson v1.14.4 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/twmb/murmur3 v1.1.6 // indirect
 	github.com/ugorji/go/codec v1.2.11 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
@@ -329,6 +344,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.16.0 // indirect
 	go.uber.org/dig v1.17.0 // indirect
 	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
+	golang.org/x/arch v0.3.0 // indirect
 	golang.org/x/mod v0.10.0 // indirect
 	golang.org/x/text v0.10.0 // indirect
 	gonum.org/v1/gonum v0.13.0 // indirect

go.sum

@@ -142,7 +142,9 @@ github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46f
 github.com/buger/goterm v1.0.3 h1:7V/HeAQHrzPk/U4BvyH2g9u+xbUW9nr4yRPyG59W4fM=
 github.com/buger/goterm v1.0.3/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE=
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
+github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
 github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
+github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -153,7 +155,9 @@ github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
+github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
+github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM=
 github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ=
@@ -389,6 +393,7 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
+github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
 github.com/gbrlsnchs/jwt/v3 v3.0.1 h1:lbUmgAKpxnClrKloyIwpxm4OuWeDl5wLk52G91ODPw4=
 github.com/gbrlsnchs/jwt/v3 v3.0.1/go.mod h1:AncDcjXz18xetI3A6STfXq2w+LuTx8pQ8bGEwRN8zVM=
 github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=
@@ -398,9 +403,12 @@ github.com/gdamore/tcell/v2 v2.2.0/go.mod h1:cTTuF84Dlj/RqmaCIV5p4w8uG1zWdk0SF6o
 github.com/georgysavva/scany/v2 v2.0.0 h1:RGXqxDv4row7/FYoK8MRXAZXqoWF/NM+NP0q50k3DKU=
 github.com/georgysavva/scany/v2 v2.0.0/go.mod h1:sigOdh+0qb/+aOs3TVhehVT10p8qJL7K/Zhyz8vWo38=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gin-contrib/pprof v1.4.0 h1:XxiBSf5jWZ5i16lNOPbMTVdgHBdhfGRD5PZ1LWazzvg=
+github.com/gin-contrib/pprof v1.4.0/go.mod h1:RrehPJasUVBPK6yTUwOl8/NP6i0vbUgmxtis+Z5KE90=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
+github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk=
 github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
 github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
@@ -447,12 +455,19 @@ github.com/go-openapi/swag v0.19.8/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfT
 github.com/go-openapi/swag v0.19.11 h1:RFTu/dlFySpyVvJDfp/7674JY4SDglYWKztbiIGFpmc=
 github.com/go-openapi/swag v0.19.11/go.mod h1:Uc0gKkdR+ojzsEpjh39QChyu92vPgIr72POcgHMAgSY=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
 github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
+github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
 github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
+github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
+github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos=
 github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js=
+github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
@@ -468,7 +483,9 @@ github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
 github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
 github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
 github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
+github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
+github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
@@ -974,7 +991,9 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
+github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
 github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
+github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
 github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E=
 github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ=
 github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E=
@@ -1410,14 +1429,16 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
-github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
 github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
+github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk=
 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=
 github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -1508,6 +1529,7 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
@@ -1607,6 +1629,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc=
@@ -1628,13 +1652,16 @@ github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhso
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
+github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg=
 github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
 github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
 github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
+github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
+github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
 github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
 github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
@@ -1798,7 +1825,9 @@ go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1
 go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg=
 go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc=
 go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU=
+golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
+golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -2059,6 +2088,7 @@ golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -2329,6 +2359,7 @@ lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1
 nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g=
 nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

itests/harmonytask_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 	"time"
 
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/lotus/itests/kit"
@@ -30,6 +31,7 @@ func withDbSetup(t *testing.T, f func(*kit.TestMiner)) {
 		kit.LatestActorsAt(-1),
 		kit.MockProofs(),
 	)
+	logging.SetLogLevel("harmonytask", "debug")
 
 	f(miner)
 }
@@ -60,12 +62,12 @@ func (t *task1) TypeDetails() harmonytask.TaskTypeDetails {
 func (t *task1) Adder(add harmonytask.AddTaskFunc) {
 	for _, vTmp := range t.toAdd {
 		v := vTmp
-		add(func(tID harmonytask.TaskID, tx *harmonydb.Tx) bool {
+		add(func(tID harmonytask.TaskID, tx *harmonydb.Tx) (bool, error) {
 			t.myPersonalTableLock.Lock()
 			defer t.myPersonalTableLock.Unlock()
 
 			t.myPersonalTable[tID] = v
-			return true
+			return true, nil
 		})
 	}
 }
@@ -120,10 +122,10 @@ func fooLetterAdder(t *testing.T, cdb *harmonydb.DB) *passthru {
 		adder: func(add harmonytask.AddTaskFunc) {
 			for _, vTmp := range []string{"A", "B"} {
 				v := vTmp
-				add(func(tID harmonytask.TaskID, tx *harmonydb.Tx) bool {
+				add(func(tID harmonytask.TaskID, tx *harmonydb.Tx) (bool, error) {
 					_, err := tx.Exec("INSERT INTO itest_scratch (some_int, content) VALUES ($1,$2)", tID, v)
 					require.NoError(t, err)
-					return true
+					return true, nil
 				})
 			}
 		},
@@ -161,7 +163,7 @@ func TestHarmonyTasksWith2PartiesPolling(t *testing.T) {
 		sender.GracefullyTerminate(time.Second * 5)
 		worker.GracefullyTerminate(time.Second * 5)
 		sort.Strings(dest)
-		require.Equal(t, dest, []string{"A", "B"})
+		require.Equal(t, []string{"A", "B"}, dest)
 	})
 }
 

lib/harmony/harmonydb/harmonydb.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"math/rand"
+	"net"
 	"regexp"
 	"sort"
 	"strconv"
@@ -139,6 +140,20 @@ func (t tracer) TraceQueryEnd(ctx context.Context, conn *pgx.Conn, data pgx.Trac
 		"milliseconds", ms)
 }
 
+func (db *DB) GetRoutableIP() (string, error) {
+	tx, err := db.pgx.Begin(context.Background())
+	if err != nil {
+		return "", err
+	}
+	defer func() { _ = tx.Rollback(context.Background()) }()
+	local := tx.Conn().PgConn().Conn().LocalAddr()
+	addr, ok := local.(*net.TCPAddr)
+	if !ok {
+		return "", fmt.Errorf("could not get local addr from %v", addr)
+	}
+	return addr.IP.String(), nil
+}
+
 // addStatsAndConnect connects a prometheus logger. Be sure to run this before using the DB.
 func (db *DB) addStatsAndConnect() error {
 

lib/harmony/harmonytask/harmonytask.go

@@ -3,13 +3,12 @@ package harmonytask
 import (
 	"context"
 	"fmt"
-	"net/http"
 	"strconv"
 	"sync"
 	"sync/atomic"
 	"time"
 
-	"github.com/gorilla/mux"
+	"github.com/gin-gonic/gin"
 
 	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
 	"github.com/filecoin-project/lotus/lib/harmony/resources"
@@ -40,7 +39,7 @@ type TaskTypeDetails struct {
 	// It should also return success if the trigger succeeded.
 	// NOTE: if refatoring tasks, see if your task is
 	// necessary. Ex: Is the sector state correct for your stage to run?
-	Follows map[string]func(TaskID, AddTaskFunc) bool
+	Follows map[string]func(TaskID, AddTaskFunc) (bool, error)
 }
 
 // TaskInterface must be implemented in order to have a task used by harmonytask.
@@ -90,7 +89,7 @@ type TaskInterface interface {
 	Adder(AddTaskFunc)
 }
 
-type AddTaskFunc func(extraInfo func(TaskID, *harmonydb.Tx) bool)
+type AddTaskFunc func(extraInfo func(TaskID, *harmonydb.Tx) (bool, error))
 
 type TaskEngine struct {
 	ctx            context.Context
@@ -107,8 +106,9 @@ type TaskEngine struct {
 	lastCleanup    atomic.Value
 }
 type followStruct struct {
-	f func(TaskID, AddTaskFunc) bool
+	f    func(TaskID, AddTaskFunc) (bool, error)
 	h    *taskTypeHandler
+	name string
 }
 
 type TaskID int
@@ -153,7 +153,7 @@ func New(
 		}
 
 		for name, fn := range c.TypeDetails().Follows {
-			e.follows[name] = append(e.follows[name], followStruct{fn, &h})
+			e.follows[name] = append(e.follows[name], followStruct{fn, &h, name})
 
 			// populate harmony_task_follows
 			_, err := db.Exec(e.ctx, `INSERT INTO harmony_task_follows (owner_id, from_task, to_task)
@@ -271,7 +271,12 @@ func (e *TaskEngine) followWorkInDB() {
 					continue
 				}
 				// we need to create this task
-				if !src.h.Follows[fromName](TaskID(workAlreadyDone), src.h.AddTask) {
+				b, err := src.h.Follows[fromName](TaskID(workAlreadyDone), src.h.AddTask)
+				if err != nil {
+					log.Errorw("Could not follow: ", "error", err)
+					continue
+				}
+				if !b {
 					// But someone may have beaten us to it.
 					log.Debugf("Unable to add task %s following Task(%d, %s)", src.h.Name, workAlreadyDone, fromName)
 				}
@@ -314,54 +319,54 @@ func (e *TaskEngine) pollerTryAllWork() {
 
 // GetHttpHandlers needs to be used by the http server to register routes.
 // This implements the receiver-side of "follows" and "bumps" the fast way.
-func (e *TaskEngine) GetHttpHandlers() http.Handler {
-	root := mux.NewRouter()
-	s := root.PathPrefix("/scheduler")
-	f := s.PathPrefix("/follows")
-	b := s.PathPrefix("/bump")
-	for name, vsTmp := range e.follows {
-		vs := vsTmp
-		f.Path("/" + name + "/{tID}").Methods("GET").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			tIDString := mux.Vars(r)["tID"]
+func (e *TaskEngine) ApplyHttpHandlers(root gin.IRouter) {
+	s := root.Group("/scheduler")
+	f := s.Group("/follows")
+	b := s.Group("/bump")
+	for name, vs := range e.follows {
+		name, vs := name, vs
+		f.GET("/"+name+"/:tID", func(c *gin.Context) {
+			tIDString := c.Param("tID")
 			tID, err := strconv.Atoi(tIDString)
 			if err != nil {
-				w.WriteHeader(401)
-				fmt.Fprint(w, err.Error())
+				c.JSON(401, map[string]any{"error": err.Error()})
 				return
 			}
 			taskAdded := false
 			for _, vTmp := range vs {
 				v := vTmp
-				taskAdded = taskAdded || v.f(TaskID(tID), v.h.AddTask)
+				b, err := v.f(TaskID(tID), v.h.AddTask)
+				if err != nil {
+					log.Errorw("Follow attempt failed", "error", err, "from", name, "to", v.name)
+				}
+				taskAdded = taskAdded || b
 			}
 			if taskAdded {
 				e.tryAllWork <- true
-				w.WriteHeader(200)
+				c.Status(200)
 				return
 			}
-			w.WriteHeader(202) // NOTE: 202 for "accepted" but not worked.
+			c.Status(202) // NOTE: 202 for "accepted" but not worked.
 		})
 	}
 	for _, hTmp := range e.handlers {
 		h := hTmp
-		b.Path("/" + h.Name + "/{tID}").Methods("GET").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			tIDString := mux.Vars(r)["tID"]
+		b.GET("/"+h.Name+"/:tID", func(c *gin.Context) {
+			tIDString := c.Param("tID")
 			tID, err := strconv.Atoi(tIDString)
 			if err != nil {
-				w.WriteHeader(401)
-				fmt.Fprint(w, err.Error())
+				c.JSON(401, map[string]any{"error": err.Error()})
 				return
 			}
 			// We NEED to block while trying to deliver
 			// this work to ease the network impact.
 			if h.considerWork("bump", []TaskID{TaskID(tID)}) {
-				w.WriteHeader(200)
+				c.Status(200)
 				return
 			}
-			w.WriteHeader(202) // NOTE: 202 for "accepted" but not worked.
+			c.Status(202) // NOTE: 202 for "accepted" but not worked.
 		})
 	}
-	return root
 }
 
 func (e *TaskEngine) bump(taskType string) {
@@ -394,10 +399,13 @@ func (e *TaskEngine) resourcesInUse() resources.Resources {
 		tmp.Cpu -= int(ct) * t.Cost.Cpu
 		tmp.Gpu -= float64(ct) * t.Cost.Gpu
 		tmp.Ram -= uint64(ct) * t.Cost.Ram
+		if len(t.Cost.GpuRam) == 0 {
+			continue
+		}
 		for i := int32(0); i < ct; i++ {
 			for grIdx, j := range tmp.GpuRam {
 				if j > t.Cost.GpuRam[0] {
-					tmp.GpuRam[grIdx] = j - t.Cost.GpuRam[0]
+					tmp.GpuRam[grIdx] = 0 // Only 1 per GPU. j - t.Cost.GpuRam[0]
 					break
 				}
 			}

lib/harmony/harmonytask/task_type_handler.go

@@ -3,6 +3,7 @@ package harmonytask
 import (
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
 	"strconv"
@@ -10,6 +11,7 @@ import (
 	"time"
 
 	logging "github.com/ipfs/go-log/v2"
+	"github.com/samber/lo"
 
 	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
 )
@@ -23,15 +25,14 @@ type taskTypeHandler struct {
 	Count      atomic.Int32
 }
 
-func (h *taskTypeHandler) AddTask(extra func(TaskID, *harmonydb.Tx) bool) {
+func (h *taskTypeHandler) AddTask(extra func(TaskID, *harmonydb.Tx) (bool, error)) {
 	var tID TaskID
-	did, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) bool {
+	did, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) (bool, error) {
 		// create taskID (from DB)
 		_, err := tx.Exec(`INSERT INTO harmony_task (name, added_by, posted_time) 
 			VALUES ($1, $2, CURRENT_TIMESTAMP) `, h.Name, h.TaskEngine.ownerID)
 		if err != nil {
-			log.Error("Could not insert into harmonyTask", err)
-			return false
+			return false, fmt.Errorf("could not insert into harmonyTask: %w", err)
 		}
 		err = tx.QueryRow("SELECT id FROM harmony_task ORDER BY update_time DESC LIMIT 1").Scan(&tID)
 		if err != nil {
@@ -59,7 +60,7 @@ top:
 
 	// 1. Can we do any more of this task type?
 	if h.Max > -1 && int(h.Count.Load()) == h.Max {
-		log.Infow("did not accept task", "name", h.Name, "reason", "at max already")
+		log.Debugw("did not accept task", "name", h.Name, "reason", "at max already")
 		return false
 	}
 
@@ -69,7 +70,7 @@ top:
 	// 2. Can we do any more work?
 	err := h.AssertMachineHasCapacity()
 	if err != nil {
-		log.Info(err)
+		log.Debugw("did not accept task", "name", h.Name, "reason", "at capacity already: "+err.Error())
 		return false
 	}
 
@@ -80,7 +81,7 @@ top:
 		return false
 	}
 	if tID == nil {
-		log.Infow("did not accept task", "task_id", ids[0], "reason", "CanAccept() refused")
+		log.Infow("did not accept task", "task_id", ids[0], "reason", "CanAccept() refused", "name", h.Name)
 		return false
 	}
 
@@ -91,7 +92,7 @@ top:
 		return false
 	}
 	if ct == 0 {
-		log.Infow("did not accept task", "task_id", strconv.Itoa(int(*tID)), "reason", "already Taken")
+		log.Infow("did not accept task", "task_id", strconv.Itoa(int(*tID)), "reason", "already Taken", "name", h.Name)
 		var tryAgain = make([]TaskID, 0, len(ids)-1)
 		for _, id := range ids {
 			if id != *tID {
@@ -104,7 +105,7 @@ top:
 
 	go func() {
 		h.Count.Add(1)
-		log.Infow("Beginning work on Task", "id", *tID, "from", from, "type", h.Name)
+		log.Infow("Beginning work on Task", "id", *tID, "from", from, "name", h.Name)
 
 		var done bool
 		var doErr error
@@ -146,19 +147,18 @@ top:
 func (h *taskTypeHandler) recordCompletion(tID TaskID, workStart time.Time, done bool, doErr error) {
 	workEnd := time.Now()
 
-	cm, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) bool {
+	cm, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) (bool, error) {
 		var postedTime time.Time
 		err := tx.QueryRow(`SELECT posted_time FROM harmony_task WHERE id=$1`, tID).Scan(&postedTime)
 		if err != nil {
-			log.Error("Could not log completion: ", err)
-			return false
+			return false, fmt.Errorf("could not log completion: %w ", err)
 		}
 		result := "unspecified error"
 		if done {
 			_, err = tx.Exec("DELETE FROM harmony_task WHERE id=$1", tID)
 			if err != nil {
-				log.Error("Could not log completion: ", err)
-				return false
+
+				return false, fmt.Errorf("could not log completion: %w", err)
 			}
 			result = ""
 		} else {
@@ -171,8 +171,7 @@ func (h *taskTypeHandler) recordCompletion(tID TaskID, workStart time.Time, done
 				err = tx.QueryRow(`SELECT count(*) FROM harmony_task_history 
 				WHERE task_id=$1 AND result=FALSE`, tID).Scan(&ct)
 				if err != nil {
-					log.Error("Could not read task history:", err)
-					return false
+					return false, fmt.Errorf("could not read task history: %w", err)
 				}
 				if ct >= h.MaxFailures {
 					deleteTask = true
@@ -181,15 +180,13 @@ func (h *taskTypeHandler) recordCompletion(tID TaskID, workStart time.Time, done
 			if deleteTask {
 				_, err = tx.Exec("DELETE FROM harmony_task WHERE id=$1", tID)
 				if err != nil {
-					log.Error("Could not delete failed job: ", err)
-					return false
+					return false, fmt.Errorf("could not delete failed job: %w", err)
 				}
 				// Note: Extra Info is left laying around for later review & clean-up
 			} else {
 				_, err := tx.Exec(`UPDATE harmony_task SET owner_id=NULL WHERE id=$1`, tID)
 				if err != nil {
-					log.Error("Could not disown failed task: ", tID, err)
-					return false
+					return false, fmt.Errorf("could not disown failed task: %v %v", tID, err)
 				}
 			}
 		}
@@ -197,10 +194,9 @@ func (h *taskTypeHandler) recordCompletion(tID TaskID, workStart time.Time, done
 												(task_id, name, posted,    work_start, work_end, result, err)
 			 VALUES ($1, $2, $3, $4, $5, $6, $7)`, tID, h.Name, postedTime, workStart, workEnd, done, result)
 		if err != nil {
-			log.Error("Could not write history: ", err)
-			return false
+			return false, fmt.Errorf("could not write history: %w", err)
 		}
-		return true
+		return true, nil
 	})
 	if err != nil {
 		log.Error("Could not record transaction: ", err)
@@ -223,8 +219,12 @@ func (h *taskTypeHandler) AssertMachineHasCapacity() error {
 	if r.Gpu-h.Cost.Gpu < 0 {
 		return errors.New("Did not accept " + h.Name + " task: out of available GPU")
 	}
+	gpuRamSum := lo.Sum(h.Cost.GpuRam)
+	if gpuRamSum == 0 {
+		goto enoughGpuRam
+	}
 	for _, u := range r.GpuRam {
-		if u > h.Cost.GpuRam[0] {
+		if u >= gpuRamSum {
 			goto enoughGpuRam
 		}
 	}
@@ -249,7 +249,11 @@ func (h *taskTypeHandler) triggerCompletionListeners(tID TaskID) {
 	inProcessDefs := h.TaskEngine.follows[h.Name]
 	inProcessFollowers := make([]string, len(inProcessDefs))
 	for _, fs := range inProcessDefs {
-		if fs.f(tID, fs.h.AddTask) {
+		b, err := fs.f(tID, fs.h.AddTask)
+		if err != nil {
+			log.Error("Could not follow", "error", err, "from", h.Name, "to", fs.name)
+		}
+		if b {
 			inProcessFollowers = append(inProcessFollowers, fs.h.Name)
 		}
 	}

lib/harmony/resources/miniopencl/mini_opencl.go

@@ -19,7 +19,11 @@ type Platform struct {
 func GetPlatforms() ([]*Platform, error) {
 	var platformIds [maxPlatforms]C.cl_platform_id
 	var nPlatforms C.cl_uint
-	if err := C.clGetPlatformIDs(C.cl_uint(maxPlatforms), &platformIds[0], &nPlatforms); err != C.CL_SUCCESS {
+	err := C.clGetPlatformIDs(C.cl_uint(maxPlatforms), &platformIds[0], &nPlatforms)
+	if err == -1001 { // No platforms found
+		return nil, nil
+	}
+	if err != C.CL_SUCCESS {
 		return nil, toError(err)
 	}
 	platforms := make([]*Platform, nPlatforms)
@@ -68,7 +72,7 @@ func toError(code C.cl_int) error {
 type ErrOther int
 
 func (e ErrOther) Error() string {
-	return fmt.Sprintf("cl: error %d", int(e))
+	return fmt.Sprintf("OpenCL: error %d", int(e))
 }
 
 // Size of global device memory in bytes.

lib/harmony/resources/resources.go

@@ -54,11 +54,12 @@ func Register(db *harmonydb.DB, hostnameAndPort string) (*Reg, error) {
 		if err != nil {
 			return nil, fmt.Errorf("could not read from harmony_machines: %w", err)
 		}
+		gpuram := lo.Sum(reg.GpuRam)
 		if len(ownerID) == 0 {
 			err = db.QueryRow(ctx, `INSERT INTO harmony_machines 
 		(host_and_port, cpu, ram, gpu, gpuram) VALUES
 		($1,$2,$3,$4,$5) RETURNING id`,
-				hostnameAndPort, reg.Cpu, reg.Ram, reg.Gpu, reg.GpuRam).Scan(&reg.Resources.MachineID)
+				hostnameAndPort, reg.Cpu, reg.Ram, reg.Gpu, gpuram).Scan(&reg.Resources.MachineID)
 			if err != nil {
 				return nil, err
 			}
@@ -67,7 +68,7 @@ func Register(db *harmonydb.DB, hostnameAndPort string) (*Reg, error) {
 			reg.MachineID = ownerID[0]
 			_, err := db.Exec(ctx, `UPDATE harmony_machines SET
 		   cpu=$1, ram=$2, gpu=$3, gpuram=$4 WHERE id=$6`,
-				reg.Cpu, reg.Ram, reg.Gpu, reg.GpuRam, reg.Resources.MachineID)
+				reg.Cpu, reg.Ram, reg.Gpu, gpuram, reg.Resources.MachineID)
 			if err != nil {
 				return nil, err
 			}
@@ -132,7 +133,7 @@ func getResources() (res Resources, err error) {
 		}
 		all := strings.ToLower(strings.Join(gpus, ","))
 		if len(gpus) > 1 || strings.Contains(all, "ati") || strings.Contains(all, "nvidia") {
-			res.Gpu = 1
+			res.Gpu = float64(len(gpus))
 		}
 	}
 

node/builder.go

@@ -341,6 +341,29 @@ func Repo(r repo.Repo) Option {
 	}
 }
 
+func Provider(r repo.Repo) Option {
+	return func(settings *Settings) error {
+		lr, err := r.Lock(settings.nodeType)
+		if err != nil {
+			return err
+		}
+		c, err := lr.Config()
+		if err != nil {
+			return err
+		}
+		_ = c
+		return Options(
+			func(s *Settings) error { s.Base = true; return nil }, // mark Base as applied
+			ApplyIf(func(s *Settings) bool { return s.Config },
+				Error(errors.New("the Base() option must be set before Config option")),
+			),
+			Override(new(repo.LockedRepo), modules.LockedRepo(lr)), // module handles closing
+			//ApplyIf(IsType(repo.WdPost), ConfigWdPost(c)),
+			//ApplyIf(IsType(repo.WinPost), ConfigWinPost(c)),
+		)(settings)
+	}
+}
+
 type StopFunc func(context.Context) error
 
 // New builds and starts new Filecoin node

node/repo/fsrepo.go

@@ -185,6 +185,30 @@ func (worker) APIInfoEnvVars() (primary string, fallbacks []string, deprecated [
 	return "WORKER_API_INFO", nil, nil
 }
 
+type provider struct{}
+
+var Provider provider
+
+func (provider) Type() string {
+	return "Provider"
+}
+
+func (provider) Config() interface{} {
+	return &struct{}{}
+}
+
+func (provider) APIFlags() []string {
+	return []string{"provider-api-url"}
+}
+
+func (provider) RepoFlags() []string {
+	return []string{"provider-repo"}
+}
+
+func (provider) APIInfoEnvVars() (primary string, fallbacks []string, deprecated []string) {
+	return "PROVIDER_API_INFO", nil, nil
+}
+
 var Wallet wallet
 
 type wallet struct {

scripts/lotus-provider.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Lotus Provider
+After=network.target
+After=lotus-daemon.service
+
+[Service]
+ExecStart=/usr/local/bin/lotus-provider run
+Environment=GOLOG_FILE="/var/log/lotus/provider.log"
+Environment=GOLOG_LOG_FMT="json"
+LimitNOFILE=1000000
+[Install]
+WantedBy=multi-user.target