Merge branch 'feat/sturdypost' into feat/wdpost-adder
This commit is contained in:
Shrenuj Bansal
commit
eacd5bb970
1
.gitignore
vendored
1
.gitignore
vendored
@ -6,6 +6,7 @@
|
||||
/lotus-chainwatch
|
||||
/lotus-shed
|
||||
/lotus-sim
|
||||
/lotus-provider
|
||||
/lotus-townhall
|
||||
/lotus-fountain
|
||||
/lotus-stats
|
||||
|
||||
@ -109,6 +109,7 @@ COPY --from=lotus-builder /opt/filecoin/lotus-wallet /usr/local/bin/
|
||||
COPY --from=lotus-builder /opt/filecoin/lotus-gateway /usr/local/bin/
|
||||
COPY --from=lotus-builder /opt/filecoin/lotus-miner /usr/local/bin/
|
||||
COPY --from=lotus-builder /opt/filecoin/lotus-worker /usr/local/bin/
|
||||
COPY --from=lotus-builder /opt/filecoin/lotus-provider /usr/local/bin/
|
||||
COPY --from=lotus-builder /opt/filecoin/lotus-stats /usr/local/bin/
|
||||
COPY --from=lotus-builder /opt/filecoin/lotus-fountain /usr/local/bin/
|
||||
|
||||
@ -117,11 +118,13 @@ RUN mkdir /var/lib/lotus
|
||||
RUN mkdir /var/lib/lotus-miner
|
||||
RUN mkdir /var/lib/lotus-worker
|
||||
RUN mkdir /var/lib/lotus-wallet
|
||||
RUN mkdir /var/lib/lotus-provider
|
||||
RUN chown fc: /var/tmp/filecoin-proof-parameters
|
||||
RUN chown fc: /var/lib/lotus
|
||||
RUN chown fc: /var/lib/lotus-miner
|
||||
RUN chown fc: /var/lib/lotus-worker
|
||||
RUN chown fc: /var/lib/lotus-wallet
|
||||
RUN chown fc: /var/lib/lotus-provider
|
||||
|
||||
|
||||
VOLUME /var/tmp/filecoin-proof-parameters
|
||||
@ -129,6 +132,7 @@ VOLUME /var/lib/lotus
|
||||
VOLUME /var/lib/lotus-miner
|
||||
VOLUME /var/lib/lotus-worker
|
||||
VOLUME /var/lib/lotus-wallet
|
||||
VOLUME /var/lib/lotus-provider
|
||||
|
||||
EXPOSE 1234
|
||||
EXPOSE 2345
|
||||
|
||||
29
Makefile
29
Makefile
@ -97,6 +97,10 @@ lotus-miner: $(BUILD_DEPS)
|
||||
.PHONY: lotus-miner
|
||||
BINS+=lotus-miner
|
||||
|
||||
lotus-provider: $(BUILD_DEPS)
|
||||
rm -f lotus-provider
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-provider ./cmd/lotus-provider
|
||||
|
||||
lotus-worker: $(BUILD_DEPS)
|
||||
rm -f lotus-worker
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-worker ./cmd/lotus-worker
|
||||
@ -129,6 +133,9 @@ install-daemon:
|
||||
install-miner:
|
||||
install -C ./lotus-miner /usr/local/bin/lotus-miner
|
||||
|
||||
install-provider:
|
||||
install -C ./lotus-provider /usr/local/bin/lotus-provider
|
||||
|
||||
install-worker:
|
||||
install -C ./lotus-worker /usr/local/bin/lotus-worker
|
||||
|
||||
@ -144,6 +151,9 @@ uninstall-daemon:
|
||||
uninstall-miner:
|
||||
rm -f /usr/local/bin/lotus-miner
|
||||
|
||||
uninstall-provider:
|
||||
rm -f /usr/local/bin/lotus-provider
|
||||
|
||||
uninstall-worker:
|
||||
rm -f /usr/local/bin/lotus-worker
|
||||
|
||||
@ -241,6 +251,14 @@ install-miner-service: install-miner install-daemon-service
|
||||
@echo
|
||||
@echo "lotus-miner service installed. Don't forget to run 'sudo systemctl start lotus-miner' to start it and 'sudo systemctl enable lotus-miner' for it to be enabled on startup."
|
||||
|
||||
install-provider-service: install-provider install-daemon-service
|
||||
mkdir -p /etc/systemd/system
|
||||
mkdir -p /var/log/lotus
|
||||
install -C -m 0644 ./scripts/lotus-provider.service /etc/systemd/system/lotus-provider.service
|
||||
systemctl daemon-reload
|
||||
@echo
|
||||
@echo "lotus-provider service installed. Don't forget to run 'sudo systemctl start lotus-provider' to start it and 'sudo systemctl enable lotus-provider' for it to be enabled on startup."
|
||||
|
||||
install-main-services: install-miner-service
|
||||
|
||||
install-all-services: install-main-services
|
||||
@ -259,6 +277,12 @@ clean-miner-service:
|
||||
rm -f /etc/systemd/system/lotus-miner.service
|
||||
systemctl daemon-reload
|
||||
|
||||
clean-provider-service:
|
||||
-systemctl stop lotus-provider
|
||||
-systemctl disable lotus-provider
|
||||
rm -f /etc/systemd/system/lotus-provider.service
|
||||
systemctl daemon-reload
|
||||
|
||||
clean-main-services: clean-daemon-service
|
||||
|
||||
clean-all-services: clean-main-services
|
||||
@ -360,15 +384,16 @@ gen: actors-code-gen type-gen cfgdoc-gen docsgen api-gen circleci fiximports
|
||||
|
||||
jen: gen
|
||||
|
||||
snap: lotus lotus-miner lotus-worker
|
||||
snap: lotus lotus-miner lotus-worker lotus-provider
|
||||
snapcraft
|
||||
# snapcraft upload ./lotus_*.snap
|
||||
|
||||
# separate from gen because it needs binaries
|
||||
docsgen-cli: lotus lotus-miner lotus-worker
|
||||
docsgen-cli: lotus lotus-miner lotus-worker lotus-provider
|
||||
python3 ./scripts/generate-lotus-cli.py
|
||||
./lotus config default > documentation/en/default-lotus-config.toml
|
||||
./lotus-miner config default > documentation/en/default-lotus-miner-config.toml
|
||||
./lotus-provider config default > documentation/en/default-lotus-provider-config.toml
|
||||
.PHONY: docsgen-cli
|
||||
|
||||
print-%:
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
@ -8,7 +9,6 @@ import (
|
||||
"syscall"
|
||||
|
||||
ufcli "github.com/urfave/cli/v2"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
type PrintHelpErr struct {
|
||||
@ -52,7 +52,7 @@ func RunApp(app *ufcli.App) {
|
||||
fmt.Fprintf(os.Stderr, "ERROR: %s\n\n", err) // nolint:errcheck
|
||||
}
|
||||
var phe *PrintHelpErr
|
||||
if xerrors.As(err, &phe) {
|
||||
if errors.As(err, &phe) {
|
||||
_ = ufcli.ShowCommandHelp(phe.Ctx, phe.Ctx.Command.Name)
|
||||
}
|
||||
os.Exit(1)
|
||||
|
||||
61
cmd/lotus-provider/config.go
Normal file
61
cmd/lotus-provider/config.go
Normal file
@ -0,0 +1,61 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
var configCmd = &cli.Command{
|
||||
Name: "config",
|
||||
Usage: "Manage node config",
|
||||
Subcommands: []*cli.Command{
|
||||
configDefaultCmd,
|
||||
configSetCmd,
|
||||
configGetCmd,
|
||||
},
|
||||
}
|
||||
|
||||
var configDefaultCmd = &cli.Command{
|
||||
Name: "default",
|
||||
Usage: "Print default system config",
|
||||
Flags: []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
Name: "no-comment",
|
||||
Usage: "don't comment default values",
|
||||
},
|
||||
},
|
||||
Action: func(cctx *cli.Context) error {
|
||||
fmt.Println("[config]\nstatus = Coming Soon")
|
||||
// [overlay.sealer1.tasks]\nsealer_task_enable = true
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var configSetCmd = &cli.Command{
|
||||
Name: "set",
|
||||
Usage: "Set all config",
|
||||
Action: func(cctx *cli.Context) error {
|
||||
fmt.Println("Coming soon")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var configGetCmd = &cli.Command{
|
||||
Name: "get",
|
||||
Usage: "Get all config",
|
||||
Flags: []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
Name: "no-comment",
|
||||
Usage: "don't comment default values",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "no-doc",
|
||||
Usage: "don't add value documentation",
|
||||
},
|
||||
},
|
||||
Action: func(cctx *cli.Context) error {
|
||||
fmt.Println("Coming soon")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
117
cmd/lotus-provider/main.go
Normal file
117
cmd/lotus-provider/main.go
Normal file
@ -0,0 +1,117 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/fatih/color"
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
"github.com/urfave/cli/v2"
|
||||
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
lcli "github.com/filecoin-project/lotus/cli"
|
||||
cliutil "github.com/filecoin-project/lotus/cli/util"
|
||||
"github.com/filecoin-project/lotus/lib/lotuslog"
|
||||
"github.com/filecoin-project/lotus/lib/tracing"
|
||||
"github.com/filecoin-project/lotus/node/repo"
|
||||
)
|
||||
|
||||
var log = logging.Logger("main")
|
||||
|
||||
func main() {
|
||||
|
||||
lotuslog.SetupLogLevels()
|
||||
|
||||
local := []*cli.Command{
|
||||
//initCmd,
|
||||
runCmd,
|
||||
stopCmd,
|
||||
configCmd,
|
||||
//backupCmd,
|
||||
//lcli.WithCategory("chain", actorCmd),
|
||||
//lcli.WithCategory("storage", sectorsCmd),
|
||||
//lcli.WithCategory("storage", provingCmd),
|
||||
//lcli.WithCategory("storage", storageCmd),
|
||||
//lcli.WithCategory("storage", sealingCmd),
|
||||
}
|
||||
|
||||
jaeger := tracing.SetupJaegerTracing("lotus")
|
||||
defer func() {
|
||||
if jaeger != nil {
|
||||
_ = jaeger.ForceFlush(context.Background())
|
||||
}
|
||||
}()
|
||||
|
||||
for _, cmd := range local {
|
||||
cmd := cmd
|
||||
originBefore := cmd.Before
|
||||
cmd.Before = func(cctx *cli.Context) error {
|
||||
if jaeger != nil {
|
||||
_ = jaeger.Shutdown(cctx.Context)
|
||||
}
|
||||
jaeger = tracing.SetupJaegerTracing("lotus/" + cmd.Name)
|
||||
|
||||
if cctx.IsSet("color") {
|
||||
color.NoColor = !cctx.Bool("color")
|
||||
}
|
||||
|
||||
if originBefore != nil {
|
||||
return originBefore(cctx)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
app := &cli.App{
|
||||
Name: "lotus-provider",
|
||||
Usage: "Filecoin decentralized storage network provider",
|
||||
Version: build.UserVersion(),
|
||||
EnableBashCompletion: true,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "listen",
|
||||
Usage: "host address and port the worker api will listen on",
|
||||
Value: "0.0.0.0:3456",
|
||||
EnvVars: []string{"LOTUS_WORKER_LISTEN"},
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
// examined in the Before above
|
||||
Name: "color",
|
||||
Usage: "use color in display output",
|
||||
DefaultText: "depends on output being a TTY",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "panic-reports",
|
||||
EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"},
|
||||
Hidden: true,
|
||||
Value: "~/.lotusprovider", // should follow --repo default
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "repo",
|
||||
EnvVars: []string{"LOTUS_PATH"},
|
||||
Hidden: true,
|
||||
Value: "~/.lotus",
|
||||
},
|
||||
cliutil.FlagVeryVerbose,
|
||||
},
|
||||
Commands: append(local, lcli.CommonCommands...),
|
||||
Before: func(c *cli.Context) error {
|
||||
return nil
|
||||
},
|
||||
After: func(c *cli.Context) error {
|
||||
if r := recover(); r != nil {
|
||||
// Generate report in LOTUS_PATH and re-raise panic
|
||||
build.GeneratePanicReport(c.String("panic-reports"), c.String(FlagProviderRepo), c.App.Name)
|
||||
panic(r)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
app.Setup()
|
||||
app.Metadata["repoType"] = repo.Provider
|
||||
lcli.RunApp(app)
|
||||
}
|
||||
|
||||
const (
|
||||
FlagProviderRepo = "provider-repo"
|
||||
)
|
||||
248
cmd/lotus-provider/run.go
Normal file
248
cmd/lotus-provider/run.go
Normal file
@ -0,0 +1,248 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gin-contrib/pprof"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/urfave/cli/v2"
|
||||
"go.opencensus.io/stats"
|
||||
"go.opencensus.io/tag"
|
||||
|
||||
"github.com/filecoin-project/go-jsonrpc/auth"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
lcli "github.com/filecoin-project/lotus/cli"
|
||||
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
|
||||
"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
|
||||
"github.com/filecoin-project/lotus/lib/ulimit"
|
||||
"github.com/filecoin-project/lotus/metrics"
|
||||
"github.com/filecoin-project/lotus/node"
|
||||
"github.com/filecoin-project/lotus/node/config"
|
||||
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||
"github.com/filecoin-project/lotus/node/repo"
|
||||
"github.com/filecoin-project/lotus/storage/paths"
|
||||
"github.com/filecoin-project/lotus/storage/sealer/storiface"
|
||||
)
|
||||
|
||||
var runCmd = &cli.Command{
|
||||
Name: "run",
|
||||
Usage: "Start a lotus provider process",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "provider-api",
|
||||
Usage: "Port (default 12300)",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "nosync",
|
||||
Usage: "don't check full-node sync status",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "halt-after-init",
|
||||
Usage: "only run init, then return",
|
||||
Hidden: true,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "manage-fdlimit",
|
||||
Usage: "manage open file limit",
|
||||
Value: true,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "db-host",
|
||||
EnvVars: []string{"LOTUS_DB_HOST"},
|
||||
Usage: "Command separated list of hostnames for yugabyte cluster",
|
||||
Value: "yugabyte",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "db-name",
|
||||
EnvVars: []string{"LOTUS_DB_NAME"},
|
||||
Value: "yugabyte",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "db-user",
|
||||
EnvVars: []string{"LOTUS_DB_USER"},
|
||||
Value: "yugabyte",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "db-password",
|
||||
EnvVars: []string{"LOTUS_DB_PASSWORD"},
|
||||
Value: "yugabyte",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "db-port",
|
||||
EnvVars: []string{"LOTUS_DB_PORT"},
|
||||
Hidden: true,
|
||||
Value: "5433",
|
||||
},
|
||||
},
|
||||
Action: func(cctx *cli.Context) error {
|
||||
if !cctx.Bool("enable-gpu-proving") {
|
||||
err := os.Setenv("BELLMAN_NO_GPU", "true")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
ctx, _ := tag.New(lcli.DaemonContext(cctx),
|
||||
tag.Insert(metrics.Version, build.BuildVersion),
|
||||
tag.Insert(metrics.Commit, build.CurrentCommit),
|
||||
tag.Insert(metrics.NodeType, "provider"),
|
||||
)
|
||||
// Register all metric views
|
||||
/*
|
||||
if err := view.Register(
|
||||
metrics.MinerNodeViews...,
|
||||
); err != nil {
|
||||
log.Fatalf("Cannot register the view: %v", err)
|
||||
}
|
||||
*/
|
||||
// Set the metric to one so it is published to the exporter
|
||||
stats.Record(ctx, metrics.LotusInfo.M(1))
|
||||
|
||||
if cctx.Bool("manage-fdlimit") {
|
||||
if _, _, err := ulimit.ManageFdLimit(); err != nil {
|
||||
log.Errorf("setting file descriptor limit: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Open repo
|
||||
|
||||
repoPath := cctx.String(FlagProviderRepo)
|
||||
r, err := repo.NewFS(repoPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ok, err := r.Exists()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
if err := r.Init(repo.Provider); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
lr, err := r.Lock(repo.Provider)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var localPaths []storiface.LocalPath
|
||||
|
||||
if err := lr.SetStorage(func(sc *storiface.StorageConfig) {
|
||||
sc.StoragePaths = append(sc.StoragePaths, localPaths...)
|
||||
}); err != nil {
|
||||
return fmt.Errorf("set storage config: %w", err)
|
||||
}
|
||||
|
||||
{
|
||||
// init datastore for r.Exists
|
||||
_, err := lr.Datastore(context.Background(), "/metadata")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := lr.Close(); err != nil {
|
||||
return fmt.Errorf("close repo: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
lr, err := r.Lock(repo.Provider)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err := lr.Close(); err != nil {
|
||||
log.Error("closing repo", err)
|
||||
}
|
||||
}()
|
||||
|
||||
db, err := harmonydb.NewFromConfig(config.HarmonyDB{
|
||||
Username: cctx.String("db_user"),
|
||||
Password: cctx.String("db_password"),
|
||||
Hosts: strings.Split(cctx.String("db_host"), ","),
|
||||
Database: cctx.String("db_name"),
|
||||
Port: cctx.String("db_port"),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
shutdownChan := make(chan struct{})
|
||||
|
||||
stop, err := node.New(ctx,
|
||||
node.Override(new(dtypes.ShutdownChan), shutdownChan),
|
||||
node.Provider(r),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating node: %w", err)
|
||||
}
|
||||
|
||||
const unspecifiedAddress = "0.0.0.0"
|
||||
address := cctx.String("listen")
|
||||
addressSlice := strings.Split(address, ":")
|
||||
if ip := net.ParseIP(addressSlice[0]); ip != nil {
|
||||
if ip.String() == unspecifiedAddress {
|
||||
rip, err := db.GetRoutableIP()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
address = rip + ":" + addressSlice[1]
|
||||
}
|
||||
}
|
||||
localStore, err := paths.NewLocal(ctx, lr, nil, []string{"http://" + address + "/remote"})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
taskEngine, err := harmonytask.New(db, []harmonytask.TaskInterface{}, address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
handler := gin.New()
|
||||
|
||||
taskEngine.ApplyHttpHandlers(handler.Group("/"))
|
||||
defer taskEngine.GracefullyTerminate(time.Hour)
|
||||
|
||||
fh := &paths.FetchHandler{Local: localStore, PfHandler: &paths.DefaultPartialFileHandler{}}
|
||||
handler.NoRoute(gin.HandlerFunc(func(c *gin.Context) {
|
||||
if !auth.HasPerm(c, nil, api.PermAdmin) {
|
||||
c.JSON(401, struct{ Error string }{"unauthorized: missing admin permission"})
|
||||
return
|
||||
}
|
||||
|
||||
fh.ServeHTTP(c.Writer, c.Request)
|
||||
}))
|
||||
// local APIs
|
||||
{
|
||||
// debugging
|
||||
handler.GET("/debug/metrics", gin.WrapH(metrics.Exporter()))
|
||||
pprof.Register(handler)
|
||||
}
|
||||
|
||||
// Serve the RPC.
|
||||
endpoint, err := r.APIEndpoint()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting API endpoint: %w", err)
|
||||
}
|
||||
rpcStopper, err := node.ServeRPC(handler, "lotus-provider", endpoint)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start json-rpc endpoint: %s", err)
|
||||
}
|
||||
|
||||
// Monitor for shutdown.
|
||||
finishCh := node.MonitorShutdown(shutdownChan,
|
||||
node.ShutdownHandler{Component: "rpc server", StopFunc: rpcStopper},
|
||||
node.ShutdownHandler{Component: "provider", StopFunc: stop},
|
||||
)
|
||||
|
||||
<-finishCh
|
||||
return nil
|
||||
},
|
||||
}
|
||||
29
cmd/lotus-provider/stop.go
Normal file
29
cmd/lotus-provider/stop.go
Normal file
@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
_ "net/http/pprof"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
|
||||
lcli "github.com/filecoin-project/lotus/cli"
|
||||
)
|
||||
|
||||
var stopCmd = &cli.Command{
|
||||
Name: "stop",
|
||||
Usage: "Stop a running lotus provider",
|
||||
Flags: []cli.Flag{},
|
||||
Action: func(cctx *cli.Context) error {
|
||||
api, closer, err := lcli.GetAPI(cctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
|
||||
err = api.Shutdown(lcli.ReqContext(cctx))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
18
go.mod
18
go.mod
@ -63,6 +63,8 @@ require (
|
||||
github.com/gbrlsnchs/jwt/v3 v3.0.1
|
||||
github.com/gdamore/tcell/v2 v2.2.0
|
||||
github.com/georgysavva/scany/v2 v2.0.0
|
||||
github.com/gin-contrib/pprof v1.4.0
|
||||
github.com/gin-gonic/gin v1.9.1
|
||||
github.com/go-openapi/spec v0.19.11
|
||||
github.com/golang/mock v1.6.0
|
||||
github.com/google/uuid v1.3.0
|
||||
@ -179,8 +181,10 @@ require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bep/debounce v1.2.1 // indirect
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bytedance/sonic v1.9.1 // indirect
|
||||
github.com/cespare/xxhash v1.1.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
|
||||
github.com/cilium/ebpf v0.9.1 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
|
||||
github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
|
||||
@ -204,8 +208,9 @@ require (
|
||||
github.com/flynn/noise v1.0.0 // indirect
|
||||
github.com/francoispqt/gojay v1.2.13 // indirect
|
||||
github.com/fsnotify/fsnotify v1.6.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
|
||||
github.com/gdamore/encoding v1.0.0 // indirect
|
||||
github.com/gin-gonic/gin v1.9.1 // indirect
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/go-kit/log v0.2.1 // indirect
|
||||
github.com/go-logfmt/logfmt v0.5.1 // indirect
|
||||
github.com/go-logr/logr v1.2.4 // indirect
|
||||
@ -214,7 +219,11 @@ require (
|
||||
github.com/go-openapi/jsonpointer v0.19.3 // indirect
|
||||
github.com/go-openapi/jsonreference v0.19.4 // indirect
|
||||
github.com/go-openapi/swag v0.19.11 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.14.0 // indirect
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
|
||||
github.com/goccy/go-json v0.10.2 // indirect
|
||||
github.com/godbus/dbus/v5 v5.1.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/glog v1.1.0 // indirect
|
||||
@ -259,10 +268,12 @@ require (
|
||||
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/jpillora/backoff v1.0.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/kilic/bls12-381 v0.1.0 // indirect
|
||||
github.com/klauspost/compress v1.16.5 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
|
||||
github.com/koron/go-ssdp v0.0.4 // indirect
|
||||
github.com/leodido/go-urn v1.2.4 // indirect
|
||||
github.com/libp2p/go-cidranger v1.1.0 // indirect
|
||||
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
|
||||
github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect
|
||||
@ -283,6 +294,8 @@ require (
|
||||
github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
|
||||
github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
|
||||
github.com/minio/sha256-simd v1.0.1 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
||||
github.com/multiformats/go-base36 v0.2.0 // indirect
|
||||
github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
|
||||
@ -293,6 +306,7 @@ require (
|
||||
github.com/onsi/ginkgo/v2 v2.9.7 // indirect
|
||||
github.com/opencontainers/runtime-spec v1.0.2 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
|
||||
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
@ -312,6 +326,7 @@ require (
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/tidwall/gjson v1.14.4 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/twmb/murmur3 v1.1.6 // indirect
|
||||
github.com/ugorji/go/codec v1.2.11 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
@ -329,6 +344,7 @@ require (
|
||||
go.opentelemetry.io/otel/trace v1.16.0 // indirect
|
||||
go.uber.org/dig v1.17.0 // indirect
|
||||
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
|
||||
golang.org/x/arch v0.3.0 // indirect
|
||||
golang.org/x/mod v0.10.0 // indirect
|
||||
golang.org/x/text v0.10.0 // indirect
|
||||
gonum.org/v1/gonum v0.13.0 // indirect
|
||||
|
||||
33
go.sum
33
go.sum
@ -142,7 +142,9 @@ github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46f
|
||||
github.com/buger/goterm v1.0.3 h1:7V/HeAQHrzPk/U4BvyH2g9u+xbUW9nr4yRPyG59W4fM=
|
||||
github.com/buger/goterm v1.0.3/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE=
|
||||
github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
|
||||
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
|
||||
github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
|
||||
github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
|
||||
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
@ -153,7 +155,9 @@ github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM=
|
||||
github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ=
|
||||
@ -389,6 +393,7 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4
|
||||
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
|
||||
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
|
||||
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
|
||||
github.com/gbrlsnchs/jwt/v3 v3.0.1 h1:lbUmgAKpxnClrKloyIwpxm4OuWeDl5wLk52G91ODPw4=
|
||||
github.com/gbrlsnchs/jwt/v3 v3.0.1/go.mod h1:AncDcjXz18xetI3A6STfXq2w+LuTx8pQ8bGEwRN8zVM=
|
||||
github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=
|
||||
@ -398,9 +403,12 @@ github.com/gdamore/tcell/v2 v2.2.0/go.mod h1:cTTuF84Dlj/RqmaCIV5p4w8uG1zWdk0SF6o
|
||||
github.com/georgysavva/scany/v2 v2.0.0 h1:RGXqxDv4row7/FYoK8MRXAZXqoWF/NM+NP0q50k3DKU=
|
||||
github.com/georgysavva/scany/v2 v2.0.0/go.mod h1:sigOdh+0qb/+aOs3TVhehVT10p8qJL7K/Zhyz8vWo38=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gin-contrib/pprof v1.4.0 h1:XxiBSf5jWZ5i16lNOPbMTVdgHBdhfGRD5PZ1LWazzvg=
|
||||
github.com/gin-contrib/pprof v1.4.0/go.mod h1:RrehPJasUVBPK6yTUwOl8/NP6i0vbUgmxtis+Z5KE90=
|
||||
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
|
||||
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
|
||||
github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
|
||||
github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk=
|
||||
github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
|
||||
github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
|
||||
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
@ -447,12 +455,19 @@ github.com/go-openapi/swag v0.19.8/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfT
|
||||
github.com/go-openapi/swag v0.19.11 h1:RFTu/dlFySpyVvJDfp/7674JY4SDglYWKztbiIGFpmc=
|
||||
github.com/go-openapi/swag v0.19.11/go.mod h1:Uc0gKkdR+ojzsEpjh39QChyu92vPgIr72POcgHMAgSY=
|
||||
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
||||
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
|
||||
github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
|
||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
||||
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
|
||||
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
|
||||
github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
|
||||
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
|
||||
github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos=
|
||||
github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js=
|
||||
github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
|
||||
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
|
||||
@ -468,7 +483,9 @@ github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
|
||||
github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||
github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
|
||||
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
|
||||
github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
|
||||
@ -974,7 +991,9 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
|
||||
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
|
||||
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
|
||||
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
|
||||
github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E=
|
||||
github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ=
|
||||
github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E=
|
||||
@ -1410,14 +1429,16 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T
|
||||
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
|
||||
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
|
||||
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
|
||||
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
|
||||
github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
|
||||
github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
|
||||
github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
|
||||
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk=
|
||||
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=
|
||||
github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
|
||||
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
@ -1508,6 +1529,7 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So
|
||||
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
|
||||
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
|
||||
@ -1607,6 +1629,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc=
|
||||
@ -1628,13 +1652,16 @@ github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhso
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg=
|
||||
github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
|
||||
github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
|
||||
github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg=
|
||||
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
|
||||
github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
|
||||
github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
|
||||
github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
|
||||
github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
|
||||
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
|
||||
@ -1798,7 +1825,9 @@ go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1
|
||||
go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg=
|
||||
go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc=
|
||||
go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU=
|
||||
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
|
||||
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
|
||||
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
@ -2059,6 +2088,7 @@ golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@ -2329,6 +2359,7 @@ lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1
|
||||
nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g=
|
||||
nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
|
||||
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
|
||||
|
||||
@ -9,6 +9,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/filecoin-project/lotus/itests/kit"
|
||||
@ -30,6 +31,7 @@ func withDbSetup(t *testing.T, f func(*kit.TestMiner)) {
|
||||
kit.LatestActorsAt(-1),
|
||||
kit.MockProofs(),
|
||||
)
|
||||
logging.SetLogLevel("harmonytask", "debug")
|
||||
|
||||
f(miner)
|
||||
}
|
||||
@ -60,12 +62,12 @@ func (t *task1) TypeDetails() harmonytask.TaskTypeDetails {
|
||||
func (t *task1) Adder(add harmonytask.AddTaskFunc) {
|
||||
for _, vTmp := range t.toAdd {
|
||||
v := vTmp
|
||||
add(func(tID harmonytask.TaskID, tx *harmonydb.Tx) bool {
|
||||
add(func(tID harmonytask.TaskID, tx *harmonydb.Tx) (bool, error) {
|
||||
t.myPersonalTableLock.Lock()
|
||||
defer t.myPersonalTableLock.Unlock()
|
||||
|
||||
t.myPersonalTable[tID] = v
|
||||
return true
|
||||
return true, nil
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -120,10 +122,10 @@ func fooLetterAdder(t *testing.T, cdb *harmonydb.DB) *passthru {
|
||||
adder: func(add harmonytask.AddTaskFunc) {
|
||||
for _, vTmp := range []string{"A", "B"} {
|
||||
v := vTmp
|
||||
add(func(tID harmonytask.TaskID, tx *harmonydb.Tx) bool {
|
||||
add(func(tID harmonytask.TaskID, tx *harmonydb.Tx) (bool, error) {
|
||||
_, err := tx.Exec("INSERT INTO itest_scratch (some_int, content) VALUES ($1,$2)", tID, v)
|
||||
require.NoError(t, err)
|
||||
return true
|
||||
return true, nil
|
||||
})
|
||||
}
|
||||
},
|
||||
@ -161,7 +163,7 @@ func TestHarmonyTasksWith2PartiesPolling(t *testing.T) {
|
||||
sender.GracefullyTerminate(time.Second * 5)
|
||||
worker.GracefullyTerminate(time.Second * 5)
|
||||
sort.Strings(dest)
|
||||
require.Equal(t, dest, []string{"A", "B"})
|
||||
require.Equal(t, []string{"A", "B"}, dest)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
@ -139,6 +140,20 @@ func (t tracer) TraceQueryEnd(ctx context.Context, conn *pgx.Conn, data pgx.Trac
|
||||
"milliseconds", ms)
|
||||
}
|
||||
|
||||
func (db *DB) GetRoutableIP() (string, error) {
|
||||
tx, err := db.pgx.Begin(context.Background())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.Background()) }()
|
||||
local := tx.Conn().PgConn().Conn().LocalAddr()
|
||||
addr, ok := local.(*net.TCPAddr)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("could not get local addr from %v", addr)
|
||||
}
|
||||
return addr.IP.String(), nil
|
||||
}
|
||||
|
||||
// addStatsAndConnect connects a prometheus logger. Be sure to run this before using the DB.
|
||||
func (db *DB) addStatsAndConnect() error {
|
||||
|
||||
|
||||
@ -3,13 +3,12 @@ package harmonytask
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
|
||||
"github.com/filecoin-project/lotus/lib/harmony/resources"
|
||||
@ -40,7 +39,7 @@ type TaskTypeDetails struct {
|
||||
// It should also return success if the trigger succeeded.
|
||||
// NOTE: if refatoring tasks, see if your task is
|
||||
// necessary. Ex: Is the sector state correct for your stage to run?
|
||||
Follows map[string]func(TaskID, AddTaskFunc) bool
|
||||
Follows map[string]func(TaskID, AddTaskFunc) (bool, error)
|
||||
}
|
||||
|
||||
// TaskInterface must be implemented in order to have a task used by harmonytask.
|
||||
@ -90,7 +89,7 @@ type TaskInterface interface {
|
||||
Adder(AddTaskFunc)
|
||||
}
|
||||
|
||||
type AddTaskFunc func(extraInfo func(TaskID, *harmonydb.Tx) bool)
|
||||
type AddTaskFunc func(extraInfo func(TaskID, *harmonydb.Tx) (bool, error))
|
||||
|
||||
type TaskEngine struct {
|
||||
ctx context.Context
|
||||
@ -107,8 +106,9 @@ type TaskEngine struct {
|
||||
lastCleanup atomic.Value
|
||||
}
|
||||
type followStruct struct {
|
||||
f func(TaskID, AddTaskFunc) bool
|
||||
h *taskTypeHandler
|
||||
f func(TaskID, AddTaskFunc) (bool, error)
|
||||
h *taskTypeHandler
|
||||
name string
|
||||
}
|
||||
|
||||
type TaskID int
|
||||
@ -153,7 +153,7 @@ func New(
|
||||
}
|
||||
|
||||
for name, fn := range c.TypeDetails().Follows {
|
||||
e.follows[name] = append(e.follows[name], followStruct{fn, &h})
|
||||
e.follows[name] = append(e.follows[name], followStruct{fn, &h, name})
|
||||
|
||||
// populate harmony_task_follows
|
||||
_, err := db.Exec(e.ctx, `INSERT INTO harmony_task_follows (owner_id, from_task, to_task)
|
||||
@ -271,7 +271,12 @@ func (e *TaskEngine) followWorkInDB() {
|
||||
continue
|
||||
}
|
||||
// we need to create this task
|
||||
if !src.h.Follows[fromName](TaskID(workAlreadyDone), src.h.AddTask) {
|
||||
b, err := src.h.Follows[fromName](TaskID(workAlreadyDone), src.h.AddTask)
|
||||
if err != nil {
|
||||
log.Errorw("Could not follow: ", "error", err)
|
||||
continue
|
||||
}
|
||||
if !b {
|
||||
// But someone may have beaten us to it.
|
||||
log.Debugf("Unable to add task %s following Task(%d, %s)", src.h.Name, workAlreadyDone, fromName)
|
||||
}
|
||||
@ -314,54 +319,54 @@ func (e *TaskEngine) pollerTryAllWork() {
|
||||
|
||||
// GetHttpHandlers needs to be used by the http server to register routes.
|
||||
// This implements the receiver-side of "follows" and "bumps" the fast way.
|
||||
func (e *TaskEngine) GetHttpHandlers() http.Handler {
|
||||
root := mux.NewRouter()
|
||||
s := root.PathPrefix("/scheduler")
|
||||
f := s.PathPrefix("/follows")
|
||||
b := s.PathPrefix("/bump")
|
||||
for name, vsTmp := range e.follows {
|
||||
vs := vsTmp
|
||||
f.Path("/" + name + "/{tID}").Methods("GET").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
tIDString := mux.Vars(r)["tID"]
|
||||
func (e *TaskEngine) ApplyHttpHandlers(root gin.IRouter) {
|
||||
s := root.Group("/scheduler")
|
||||
f := s.Group("/follows")
|
||||
b := s.Group("/bump")
|
||||
for name, vs := range e.follows {
|
||||
name, vs := name, vs
|
||||
f.GET("/"+name+"/:tID", func(c *gin.Context) {
|
||||
tIDString := c.Param("tID")
|
||||
tID, err := strconv.Atoi(tIDString)
|
||||
if err != nil {
|
||||
w.WriteHeader(401)
|
||||
fmt.Fprint(w, err.Error())
|
||||
c.JSON(401, map[string]any{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
taskAdded := false
|
||||
for _, vTmp := range vs {
|
||||
v := vTmp
|
||||
taskAdded = taskAdded || v.f(TaskID(tID), v.h.AddTask)
|
||||
b, err := v.f(TaskID(tID), v.h.AddTask)
|
||||
if err != nil {
|
||||
log.Errorw("Follow attempt failed", "error", err, "from", name, "to", v.name)
|
||||
}
|
||||
taskAdded = taskAdded || b
|
||||
}
|
||||
if taskAdded {
|
||||
e.tryAllWork <- true
|
||||
w.WriteHeader(200)
|
||||
c.Status(200)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(202) // NOTE: 202 for "accepted" but not worked.
|
||||
c.Status(202) // NOTE: 202 for "accepted" but not worked.
|
||||
})
|
||||
}
|
||||
for _, hTmp := range e.handlers {
|
||||
h := hTmp
|
||||
b.Path("/" + h.Name + "/{tID}").Methods("GET").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
tIDString := mux.Vars(r)["tID"]
|
||||
b.GET("/"+h.Name+"/:tID", func(c *gin.Context) {
|
||||
tIDString := c.Param("tID")
|
||||
tID, err := strconv.Atoi(tIDString)
|
||||
if err != nil {
|
||||
w.WriteHeader(401)
|
||||
fmt.Fprint(w, err.Error())
|
||||
c.JSON(401, map[string]any{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
// We NEED to block while trying to deliver
|
||||
// this work to ease the network impact.
|
||||
if h.considerWork("bump", []TaskID{TaskID(tID)}) {
|
||||
w.WriteHeader(200)
|
||||
c.Status(200)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(202) // NOTE: 202 for "accepted" but not worked.
|
||||
c.Status(202) // NOTE: 202 for "accepted" but not worked.
|
||||
})
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
func (e *TaskEngine) bump(taskType string) {
|
||||
@ -394,10 +399,13 @@ func (e *TaskEngine) resourcesInUse() resources.Resources {
|
||||
tmp.Cpu -= int(ct) * t.Cost.Cpu
|
||||
tmp.Gpu -= float64(ct) * t.Cost.Gpu
|
||||
tmp.Ram -= uint64(ct) * t.Cost.Ram
|
||||
if len(t.Cost.GpuRam) == 0 {
|
||||
continue
|
||||
}
|
||||
for i := int32(0); i < ct; i++ {
|
||||
for grIdx, j := range tmp.GpuRam {
|
||||
if j > t.Cost.GpuRam[0] {
|
||||
tmp.GpuRam[grIdx] = j - t.Cost.GpuRam[0]
|
||||
tmp.GpuRam[grIdx] = 0 // Only 1 per GPU. j - t.Cost.GpuRam[0]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
@ -3,6 +3,7 @@ package harmonytask
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
@ -10,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
"github.com/samber/lo"
|
||||
|
||||
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
|
||||
)
|
||||
@ -23,15 +25,14 @@ type taskTypeHandler struct {
|
||||
Count atomic.Int32
|
||||
}
|
||||
|
||||
func (h *taskTypeHandler) AddTask(extra func(TaskID, *harmonydb.Tx) bool) {
|
||||
func (h *taskTypeHandler) AddTask(extra func(TaskID, *harmonydb.Tx) (bool, error)) {
|
||||
var tID TaskID
|
||||
did, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) bool {
|
||||
did, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) (bool, error) {
|
||||
// create taskID (from DB)
|
||||
_, err := tx.Exec(`INSERT INTO harmony_task (name, added_by, posted_time)
|
||||
VALUES ($1, $2, CURRENT_TIMESTAMP) `, h.Name, h.TaskEngine.ownerID)
|
||||
if err != nil {
|
||||
log.Error("Could not insert into harmonyTask", err)
|
||||
return false
|
||||
return false, fmt.Errorf("could not insert into harmonyTask: %w", err)
|
||||
}
|
||||
err = tx.QueryRow("SELECT id FROM harmony_task ORDER BY update_time DESC LIMIT 1").Scan(&tID)
|
||||
if err != nil {
|
||||
@ -59,7 +60,7 @@ top:
|
||||
|
||||
// 1. Can we do any more of this task type?
|
||||
if h.Max > -1 && int(h.Count.Load()) == h.Max {
|
||||
log.Infow("did not accept task", "name", h.Name, "reason", "at max already")
|
||||
log.Debugw("did not accept task", "name", h.Name, "reason", "at max already")
|
||||
return false
|
||||
}
|
||||
|
||||
@ -69,7 +70,7 @@ top:
|
||||
// 2. Can we do any more work?
|
||||
err := h.AssertMachineHasCapacity()
|
||||
if err != nil {
|
||||
log.Info(err)
|
||||
log.Debugw("did not accept task", "name", h.Name, "reason", "at capacity already: "+err.Error())
|
||||
return false
|
||||
}
|
||||
|
||||
@ -80,7 +81,7 @@ top:
|
||||
return false
|
||||
}
|
||||
if tID == nil {
|
||||
log.Infow("did not accept task", "task_id", ids[0], "reason", "CanAccept() refused")
|
||||
log.Infow("did not accept task", "task_id", ids[0], "reason", "CanAccept() refused", "name", h.Name)
|
||||
return false
|
||||
}
|
||||
|
||||
@ -91,7 +92,7 @@ top:
|
||||
return false
|
||||
}
|
||||
if ct == 0 {
|
||||
log.Infow("did not accept task", "task_id", strconv.Itoa(int(*tID)), "reason", "already Taken")
|
||||
log.Infow("did not accept task", "task_id", strconv.Itoa(int(*tID)), "reason", "already Taken", "name", h.Name)
|
||||
var tryAgain = make([]TaskID, 0, len(ids)-1)
|
||||
for _, id := range ids {
|
||||
if id != *tID {
|
||||
@ -104,7 +105,7 @@ top:
|
||||
|
||||
go func() {
|
||||
h.Count.Add(1)
|
||||
log.Infow("Beginning work on Task", "id", *tID, "from", from, "type", h.Name)
|
||||
log.Infow("Beginning work on Task", "id", *tID, "from", from, "name", h.Name)
|
||||
|
||||
var done bool
|
||||
var doErr error
|
||||
@ -146,19 +147,18 @@ top:
|
||||
func (h *taskTypeHandler) recordCompletion(tID TaskID, workStart time.Time, done bool, doErr error) {
|
||||
workEnd := time.Now()
|
||||
|
||||
cm, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) bool {
|
||||
cm, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) (bool, error) {
|
||||
var postedTime time.Time
|
||||
err := tx.QueryRow(`SELECT posted_time FROM harmony_task WHERE id=$1`, tID).Scan(&postedTime)
|
||||
if err != nil {
|
||||
log.Error("Could not log completion: ", err)
|
||||
return false
|
||||
return false, fmt.Errorf("could not log completion: %w ", err)
|
||||
}
|
||||
result := "unspecified error"
|
||||
if done {
|
||||
_, err = tx.Exec("DELETE FROM harmony_task WHERE id=$1", tID)
|
||||
if err != nil {
|
||||
log.Error("Could not log completion: ", err)
|
||||
return false
|
||||
|
||||
return false, fmt.Errorf("could not log completion: %w", err)
|
||||
}
|
||||
result = ""
|
||||
} else {
|
||||
@ -171,8 +171,7 @@ func (h *taskTypeHandler) recordCompletion(tID TaskID, workStart time.Time, done
|
||||
err = tx.QueryRow(`SELECT count(*) FROM harmony_task_history
|
||||
WHERE task_id=$1 AND result=FALSE`, tID).Scan(&ct)
|
||||
if err != nil {
|
||||
log.Error("Could not read task history:", err)
|
||||
return false
|
||||
return false, fmt.Errorf("could not read task history: %w", err)
|
||||
}
|
||||
if ct >= h.MaxFailures {
|
||||
deleteTask = true
|
||||
@ -181,15 +180,13 @@ func (h *taskTypeHandler) recordCompletion(tID TaskID, workStart time.Time, done
|
||||
if deleteTask {
|
||||
_, err = tx.Exec("DELETE FROM harmony_task WHERE id=$1", tID)
|
||||
if err != nil {
|
||||
log.Error("Could not delete failed job: ", err)
|
||||
return false
|
||||
return false, fmt.Errorf("could not delete failed job: %w", err)
|
||||
}
|
||||
// Note: Extra Info is left laying around for later review & clean-up
|
||||
} else {
|
||||
_, err := tx.Exec(`UPDATE harmony_task SET owner_id=NULL WHERE id=$1`, tID)
|
||||
if err != nil {
|
||||
log.Error("Could not disown failed task: ", tID, err)
|
||||
return false
|
||||
return false, fmt.Errorf("could not disown failed task: %v %v", tID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -197,10 +194,9 @@ func (h *taskTypeHandler) recordCompletion(tID TaskID, workStart time.Time, done
|
||||
(task_id, name, posted, work_start, work_end, result, err)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)`, tID, h.Name, postedTime, workStart, workEnd, done, result)
|
||||
if err != nil {
|
||||
log.Error("Could not write history: ", err)
|
||||
return false
|
||||
return false, fmt.Errorf("could not write history: %w", err)
|
||||
}
|
||||
return true
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("Could not record transaction: ", err)
|
||||
@ -223,8 +219,12 @@ func (h *taskTypeHandler) AssertMachineHasCapacity() error {
|
||||
if r.Gpu-h.Cost.Gpu < 0 {
|
||||
return errors.New("Did not accept " + h.Name + " task: out of available GPU")
|
||||
}
|
||||
gpuRamSum := lo.Sum(h.Cost.GpuRam)
|
||||
if gpuRamSum == 0 {
|
||||
goto enoughGpuRam
|
||||
}
|
||||
for _, u := range r.GpuRam {
|
||||
if u > h.Cost.GpuRam[0] {
|
||||
if u >= gpuRamSum {
|
||||
goto enoughGpuRam
|
||||
}
|
||||
}
|
||||
@ -249,7 +249,11 @@ func (h *taskTypeHandler) triggerCompletionListeners(tID TaskID) {
|
||||
inProcessDefs := h.TaskEngine.follows[h.Name]
|
||||
inProcessFollowers := make([]string, len(inProcessDefs))
|
||||
for _, fs := range inProcessDefs {
|
||||
if fs.f(tID, fs.h.AddTask) {
|
||||
b, err := fs.f(tID, fs.h.AddTask)
|
||||
if err != nil {
|
||||
log.Error("Could not follow", "error", err, "from", h.Name, "to", fs.name)
|
||||
}
|
||||
if b {
|
||||
inProcessFollowers = append(inProcessFollowers, fs.h.Name)
|
||||
}
|
||||
}
|
||||
|
||||
@ -19,7 +19,11 @@ type Platform struct {
|
||||
func GetPlatforms() ([]*Platform, error) {
|
||||
var platformIds [maxPlatforms]C.cl_platform_id
|
||||
var nPlatforms C.cl_uint
|
||||
if err := C.clGetPlatformIDs(C.cl_uint(maxPlatforms), &platformIds[0], &nPlatforms); err != C.CL_SUCCESS {
|
||||
err := C.clGetPlatformIDs(C.cl_uint(maxPlatforms), &platformIds[0], &nPlatforms)
|
||||
if err == -1001 { // No platforms found
|
||||
return nil, nil
|
||||
}
|
||||
if err != C.CL_SUCCESS {
|
||||
return nil, toError(err)
|
||||
}
|
||||
platforms := make([]*Platform, nPlatforms)
|
||||
@ -68,7 +72,7 @@ func toError(code C.cl_int) error {
|
||||
type ErrOther int
|
||||
|
||||
func (e ErrOther) Error() string {
|
||||
return fmt.Sprintf("cl: error %d", int(e))
|
||||
return fmt.Sprintf("OpenCL: error %d", int(e))
|
||||
}
|
||||
|
||||
// Size of global device memory in bytes.
|
||||
|
||||
@ -54,11 +54,12 @@ func Register(db *harmonydb.DB, hostnameAndPort string) (*Reg, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not read from harmony_machines: %w", err)
|
||||
}
|
||||
gpuram := lo.Sum(reg.GpuRam)
|
||||
if len(ownerID) == 0 {
|
||||
err = db.QueryRow(ctx, `INSERT INTO harmony_machines
|
||||
(host_and_port, cpu, ram, gpu, gpuram) VALUES
|
||||
($1,$2,$3,$4,$5) RETURNING id`,
|
||||
hostnameAndPort, reg.Cpu, reg.Ram, reg.Gpu, reg.GpuRam).Scan(®.Resources.MachineID)
|
||||
hostnameAndPort, reg.Cpu, reg.Ram, reg.Gpu, gpuram).Scan(®.Resources.MachineID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -67,7 +68,7 @@ func Register(db *harmonydb.DB, hostnameAndPort string) (*Reg, error) {
|
||||
reg.MachineID = ownerID[0]
|
||||
_, err := db.Exec(ctx, `UPDATE harmony_machines SET
|
||||
cpu=$1, ram=$2, gpu=$3, gpuram=$4 WHERE id=$6`,
|
||||
reg.Cpu, reg.Ram, reg.Gpu, reg.GpuRam, reg.Resources.MachineID)
|
||||
reg.Cpu, reg.Ram, reg.Gpu, gpuram, reg.Resources.MachineID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -132,7 +133,7 @@ func getResources() (res Resources, err error) {
|
||||
}
|
||||
all := strings.ToLower(strings.Join(gpus, ","))
|
||||
if len(gpus) > 1 || strings.Contains(all, "ati") || strings.Contains(all, "nvidia") {
|
||||
res.Gpu = 1
|
||||
res.Gpu = float64(len(gpus))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -341,6 +341,29 @@ func Repo(r repo.Repo) Option {
|
||||
}
|
||||
}
|
||||
|
||||
func Provider(r repo.Repo) Option {
|
||||
return func(settings *Settings) error {
|
||||
lr, err := r.Lock(settings.nodeType)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c, err := lr.Config()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = c
|
||||
return Options(
|
||||
func(s *Settings) error { s.Base = true; return nil }, // mark Base as applied
|
||||
ApplyIf(func(s *Settings) bool { return s.Config },
|
||||
Error(errors.New("the Base() option must be set before Config option")),
|
||||
),
|
||||
Override(new(repo.LockedRepo), modules.LockedRepo(lr)), // module handles closing
|
||||
//ApplyIf(IsType(repo.WdPost), ConfigWdPost(c)),
|
||||
//ApplyIf(IsType(repo.WinPost), ConfigWinPost(c)),
|
||||
)(settings)
|
||||
}
|
||||
}
|
||||
|
||||
type StopFunc func(context.Context) error
|
||||
|
||||
// New builds and starts new Filecoin node
|
||||
|
||||
@ -185,6 +185,30 @@ func (worker) APIInfoEnvVars() (primary string, fallbacks []string, deprecated [
|
||||
return "WORKER_API_INFO", nil, nil
|
||||
}
|
||||
|
||||
type provider struct{}
|
||||
|
||||
var Provider provider
|
||||
|
||||
func (provider) Type() string {
|
||||
return "Provider"
|
||||
}
|
||||
|
||||
func (provider) Config() interface{} {
|
||||
return &struct{}{}
|
||||
}
|
||||
|
||||
func (provider) APIFlags() []string {
|
||||
return []string{"provider-api-url"}
|
||||
}
|
||||
|
||||
func (provider) RepoFlags() []string {
|
||||
return []string{"provider-repo"}
|
||||
}
|
||||
|
||||
func (provider) APIInfoEnvVars() (primary string, fallbacks []string, deprecated []string) {
|
||||
return "PROVIDER_API_INFO", nil, nil
|
||||
}
|
||||
|
||||
var Wallet wallet
|
||||
|
||||
type wallet struct {
|
||||
|
||||
12
scripts/lotus-provider.service
Normal file
12
scripts/lotus-provider.service
Normal file
@ -0,0 +1,12 @@
|
||||
[Unit]
|
||||
Description=Lotus Provider
|
||||
After=network.target
|
||||
After=lotus-daemon.service
|
||||
|
||||
[Service]
|
||||
ExecStart=/usr/local/bin/lotus-provider run
|
||||
Environment=GOLOG_FILE="/var/log/lotus/provider.log"
|
||||
Environment=GOLOG_LOG_FMT="json"
|
||||
LimitNOFILE=1000000
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Loading…
Reference in New Issue
Block a user