From aab3bd617a99e10bbc4f34c3db087d338ac8a3dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 23 Oct 2019 11:18:22 +0200 Subject: [PATCH] REST file import endpoint --- api/api.go | 6 ++--- api/permissioned.go | 43 ++++++++++++++++++++------------- api/struct.go | 10 ++++---- cmd/lotus-storage-miner/info.go | 2 +- cmd/lotus/rpc.go | 40 +++++++++++++++++++++++++++++- docs/API.md | 11 +++++++-- lib/auth/handler.go | 2 +- node/impl/client/client.go | 25 +++++++++++++++++++ node/impl/common.go | 4 +-- 9 files changed, 111 insertions(+), 32 deletions(-) diff --git a/api/api.go b/api/api.go index c7406ee52..5e6373fac 100644 --- a/api/api.go +++ b/api/api.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" "github.com/ipfs/go-filestore" cbor "github.com/ipfs/go-ipld-cbor" @@ -14,7 +15,6 @@ import ( "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" ) func init() { @@ -23,8 +23,8 @@ func init() { type Common interface { // Auth - AuthVerify(ctx context.Context, token string) ([]string, error) - AuthNew(ctx context.Context, perms []string) ([]byte, error) + AuthVerify(ctx context.Context, token string) ([]Permission, error) + AuthNew(ctx context.Context, perms []Permission) ([]byte, error) // network diff --git a/api/permissioned.go b/api/permissioned.go index 71c93f8e7..44a967707 100644 --- a/api/permissioned.go +++ b/api/permissioned.go @@ -11,19 +11,21 @@ type permKey int var permCtxKey permKey +type Permission = string + const ( // When changing these, update docs/API.md too - PermRead = "read" // default - PermWrite = "write" - PermSign = "sign" // Use wallet keys for signing - PermAdmin = "admin" // Manage permissions + PermRead Permission = "read" // default + PermWrite Permission = "write" + PermSign Permission = "sign" // Use wallet keys for signing + PermAdmin Permission = "admin" // Manage permissions ) -var AllPermissions = []string{PermRead, PermWrite, PermSign, PermAdmin} -var defaultPerms = []string{PermRead} +var AllPermissions = []Permission{PermRead, PermWrite, PermSign, PermAdmin} +var defaultPerms = []Permission{PermRead} -func WithPerm(ctx context.Context, perms []string) context.Context { +func WithPerm(ctx context.Context, perms []Permission) context.Context { return context.WithValue(ctx, permCtxKey, perms) } @@ -41,13 +43,27 @@ func PermissionedFullAPI(a FullNode) FullNode { return &out } +func HasPerm(ctx context.Context, perm Permission) bool { + callerPerms, ok := ctx.Value(permCtxKey).([]Permission) + if !ok { + callerPerms = defaultPerms + } + + for _, callerPerm := range callerPerms { + if callerPerm == perm { + return true + } + } + return false +} + func permissionedAny(in interface{}, out interface{}) { rint := reflect.ValueOf(out).Elem() ra := reflect.ValueOf(in) for f := 0; f < rint.NumField(); f++ { field := rint.Type().Field(f) - requiredPerm := field.Tag.Get("perm") + requiredPerm := Permission(field.Tag.Get("perm")) if requiredPerm == "" { panic("missing 'perm' tag on " + field.Name) } @@ -68,15 +84,8 @@ func permissionedAny(in interface{}, out interface{}) { rint.Field(f).Set(reflect.MakeFunc(field.Type, func(args []reflect.Value) (results []reflect.Value) { ctx := args[0].Interface().(context.Context) - callerPerms, ok := ctx.Value(permCtxKey).([]string) - if !ok { - callerPerms = defaultPerms - } - - for _, callerPerm := range callerPerms { - if callerPerm == requiredPerm { - return fn.Call(args) - } + if HasPerm(ctx, requiredPerm) { + return fn.Call(args) } err := xerrors.Errorf("missing permission to invoke '%s' (need '%s')", field.Name, requiredPerm) diff --git a/api/struct.go b/api/struct.go index 632488e23..b4bd36758 100644 --- a/api/struct.go +++ b/api/struct.go @@ -18,8 +18,8 @@ var _ = AllPermissions type CommonStruct struct { Internal struct { - AuthVerify func(ctx context.Context, token string) ([]string, error) `perm:"read"` - AuthNew func(ctx context.Context, perms []string) ([]byte, error) `perm:"admin"` + AuthVerify func(ctx context.Context, token string) ([]Permission, error) `perm:"read"` + AuthNew func(ctx context.Context, perms []Permission) ([]byte, error) `perm:"admin"` NetConnectedness func(context.Context, peer.ID) (network.Connectedness, error) `perm:"read"` NetPeers func(context.Context) ([]peer.AddrInfo, error) `perm:"read"` @@ -76,7 +76,7 @@ type FullNodeStruct struct { MpoolGetNonce func(context.Context, address.Address) (uint64, error) `perm:"read"` - ClientImport func(ctx context.Context, path string) (cid.Cid, error) `perm:"write"` + ClientImport func(ctx context.Context, path string) (cid.Cid, error) `perm:"admin"` ClientListImports func(ctx context.Context) ([]Import, error) `perm:"write"` ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` ClientFindData func(ctx context.Context, root cid.Cid) ([]QueryOffer, error) `perm:"read"` @@ -132,11 +132,11 @@ type StorageMinerStruct struct { } } -func (c *CommonStruct) AuthVerify(ctx context.Context, token string) ([]string, error) { +func (c *CommonStruct) AuthVerify(ctx context.Context, token string) ([]Permission, error) { return c.Internal.AuthVerify(ctx, token) } -func (c *CommonStruct) AuthNew(ctx context.Context, perms []string) ([]byte, error) { +func (c *CommonStruct) AuthNew(ctx context.Context, perms []Permission) ([]byte, error) { return c.Internal.AuthNew(ctx, perms) } diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 0f34d7b28..436fc4686 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -6,10 +6,10 @@ import ( "gopkg.in/urfave/cli.v2" + sectorstate "github.com/filecoin-project/go-sectorbuilder/sealing_state" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" - sectorstate "github.com/filecoin-project/go-sectorbuilder/sealing_state" ) var infoCmd = &cli.Command{ diff --git a/cmd/lotus/rpc.go b/cmd/lotus/rpc.go index d3c1d8295..18337d3e4 100644 --- a/cmd/lotus/rpc.go +++ b/cmd/lotus/rpc.go @@ -2,17 +2,21 @@ package main import ( "context" + "encoding/json" "net/http" _ "net/http/pprof" "os" "os/signal" "syscall" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/lib/auth" "github.com/filecoin-project/lotus/lib/jsonrpc" "github.com/filecoin-project/lotus/node" - logging "github.com/ipfs/go-log" + "github.com/filecoin-project/lotus/node/impl" ) var log = logging.Logger("main") @@ -28,6 +32,13 @@ func serveRPC(a api.FullNode, stop node.StopFunc, addr string) error { http.Handle("/rpc/v0", ah) + importAH := &auth.Handler{ + Verify: a.AuthVerify, + Next: handleImport(a.(*impl.FullNodeAPI)), + } + + http.Handle("/rest/v0/import", importAH) + srv := &http.Server{Addr: addr, Handler: http.DefaultServeMux} sigChan := make(chan os.Signal, 2) @@ -44,3 +55,30 @@ func serveRPC(a api.FullNode, stop node.StopFunc, addr string) error { return srv.ListenAndServe() } + +func handleImport(a *impl.FullNodeAPI) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + if r.Method != "PUT" { + w.WriteHeader(404) + return + } + if !api.HasPerm(r.Context(), api.PermWrite) { + w.WriteHeader(401) + json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing write permission"}) + return + } + + c, err := a.ClientImportLocal(r.Context(), r.Body) + if err != nil { + w.WriteHeader(500) + json.NewEncoder(w).Encode(struct{ Error string }{err.Error()}) + return + } + w.WriteHeader(200) + err = json.NewEncoder(w).Encode(struct{ Cid cid.Cid }{c}) + if err != nil { + log.Errorf("/rest/v0/import: Writing response failed: %+v", err) + return + } + } +} diff --git a/docs/API.md b/docs/API.md index 2b244b38b..eada8460a 100644 --- a/docs/API.md +++ b/docs/API.md @@ -4,8 +4,13 @@ TODO: make this into a nicer doc By default `127.0.0.1:1234` - daemon stores the api endpoint multiaddr in `~/.lotus/api` -* `http://[api:port]/rpc/v0` - jsonrpc http endpoint -* `ws://[api:port]/rpc/v0` - jsonrpc websocket endpoint +* `http://[api:port]/rpc/v0` - JsonRPC http endpoint +* `ws://[api:port]/rpc/v0` - JsonRPC websocket endpoint +* `PUT http://[api:port]/rest/v0/import` - import file to the node repo + * Requires write permission + +For JsonRPC interface definition see `api/api.go`. Required permissions are +defined in `api/struct.go` ### Auth: @@ -23,3 +28,5 @@ Payload: "Allow": ["read", "write", ...] } ``` + +Admin token is stored in `~/.lotus/token` diff --git a/lib/auth/handler.go b/lib/auth/handler.go index c47a4d789..67fc1ac9b 100644 --- a/lib/auth/handler.go +++ b/lib/auth/handler.go @@ -12,7 +12,7 @@ import ( var log = logging.Logger("auth") type Handler struct { - Verify func(ctx context.Context, token string) ([]string, error) + Verify func(ctx context.Context, token string) ([]api.Permission, error) Next http.HandlerFunc } diff --git a/node/impl/client/client.go b/node/impl/client/client.go index a06e211ef..6269fb932 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -4,6 +4,7 @@ import ( "context" "errors" "golang.org/x/xerrors" + "io" "os" "github.com/ipfs/go-blockservice" @@ -213,6 +214,30 @@ func (a *API) ClientImport(ctx context.Context, path string) (cid.Cid, error) { return nd.Cid(), bufferedDS.Commit() } +func (a *API) ClientImportLocal(ctx context.Context, f io.Reader) (cid.Cid, error) { + file := files.NewReaderFile(f) + + bufferedDS := ipld.NewBufferedDAG(ctx, a.LocalDAG) + + params := ihelper.DagBuilderParams{ + Maxlinks: build.UnixfsLinksPerLevel, + RawLeaves: true, + CidBuilder: nil, + Dagserv: bufferedDS, + } + + db, err := params.New(chunker.NewSizeSplitter(file, int64(build.UnixfsChunkSize))) + if err != nil { + return cid.Undef, err + } + nd, err := balanced.Layout(db) + if err != nil { + return cid.Undef, err + } + + return nd.Cid(), bufferedDS.Commit() +} + func (a *API) ClientListImports(ctx context.Context) ([]api.Import, error) { if a.Filestore == nil { return nil, errors.New("listing imports is not supported with in-memory dag yet") diff --git a/node/impl/common.go b/node/impl/common.go index 26b0eef44..74f614ee3 100644 --- a/node/impl/common.go +++ b/node/impl/common.go @@ -27,7 +27,7 @@ type jwtPayload struct { Allow []string } -func (a *CommonAPI) AuthVerify(ctx context.Context, token string) ([]string, error) { +func (a *CommonAPI) AuthVerify(ctx context.Context, token string) ([]api.Permission, error) { var payload jwtPayload if _, err := jwt.Verify([]byte(token), (*jwt.HMACSHA)(a.APISecret), &payload); err != nil { return nil, xerrors.Errorf("JWT Verification failed: %w", err) @@ -36,7 +36,7 @@ func (a *CommonAPI) AuthVerify(ctx context.Context, token string) ([]string, err return payload.Allow, nil } -func (a *CommonAPI) AuthNew(ctx context.Context, perms []string) ([]byte, error) { +func (a *CommonAPI) AuthNew(ctx context.Context, perms []api.Permission) ([]byte, error) { p := jwtPayload{ Allow: perms, // TODO: consider checking validity }