Merge pull request #464 from filecoin-project/feat/rest-import

REST file import endpoint

api/api.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-filestore"
 	cbor "github.com/ipfs/go-ipld-cbor"
@@ -14,7 +15,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/address"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
-	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 )
 
 func init() {
@@ -23,8 +23,8 @@ func init() {
 
 type Common interface {
 	// Auth
-	AuthVerify(ctx context.Context, token string) ([]string, error)
+	AuthVerify(ctx context.Context, token string) ([]Permission, error)
-	AuthNew(ctx context.Context, perms []string) ([]byte, error)
+	AuthNew(ctx context.Context, perms []Permission) ([]byte, error)
 
 	// network
 

api/permissioned.go

@@ -11,19 +11,21 @@ type permKey int
 
 var permCtxKey permKey
 
+type Permission = string
+
 const (
 	// When changing these, update docs/API.md too
 
-	PermRead  = "read" // default
+	PermRead  Permission = "read" // default
-	PermWrite = "write"
+	PermWrite Permission = "write"
-	PermSign  = "sign"  // Use wallet keys for signing
+	PermSign  Permission = "sign"  // Use wallet keys for signing
-	PermAdmin = "admin" // Manage permissions
+	PermAdmin Permission = "admin" // Manage permissions
 )
 
-var AllPermissions = []string{PermRead, PermWrite, PermSign, PermAdmin}
+var AllPermissions = []Permission{PermRead, PermWrite, PermSign, PermAdmin}
-var defaultPerms = []string{PermRead}
+var defaultPerms = []Permission{PermRead}
 
-func WithPerm(ctx context.Context, perms []string) context.Context {
+func WithPerm(ctx context.Context, perms []Permission) context.Context {
 	return context.WithValue(ctx, permCtxKey, perms)
 }
 
@@ -41,13 +43,27 @@ func PermissionedFullAPI(a FullNode) FullNode {
 	return &out
 }
 
+func HasPerm(ctx context.Context, perm Permission) bool {
+	callerPerms, ok := ctx.Value(permCtxKey).([]Permission)
+	if !ok {
+		callerPerms = defaultPerms
+	}
+
+	for _, callerPerm := range callerPerms {
+		if callerPerm == perm {
+			return true
+		}
+	}
+	return false
+}
+
 func permissionedAny(in interface{}, out interface{}) {
 	rint := reflect.ValueOf(out).Elem()
 	ra := reflect.ValueOf(in)
 
 	for f := 0; f < rint.NumField(); f++ {
 		field := rint.Type().Field(f)
-		requiredPerm := field.Tag.Get("perm")
+		requiredPerm := Permission(field.Tag.Get("perm"))
 		if requiredPerm == "" {
 			panic("missing 'perm' tag on " + field.Name)
 		}
@@ -68,16 +84,9 @@ func permissionedAny(in interface{}, out interface{}) {
 
 		rint.Field(f).Set(reflect.MakeFunc(field.Type, func(args []reflect.Value) (results []reflect.Value) {
 			ctx := args[0].Interface().(context.Context)
-			callerPerms, ok := ctx.Value(permCtxKey).([]string)
+			if HasPerm(ctx, requiredPerm) {
-			if !ok {
-				callerPerms = defaultPerms
-			}
-
-			for _, callerPerm := range callerPerms {
-				if callerPerm == requiredPerm {
 				return fn.Call(args)
 			}
-			}
 
 			err := xerrors.Errorf("missing permission to invoke '%s' (need '%s')", field.Name, requiredPerm)
 			rerr := reflect.ValueOf(&err).Elem()

api/struct.go

@@ -18,8 +18,8 @@ var _ = AllPermissions
 
 type CommonStruct struct {
 	Internal struct {
-		AuthVerify func(ctx context.Context, token string) ([]string, error) `perm:"read"`
+		AuthVerify func(ctx context.Context, token string) ([]Permission, error) `perm:"read"`
-		AuthNew    func(ctx context.Context, perms []string) ([]byte, error) `perm:"admin"`
+		AuthNew    func(ctx context.Context, perms []Permission) ([]byte, error) `perm:"admin"`
 
 		NetConnectedness func(context.Context, peer.ID) (network.Connectedness, error) `perm:"read"`
 		NetPeers         func(context.Context) ([]peer.AddrInfo, error)                `perm:"read"`
@@ -76,7 +76,7 @@ type FullNodeStruct struct {
 
 		MpoolGetNonce func(context.Context, address.Address) (uint64, error) `perm:"read"`
 
-		ClientImport      func(ctx context.Context, path string) (cid.Cid, error)                                                                     `perm:"write"`
+		ClientImport      func(ctx context.Context, path string) (cid.Cid, error)                                                                     `perm:"admin"`
 		ClientListImports func(ctx context.Context) ([]Import, error)                                                                                 `perm:"write"`
 		ClientHasLocal    func(ctx context.Context, root cid.Cid) (bool, error)                                                                       `perm:"write"`
 		ClientFindData    func(ctx context.Context, root cid.Cid) ([]QueryOffer, error)                                                               `perm:"read"`
@@ -132,11 +132,11 @@ type StorageMinerStruct struct {
 	}
 }
 
-func (c *CommonStruct) AuthVerify(ctx context.Context, token string) ([]string, error) {
+func (c *CommonStruct) AuthVerify(ctx context.Context, token string) ([]Permission, error) {
 	return c.Internal.AuthVerify(ctx, token)
 }
 
-func (c *CommonStruct) AuthNew(ctx context.Context, perms []string) ([]byte, error) {
+func (c *CommonStruct) AuthNew(ctx context.Context, perms []Permission) ([]byte, error) {
 	return c.Internal.AuthNew(ctx, perms)
 }
 

cmd/lotus-storage-miner/info.go

@@ -6,10 +6,10 @@ import (
 
 	"gopkg.in/urfave/cli.v2"
 
+	sectorstate "github.com/filecoin-project/go-sectorbuilder/sealing_state"
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
-	sectorstate "github.com/filecoin-project/go-sectorbuilder/sealing_state"
 )
 
 var infoCmd = &cli.Command{

cmd/lotus/rpc.go

@@ -2,17 +2,21 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
 	"os/signal"
 	"syscall"
 
+	"github.com/ipfs/go-cid"
+	logging "github.com/ipfs/go-log"
+
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/lib/auth"
 	"github.com/filecoin-project/lotus/lib/jsonrpc"
 	"github.com/filecoin-project/lotus/node"
-	logging "github.com/ipfs/go-log"
+	"github.com/filecoin-project/lotus/node/impl"
 )
 
 var log = logging.Logger("main")
@@ -28,6 +32,13 @@ func serveRPC(a api.FullNode, stop node.StopFunc, addr string) error {
 
 	http.Handle("/rpc/v0", ah)
 
+	importAH := &auth.Handler{
+		Verify: a.AuthVerify,
+		Next:   handleImport(a.(*impl.FullNodeAPI)),
+	}
+
+	http.Handle("/rest/v0/import", importAH)
+
 	srv := &http.Server{Addr: addr, Handler: http.DefaultServeMux}
 
 	sigChan := make(chan os.Signal, 2)
@@ -44,3 +55,30 @@ func serveRPC(a api.FullNode, stop node.StopFunc, addr string) error {
 
 	return srv.ListenAndServe()
 }
+
+func handleImport(a *impl.FullNodeAPI) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != "PUT" {
+			w.WriteHeader(404)
+			return
+		}
+		if !api.HasPerm(r.Context(), api.PermWrite) {
+			w.WriteHeader(401)
+			json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing write permission"})
+			return
+		}
+
+		c, err := a.ClientImportLocal(r.Context(), r.Body)
+		if err != nil {
+			w.WriteHeader(500)
+			json.NewEncoder(w).Encode(struct{ Error string }{err.Error()})
+			return
+		}
+		w.WriteHeader(200)
+		err = json.NewEncoder(w).Encode(struct{ Cid cid.Cid }{c})
+		if err != nil {
+			log.Errorf("/rest/v0/import: Writing response failed: %+v", err)
+			return
+		}
+	}
+}

docs/API.md

@@ -4,8 +4,13 @@ TODO: make this into a nicer doc
 
 By default `127.0.0.1:1234` - daemon stores the api endpoint multiaddr in `~/.lotus/api`
 
-* `http://[api:port]/rpc/v0` - jsonrpc http endpoint
+* `http://[api:port]/rpc/v0` - JsonRPC http endpoint
-* `ws://[api:port]/rpc/v0` - jsonrpc websocket endpoint
+* `ws://[api:port]/rpc/v0` - JsonRPC websocket endpoint
+* `PUT http://[api:port]/rest/v0/import` - import file to the node repo
+  * Requires write permission
+
+For JsonRPC interface definition see `api/api.go`. Required permissions are
+defined in `api/struct.go`
 
 ### Auth:
 
@@ -23,3 +28,5 @@ Payload:
   "Allow": ["read", "write", ...]
 }
 ```
+
+Admin token is stored in `~/.lotus/token`

lib/auth/handler.go

@@ -12,7 +12,7 @@ import (
 var log = logging.Logger("auth")
 
 type Handler struct {
-	Verify func(ctx context.Context, token string) ([]string, error)
+	Verify func(ctx context.Context, token string) ([]api.Permission, error)
 	Next   http.HandlerFunc
 }
 

node/impl/client/client.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"golang.org/x/xerrors"
+	"io"
 	"os"
 
 	"github.com/ipfs/go-blockservice"
@@ -213,6 +214,30 @@ func (a *API) ClientImport(ctx context.Context, path string) (cid.Cid, error) {
 	return nd.Cid(), bufferedDS.Commit()
 }
 
+func (a *API) ClientImportLocal(ctx context.Context, f io.Reader) (cid.Cid, error) {
+	file := files.NewReaderFile(f)
+
+	bufferedDS := ipld.NewBufferedDAG(ctx, a.LocalDAG)
+
+	params := ihelper.DagBuilderParams{
+		Maxlinks:   build.UnixfsLinksPerLevel,
+		RawLeaves:  true,
+		CidBuilder: nil,
+		Dagserv:    bufferedDS,
+	}
+
+	db, err := params.New(chunker.NewSizeSplitter(file, int64(build.UnixfsChunkSize)))
+	if err != nil {
+		return cid.Undef, err
+	}
+	nd, err := balanced.Layout(db)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	return nd.Cid(), bufferedDS.Commit()
+}
+
 func (a *API) ClientListImports(ctx context.Context) ([]api.Import, error) {
 	if a.Filestore == nil {
 		return nil, errors.New("listing imports is not supported with in-memory dag yet")

node/impl/common.go

@@ -27,7 +27,7 @@ type jwtPayload struct {
 	Allow []string
 }
 
-func (a *CommonAPI) AuthVerify(ctx context.Context, token string) ([]string, error) {
+func (a *CommonAPI) AuthVerify(ctx context.Context, token string) ([]api.Permission, error) {
 	var payload jwtPayload
 	if _, err := jwt.Verify([]byte(token), (*jwt.HMACSHA)(a.APISecret), &payload); err != nil {
 		return nil, xerrors.Errorf("JWT Verification failed: %w", err)
@@ -36,7 +36,7 @@ func (a *CommonAPI) AuthVerify(ctx context.Context, token string) ([]string, err
 	return payload.Allow, nil
 }
 
-func (a *CommonAPI) AuthNew(ctx context.Context, perms []string) ([]byte, error) {
+func (a *CommonAPI) AuthNew(ctx context.Context, perms []api.Permission) ([]byte, error) {
 	p := jwtPayload{
 		Allow: perms, // TODO: consider checking validity
 	}