lotus/lib/jsonrpc/websocket.go

549 lines
12 KiB
Go
Raw Normal View History

2019-07-12 17:12:38 +00:00
package jsonrpc
import (
"context"
"encoding/json"
"errors"
"io"
2019-07-15 16:21:48 +00:00
"io/ioutil"
2019-07-22 18:13:41 +00:00
"reflect"
2019-07-15 16:21:48 +00:00
"sync"
2019-07-22 18:13:41 +00:00
"sync/atomic"
"time"
2019-07-12 17:12:38 +00:00
"github.com/gorilla/websocket"
2020-01-20 18:21:16 +00:00
"golang.org/x/xerrors"
2019-07-12 17:12:38 +00:00
)
2019-07-15 16:21:48 +00:00
const wsCancel = "xrpc.cancel"
2019-07-22 18:13:41 +00:00
const chValue = "xrpc.ch.val"
const chClose = "xrpc.ch.close"
2019-07-15 16:21:48 +00:00
2019-07-12 17:12:38 +00:00
type frame struct {
// common
Jsonrpc string `json:"jsonrpc"`
ID *int64 `json:"id,omitempty"`
Meta map[string]string `json:"meta,omitempty"`
2019-07-12 17:12:38 +00:00
// request
2019-07-15 16:32:43 +00:00
Method string `json:"method,omitempty"`
Params []param `json:"params,omitempty"`
2019-07-12 17:12:38 +00:00
// response
2019-07-22 20:28:40 +00:00
Result json.RawMessage `json:"result,omitempty"`
Error *respError `json:"error,omitempty"`
2019-07-12 17:12:38 +00:00
}
type outChanReg struct {
reqID int64
chID uint64
ch reflect.Value
}
type wsConn struct {
// outside params
conn *websocket.Conn
connFactory func() (*websocket.Conn, error)
reconnectInterval time.Duration
handler handlers
requests <-chan clientRequest
stop <-chan struct{}
exiting chan struct{}
// incoming messages
incoming chan io.Reader
incomingErr error
// outgoing messages
writeLk sync.Mutex
// ////
// Client related
// inflight are requests we've sent to the remote
inflight map[int64]clientRequest
// chanHandlers is a map of client-side channel handlers
chanHandlers map[uint64]func(m []byte, ok bool)
// ////
// Server related
// handling are the calls we handle
handling map[int64]context.CancelFunc
handlingLk sync.Mutex
spawnOutChanHandlerOnce sync.Once
// chanCtr is a counter used for identifying output channels on the server side
chanCtr uint64
registerCh chan outChanReg
}
// //
// WebSocket Message utils //
// //
// nextMessage wait for one message and puts it to the incoming channel
func (c *wsConn) nextMessage() {
msgType, r, err := c.conn.NextReader()
if err != nil {
c.incomingErr = err
close(c.incoming)
return
}
if msgType != websocket.BinaryMessage && msgType != websocket.TextMessage {
c.incomingErr = errors.New("unsupported message type")
close(c.incoming)
return
2019-07-12 17:12:38 +00:00
}
c.incoming <- r
}
2019-07-12 17:12:38 +00:00
// nextWriter waits for writeLk and invokes the cb callback with WS message
// writer when the lock is acquired
func (c *wsConn) nextWriter(cb func(io.Writer)) {
c.writeLk.Lock()
defer c.writeLk.Unlock()
2019-07-15 16:21:48 +00:00
wcl, err := c.conn.NextWriter(websocket.TextMessage)
if err != nil {
log.Error("handle me:", err)
return
}
2019-07-15 16:21:48 +00:00
cb(wcl)
2019-07-15 16:21:48 +00:00
if err := wcl.Close(); err != nil {
log.Error("handle me:", err)
return
2019-07-15 16:21:48 +00:00
}
}
2019-07-15 16:21:48 +00:00
func (c *wsConn) sendRequest(req request) {
c.writeLk.Lock()
if err := c.conn.WriteJSON(req); err != nil {
log.Error("handle me:", err)
c.writeLk.Unlock()
return
2019-07-22 18:13:41 +00:00
}
c.writeLk.Unlock()
}
2019-07-22 18:13:41 +00:00
// //
// Output channels //
// //
2019-07-22 18:13:41 +00:00
// handleOutChans handles channel communication on the server side
// (forwards channel messages to client)
func (c *wsConn) handleOutChans() {
regV := reflect.ValueOf(c.registerCh)
2020-01-20 18:21:16 +00:00
exitV := reflect.ValueOf(c.exiting)
2019-07-22 18:13:41 +00:00
cases := []reflect.SelectCase{
{ // registration chan always 0
Dir: reflect.SelectRecv,
Chan: regV,
},
2020-01-20 18:21:16 +00:00
{ // exit chan always 1
Dir: reflect.SelectRecv,
Chan: exitV,
},
}
2020-01-20 18:21:16 +00:00
internal := len(cases)
var caseToID []uint64
2019-07-22 18:13:41 +00:00
for {
chosen, val, ok := reflect.Select(cases)
2019-07-22 18:13:41 +00:00
2020-01-20 18:21:16 +00:00
switch chosen {
case 0: // registration channel
if !ok {
// control channel closed - signals closed connection
2020-01-20 18:21:16 +00:00
// This shouldn't happen, instead the exiting channel should get closed
log.Warn("control channel closed")
return
2019-07-22 18:13:41 +00:00
}
registration := val.Interface().(outChanReg)
caseToID = append(caseToID, registration.chID)
cases = append(cases, reflect.SelectCase{
Dir: reflect.SelectRecv,
Chan: registration.ch,
})
c.nextWriter(func(w io.Writer) {
resp := &response{
Jsonrpc: "2.0",
ID: registration.reqID,
Result: registration.chID,
}
if err := json.NewEncoder(w).Encode(resp); err != nil {
log.Error(err)
return
}
})
2020-01-20 18:21:16 +00:00
continue
case 1: // exiting channel
if !ok {
// exiting channel closed - signals closed connection
//
// We're not closing any channels as we're on receiving end.
// Also, context cancellation below should take care of any running
// requests
return
}
log.Warn("exiting channel received a message")
continue
}
2019-07-22 18:13:41 +00:00
if !ok {
// Output channel closed, cleanup, and tell remote that this happened
2020-01-20 18:21:16 +00:00
n := len(cases) - 1
if n > 0 {
cases[chosen] = cases[n]
2020-01-20 18:21:16 +00:00
caseToID[chosen-internal] = caseToID[n-internal]
2019-07-22 18:13:41 +00:00
}
2020-01-20 18:21:16 +00:00
id := caseToID[chosen-internal]
cases = cases[:n]
2020-01-20 18:21:16 +00:00
caseToID = caseToID[:n-internal]
c.sendRequest(request{
2019-07-22 18:13:41 +00:00
Jsonrpc: "2.0",
2019-07-22 20:28:40 +00:00
ID: nil, // notification
Method: chClose,
Params: []param{{v: reflect.ValueOf(id)}},
2019-07-22 18:13:41 +00:00
})
continue
2019-07-22 18:13:41 +00:00
}
// forward message
c.sendRequest(request{
Jsonrpc: "2.0",
ID: nil, // notification
Method: chValue,
2020-01-20 18:21:16 +00:00
Params: []param{{v: reflect.ValueOf(caseToID[chosen-internal])}, {v: val}},
2019-07-22 18:13:41 +00:00
})
}
}
// handleChanOut registers output channel for forwarding to client
func (c *wsConn) handleChanOut(ch reflect.Value, req int64) error {
c.spawnOutChanHandlerOnce.Do(func() {
go c.handleOutChans()
})
id := atomic.AddUint64(&c.chanCtr, 1)
2019-07-22 18:13:41 +00:00
2020-01-20 18:21:16 +00:00
select {
case c.registerCh <- outChanReg{
reqID: req,
chID: id,
ch: ch,
2020-01-20 18:21:16 +00:00
}:
return nil
2020-01-20 18:21:16 +00:00
case <-c.exiting:
return xerrors.New("connection closing")
}
}
// //
// Context.Done propagation //
// //
// handleCtxAsync handles context lifetimes for client
// TODO: this should be aware of events going through chanHandlers, and quit
// when the related channel is closed.
// This should also probably be a single goroutine,
// Note that not doing this should be fine for now as long as we are using
// contexts correctly (cancelling when async functions are no longer is use)
func (c *wsConn) handleCtxAsync(actx context.Context, id int64) {
<-actx.Done()
c.sendRequest(request{
Jsonrpc: "2.0",
Method: wsCancel,
Params: []param{{v: reflect.ValueOf(id)}},
})
}
// cancelCtx is a built-in rpc which handles context cancellation over rpc
func (c *wsConn) cancelCtx(req frame) {
if req.ID != nil {
log.Warnf("%s call with ID set, won't respond", wsCancel)
}
var id int64
if err := json.Unmarshal(req.Params[0].data, &id); err != nil {
log.Error("handle me:", err)
return
}
c.handlingLk.Lock()
defer c.handlingLk.Unlock()
cf, ok := c.handling[id]
if ok {
cf()
}
}
// //
// Main Handling logic //
// //
func (c *wsConn) handleChanMessage(frame frame) {
var chid uint64
if err := json.Unmarshal(frame.Params[0].data, &chid); err != nil {
log.Error("failed to unmarshal channel id in xrpc.ch.val: %s", err)
return
}
hnd, ok := c.chanHandlers[chid]
if !ok {
log.Errorf("xrpc.ch.val: handler %d not found", chid)
return
}
hnd(frame.Params[1].data, true)
}
func (c *wsConn) handleChanClose(frame frame) {
var chid uint64
if err := json.Unmarshal(frame.Params[0].data, &chid); err != nil {
log.Error("failed to unmarshal channel id in xrpc.ch.val: %s", err)
return
}
hnd, ok := c.chanHandlers[chid]
if !ok {
log.Errorf("xrpc.ch.val: handler %d not found", chid)
return
}
delete(c.chanHandlers, chid)
hnd(nil, false)
}
func (c *wsConn) handleResponse(frame frame) {
req, ok := c.inflight[*frame.ID]
if !ok {
log.Error("client got unknown ID in response")
return
}
if req.retCh != nil && frame.Result != nil {
// output is channel
var chid uint64
if err := json.Unmarshal(frame.Result, &chid); err != nil {
log.Errorf("failed to unmarshal channel id response: %s, data '%s'", err, string(frame.Result))
return
2019-07-22 18:13:41 +00:00
}
var chanCtx context.Context
chanCtx, c.chanHandlers[chid] = req.retCh()
go c.handleCtxAsync(chanCtx, *frame.ID)
}
req.ready <- clientResponse{
Jsonrpc: frame.Jsonrpc,
Result: frame.Result,
ID: *frame.ID,
Error: frame.Error,
2019-07-22 18:13:41 +00:00
}
delete(c.inflight, *frame.ID)
}
func (c *wsConn) handleCall(ctx context.Context, frame frame) {
req := request{
Jsonrpc: frame.Jsonrpc,
ID: frame.ID,
Meta: frame.Meta,
Method: frame.Method,
Params: frame.Params,
}
ctx, cancel := context.WithCancel(ctx)
nextWriter := func(cb func(io.Writer)) {
cb(ioutil.Discard)
}
done := func(keepCtx bool) {
if !keepCtx {
cancel()
}
}
if frame.ID != nil {
nextWriter = c.nextWriter
c.handlingLk.Lock()
c.handling[*frame.ID] = cancel
c.handlingLk.Unlock()
done = func(keepctx bool) {
c.handlingLk.Lock()
defer c.handlingLk.Unlock()
if !keepctx {
cancel()
delete(c.handling, *frame.ID)
}
}
}
go c.handler.handle(ctx, req, nextWriter, rpcError, done, c.handleChanOut)
}
// handleFrame handles all incoming messages (calls and responses)
func (c *wsConn) handleFrame(ctx context.Context, frame frame) {
// Get message type by method name:
// "" - response
// "xrpc.*" - builtin
// anything else - incoming remote call
switch frame.Method {
case "": // Response to our call
c.handleResponse(frame)
case wsCancel:
c.cancelCtx(frame)
case chValue:
c.handleChanMessage(frame)
case chClose:
c.handleChanClose(frame)
default: // Remote call
c.handleCall(ctx, frame)
}
}
func (c *wsConn) closeInFlight() {
for id, req := range c.inflight {
req.ready <- clientResponse{
Jsonrpc: "2.0",
ID: id,
Error: &respError{
Message: "handler: websocket connection closed",
Code: 2,
},
}
c.handlingLk.Lock()
for _, cancel := range c.handling {
cancel()
}
c.handlingLk.Unlock()
}
c.inflight = map[int64]clientRequest{}
c.handling = map[int64]context.CancelFunc{}
}
func (c *wsConn) closeChans() {
for chid := range c.chanHandlers {
hnd := c.chanHandlers[chid]
delete(c.chanHandlers, chid)
hnd(nil, false)
}
}
func (c *wsConn) handleWsConn(ctx context.Context) {
c.incoming = make(chan io.Reader)
c.inflight = map[int64]clientRequest{}
c.handling = map[int64]context.CancelFunc{}
c.chanHandlers = map[uint64]func(m []byte, ok bool){}
c.registerCh = make(chan outChanReg)
2019-10-08 08:29:37 +00:00
defer close(c.exiting)
2019-07-22 18:13:41 +00:00
// ////
// on close, make sure to return from all pending calls, and cancel context
// on all calls we handle
defer c.closeInFlight()
// wait for the first message
go c.nextMessage()
2019-07-12 17:12:38 +00:00
for {
select {
case r, ok := <-c.incoming:
2019-07-13 12:44:20 +00:00
if !ok {
if c.incomingErr != nil {
2019-10-03 18:12:30 +00:00
if !websocket.IsCloseError(c.incomingErr, websocket.CloseNormalClosure) {
2019-12-17 22:23:43 +00:00
log.Debugw("websocket error", "error", c.incomingErr)
// connection dropped unexpectedly, do our best to recover it
c.closeInFlight()
c.closeChans()
c.incoming = make(chan io.Reader) // listen again for responses
go func() {
var conn *websocket.Conn
for conn == nil {
time.Sleep(c.reconnectInterval)
var err error
if conn, err = c.connFactory(); err != nil {
log.Debugw("websocket connection retried failed", "error", err)
}
}
c.writeLk.Lock()
c.conn = conn
c.incomingErr = nil
c.writeLk.Unlock()
go c.nextMessage()
}()
continue
2019-10-03 18:12:30 +00:00
}
2019-07-13 12:44:20 +00:00
}
return // remote closed
}
2019-07-22 22:47:02 +00:00
// debug util - dump all messages to stderr
// r = io.TeeReader(r, os.Stderr)
2019-07-12 17:12:38 +00:00
var frame frame
if err := json.NewDecoder(r).Decode(&frame); err != nil {
log.Error("handle me:", err)
return
}
c.handleFrame(ctx, frame)
go c.nextMessage()
case req := <-c.requests:
c.writeLk.Lock()
2019-07-15 16:21:48 +00:00
if req.req.ID != nil {
if c.incomingErr != nil { // No conn?, immediate fail
req.ready <- clientResponse{
Jsonrpc: "2.0",
ID: *req.req.ID,
Error: &respError{
Message: "handler: websocket connection closed",
Code: 2,
},
}
c.writeLk.Unlock()
break
}
c.inflight[*req.req.ID] = req
2019-07-15 16:21:48 +00:00
}
c.writeLk.Unlock()
c.sendRequest(req.req)
case <-c.stop:
2019-10-04 16:02:25 +00:00
c.writeLk.Lock()
2019-10-03 18:12:30 +00:00
cmsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")
if err := c.conn.WriteMessage(websocket.CloseMessage, cmsg); err != nil {
log.Warn("failed to write close message: ", err)
}
if err := c.conn.Close(); err != nil {
2019-09-26 23:07:40 +00:00
log.Warnw("websocket close error", "error", err)
2019-07-13 12:44:20 +00:00
}
2019-10-04 16:02:25 +00:00
c.writeLk.Unlock()
2019-07-12 17:12:38 +00:00
return
}
}
}