918a30dcc9
* Statediff Geth Handle conflicts (#244) * Handle conflicts * Update go mod file versions * Make lint changes Disassociate block number from the indexer object Update ipld-eth-db ref Refactor builder code to make it reusable Use prefix comparison for account selective statediffing Update builder unit tests Add mode to write to CSV files in statediff file writer (#249) * Change file writing mode to csv files * Implement writer interface for file indexer * Implement option for csv or sql in file mode * Close files in CSV writer * Add tests for CSV file mode * Implement CSV file for watched addresses * Separate test configs for CSV and SQL * Refactor common code for file indexer tests Update indexer to include block hash in receipts and logs (#256) * Update indexer to include block hash in receipts and logs * Upgrade ipld-eth-db image in docker-compose to run tests Use watched addresses from direct indexing params by default while serving statediff APIs (#262) * Use watched addresses from direct indexing params in statediff APIs by default * Avoid using indexer object when direct indexing is off * Add nil check before accessing watched addresses from direct indexing params Rebase missed these changes needed at 1.10.20 Flags cleanup for CLI changes and linter complaints Linter appeasements to achieve perfection enforce go 1.18 for check (#267) * enforce go 1.18 for check * tests on 1.18 as well * adding db yml for possible change in docker-compose behavior in yml parsing Add indexer tests for handling non canonical blocks (#254) * Add indexer tests for header and transactions in a non canonical block * Add indexer tests for receipts in a non-canonical block and refactor * Add indexer tests for logs in a non-canonical block * Add indexer tests for state and storage nodes in a non-canonical block * Add indexer tests for non-canonical block at another height * Avoid passing address of a pointer * Update refs in GitHub workflow * Add genesis file path to stack-orchestrator config in GitHub workflow * Add descriptive comments fix non-deterministic ordering in unit tests Refactor indexer tests to avoid duplicate code (#270) * Refactor indexer tests to avoid duplicate code * Refactor file mode indexer tests * Fix expected db stats for sqlx after tx closure * Refactor indexer tests for legacy block * Refactor mainnet indexer tests * Refactor tests for watched addressess methods * Fix query in legacy indexer test rebase and resolve onto 1.10.23... still error out of index related to GetLeafKeys changed trie.Commit behavior was subtle about not not flushing to disk without an Update * no merge nodeset throws nil * linter appeasement Co-authored-by: Abdul Rabbani <abdulrabbani00@gmail.com>
300 lines
8.8 KiB
Go
300 lines
8.8 KiB
Go
// Copyright 2015 The go-ethereum Authors
|
|
// This file is part of the go-ethereum library.
|
|
//
|
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Lesser General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Lesser General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Lesser General Public License
|
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
package rpc
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"mime"
|
|
"net/http"
|
|
"net/url"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
maxRequestContentLength = 1024 * 1024 * 12
|
|
contentType = "application/json"
|
|
)
|
|
|
|
// https://www.jsonrpc.org/historical/json-rpc-over-http.html#id13
|
|
var acceptedContentTypes = []string{contentType, "application/json-rpc", "application/jsonrequest"}
|
|
|
|
type httpConn struct {
|
|
client *http.Client
|
|
url string
|
|
closeOnce sync.Once
|
|
closeCh chan interface{}
|
|
mu sync.Mutex // protects headers
|
|
headers http.Header
|
|
}
|
|
|
|
// httpConn implements ServerCodec, but it is treated specially by Client
|
|
// and some methods don't work. The panic() stubs here exist to ensure
|
|
// this special treatment is correct.
|
|
|
|
func (hc *httpConn) writeJSON(context.Context, interface{}) error {
|
|
panic("writeJSON called on httpConn")
|
|
}
|
|
|
|
func (hc *httpConn) peerInfo() PeerInfo {
|
|
panic("peerInfo called on httpConn")
|
|
}
|
|
|
|
func (hc *httpConn) remoteAddr() string {
|
|
return hc.url
|
|
}
|
|
|
|
func (hc *httpConn) readBatch() ([]*jsonrpcMessage, bool, error) {
|
|
<-hc.closeCh
|
|
return nil, false, io.EOF
|
|
}
|
|
|
|
func (hc *httpConn) close() {
|
|
hc.closeOnce.Do(func() { close(hc.closeCh) })
|
|
}
|
|
|
|
func (hc *httpConn) closed() <-chan interface{} {
|
|
return hc.closeCh
|
|
}
|
|
|
|
// HTTPTimeouts represents the configuration params for the HTTP RPC server.
|
|
type HTTPTimeouts struct {
|
|
// ReadTimeout is the maximum duration for reading the entire
|
|
// request, including the body.
|
|
//
|
|
// Because ReadTimeout does not let Handlers make per-request
|
|
// decisions on each request body's acceptable deadline or
|
|
// upload rate, most users will prefer to use
|
|
// ReadHeaderTimeout. It is valid to use them both.
|
|
ReadTimeout time.Duration
|
|
|
|
// ReadHeaderTimeout is the amount of time allowed to read
|
|
// request headers. The connection's read deadline is reset
|
|
// after reading the headers and the Handler can decide what
|
|
// is considered too slow for the body. If ReadHeaderTimeout
|
|
// is zero, the value of ReadTimeout is used. If both are
|
|
// zero, there is no timeout.
|
|
ReadHeaderTimeout time.Duration
|
|
|
|
// WriteTimeout is the maximum duration before timing out
|
|
// writes of the response. It is reset whenever a new
|
|
// request's header is read. Like ReadTimeout, it does not
|
|
// let Handlers make decisions on a per-request basis.
|
|
WriteTimeout time.Duration
|
|
|
|
// IdleTimeout is the maximum amount of time to wait for the
|
|
// next request when keep-alives are enabled. If IdleTimeout
|
|
// is zero, the value of ReadTimeout is used. If both are
|
|
// zero, ReadHeaderTimeout is used.
|
|
IdleTimeout time.Duration
|
|
}
|
|
|
|
// DefaultHTTPTimeouts represents the default timeout values used if further
|
|
// configuration is not provided.
|
|
var DefaultHTTPTimeouts = HTTPTimeouts{
|
|
ReadTimeout: 30 * time.Second,
|
|
ReadHeaderTimeout: 30 * time.Second,
|
|
WriteTimeout: 30 * time.Second,
|
|
IdleTimeout: 120 * time.Second,
|
|
}
|
|
|
|
// DialHTTPWithClient creates a new RPC client that connects to an RPC server over HTTP
|
|
// using the provided HTTP Client.
|
|
func DialHTTPWithClient(endpoint string, client *http.Client) (*Client, error) {
|
|
// Sanity check URL so we don't end up with a client that will fail every request.
|
|
_, err := url.Parse(endpoint)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
initctx := context.Background()
|
|
headers := make(http.Header, 2)
|
|
headers.Set("accept", contentType)
|
|
headers.Set("content-type", contentType)
|
|
return newClient(initctx, func(context.Context) (ServerCodec, error) {
|
|
hc := &httpConn{
|
|
client: client,
|
|
headers: headers,
|
|
url: endpoint,
|
|
closeCh: make(chan interface{}),
|
|
}
|
|
return hc, nil
|
|
})
|
|
}
|
|
|
|
// DialHTTP creates a new RPC client that connects to an RPC server over HTTP.
|
|
func DialHTTP(endpoint string) (*Client, error) {
|
|
return DialHTTPWithClient(endpoint, new(http.Client))
|
|
}
|
|
|
|
func (c *Client) sendHTTP(ctx context.Context, op *requestOp, msg interface{}) error {
|
|
hc := c.writeConn.(*httpConn)
|
|
respBody, err := hc.doRequest(ctx, msg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer respBody.Close()
|
|
|
|
var respmsg jsonrpcMessage
|
|
if err := json.NewDecoder(respBody).Decode(&respmsg); err != nil {
|
|
return err
|
|
}
|
|
op.resp <- &respmsg
|
|
return nil
|
|
}
|
|
|
|
func (c *Client) sendBatchHTTP(ctx context.Context, op *requestOp, msgs []*jsonrpcMessage) error {
|
|
hc := c.writeConn.(*httpConn)
|
|
respBody, err := hc.doRequest(ctx, msgs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer respBody.Close()
|
|
var respmsgs []jsonrpcMessage
|
|
if err := json.NewDecoder(respBody).Decode(&respmsgs); err != nil {
|
|
return err
|
|
}
|
|
for i := 0; i < len(respmsgs); i++ {
|
|
op.resp <- &respmsgs[i]
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (hc *httpConn) doRequest(ctx context.Context, msg interface{}) (io.ReadCloser, error) {
|
|
body, err := json.Marshal(msg)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, "POST", hc.url, io.NopCloser(bytes.NewReader(body)))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.ContentLength = int64(len(body))
|
|
req.GetBody = func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(body)), nil }
|
|
|
|
// set headers
|
|
hc.mu.Lock()
|
|
req.Header = hc.headers.Clone()
|
|
hc.mu.Unlock()
|
|
|
|
// do request
|
|
resp, err := hc.client.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
var buf bytes.Buffer
|
|
var body []byte
|
|
if _, err := buf.ReadFrom(resp.Body); err == nil {
|
|
body = buf.Bytes()
|
|
}
|
|
|
|
return nil, HTTPError{
|
|
Status: resp.Status,
|
|
StatusCode: resp.StatusCode,
|
|
Body: body,
|
|
}
|
|
}
|
|
return resp.Body, nil
|
|
}
|
|
|
|
// httpServerConn turns a HTTP connection into a Conn.
|
|
type httpServerConn struct {
|
|
io.Reader
|
|
io.Writer
|
|
r *http.Request
|
|
}
|
|
|
|
func newHTTPServerConn(r *http.Request, w http.ResponseWriter) ServerCodec {
|
|
body := io.LimitReader(r.Body, maxRequestContentLength)
|
|
conn := &httpServerConn{Reader: body, Writer: w, r: r}
|
|
return NewCodec(conn)
|
|
}
|
|
|
|
// Close does nothing and always returns nil.
|
|
func (t *httpServerConn) Close() error { return nil }
|
|
|
|
// RemoteAddr returns the peer address of the underlying connection.
|
|
func (t *httpServerConn) RemoteAddr() string {
|
|
return t.r.RemoteAddr
|
|
}
|
|
|
|
// SetWriteDeadline does nothing and always returns nil.
|
|
func (t *httpServerConn) SetWriteDeadline(time.Time) error { return nil }
|
|
|
|
// ServeHTTP serves JSON-RPC requests over HTTP.
|
|
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
// Permit dumb empty requests for remote health-checks (AWS)
|
|
if r.Method == http.MethodGet && r.ContentLength == 0 && r.URL.RawQuery == "" {
|
|
w.WriteHeader(http.StatusOK)
|
|
return
|
|
}
|
|
if code, err := validateRequest(r); err != nil {
|
|
http.Error(w, err.Error(), code)
|
|
return
|
|
}
|
|
|
|
// Create request-scoped context.
|
|
connInfo := PeerInfo{Transport: "http", RemoteAddr: r.RemoteAddr}
|
|
connInfo.HTTP.Version = r.Proto
|
|
connInfo.HTTP.Host = r.Host
|
|
connInfo.HTTP.Origin = r.Header.Get("Origin")
|
|
connInfo.HTTP.UserAgent = r.Header.Get("User-Agent")
|
|
ctx := r.Context()
|
|
ctx = context.WithValue(ctx, peerInfoContextKey{}, connInfo)
|
|
|
|
// All checks passed, create a codec that reads directly from the request body
|
|
// until EOF, writes the response to w, and orders the server to process a
|
|
// single request.
|
|
w.Header().Set("content-type", contentType)
|
|
codec := newHTTPServerConn(r, w)
|
|
defer codec.close()
|
|
s.serveSingleRequest(ctx, codec)
|
|
}
|
|
|
|
// validateRequest returns a non-zero response code and error message if the
|
|
// request is invalid.
|
|
func validateRequest(r *http.Request) (int, error) {
|
|
if r.Method == http.MethodPut || r.Method == http.MethodDelete {
|
|
return http.StatusMethodNotAllowed, errors.New("method not allowed")
|
|
}
|
|
if r.ContentLength > maxRequestContentLength {
|
|
err := fmt.Errorf("content length too large (%d>%d)", r.ContentLength, maxRequestContentLength)
|
|
return http.StatusRequestEntityTooLarge, err
|
|
}
|
|
// Allow OPTIONS (regardless of content-type)
|
|
if r.Method == http.MethodOptions {
|
|
return 0, nil
|
|
}
|
|
// Check content-type
|
|
if mt, _, err := mime.ParseMediaType(r.Header.Get("content-type")); err == nil {
|
|
for _, accepted := range acceptedContentTypes {
|
|
if accepted == mt {
|
|
return 0, nil
|
|
}
|
|
}
|
|
}
|
|
// Invalid content-type
|
|
err := fmt.Errorf("invalid content type, only %s is supported", contentType)
|
|
return http.StatusUnsupportedMediaType, err
|
|
}
|