Add RPC endpoint to write statediff.

This commit is contained in:
Arijit Das 2021-10-19 16:00:05 +05:30
parent c4b95cc1b7
commit 05b90107ca
3 changed files with 158 additions and 13 deletions

View File

@ -16,13 +16,18 @@
package cmd package cmd
import ( import (
"github.com/sirupsen/logrus" "fmt"
"github.com/spf13/cobra" "net/http"
"github.com/spf13/viper" "strconv"
"sync"
"time"
gethsd "github.com/ethereum/go-ethereum/statediff" gethsd "github.com/ethereum/go-ethereum/statediff"
ind "github.com/ethereum/go-ethereum/statediff/indexer" ind "github.com/ethereum/go-ethereum/statediff/indexer"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
sd "github.com/vulcanize/eth-statediff-service/pkg" sd "github.com/vulcanize/eth-statediff-service/pkg"
) )
@ -36,15 +41,20 @@ var writeCmd = &cobra.Command{
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
subCommand = cmd.CalledAs() subCommand = cmd.CalledAs()
logWithCommand = *logrus.WithField("SubCommand", subCommand) logWithCommand = *logrus.WithField("SubCommand", subCommand)
write()
addr, _ := cmd.Flags().GetString("serve")
write(addr)
}, },
} }
type blockRange [2]uint64
func init() { func init() {
rootCmd.AddCommand(writeCmd) rootCmd.AddCommand(writeCmd)
writeCmd.Flags().String("serve", ":8888", "starts a server which handles write request through endpoints")
} }
func write() { func write(addr string) {
logWithCommand.Info("Starting statediff writer") logWithCommand.Info("Starting statediff writer")
// load params // load params
@ -85,7 +95,7 @@ func write() {
} }
// Read all defined block ranges, write statediffs to database // Read all defined block ranges, write statediffs to database
var blockRanges [][2]uint64 var blockRanges []blockRange
diffParams := gethsd.Params{ // todo: configurable? diffParams := gethsd.Params{ // todo: configurable?
IntermediateStateNodes: true, IntermediateStateNodes: true,
IntermediateStorageNodes: true, IntermediateStorageNodes: true,
@ -97,13 +107,71 @@ func write() {
viper.UnmarshalKey("write.ranges", &blockRanges) viper.UnmarshalKey("write.ranges", &blockRanges)
viper.UnmarshalKey("write.params", &diffParams) viper.UnmarshalKey("write.params", &diffParams)
for _, rng := range blockRanges { blockRangesCh := make(chan blockRange, 100)
if rng[1] < rng[0] { go func() {
logWithCommand.Fatal("range ending block number needs to be greater than starting block number") for _, r := range blockRanges {
blockRangesCh <- r
} }
logrus.Infof("Writing statediffs from block %d to %d", rng[0], rng[1]) if addr == "" {
for height := rng[0]; height <= rng[1]; height++ { close(blockRangesCh)
statediffService.WriteStateDiffAt(height, diffParams) return
} }
} startServer(addr, blockRangesCh)
}()
processRanges(statediffService, diffParams, blockRangesCh)
}
func startServer(addr string, blockRangesCh chan<- blockRange) {
handler := func(w http.ResponseWriter, req *http.Request) {
start, err := strconv.Atoi(req.URL.Query().Get("start"))
if err != nil {
http.Error(w, fmt.Sprintf("failed to parse start value: %v", err), http.StatusInternalServerError)
return
}
end, err := strconv.Atoi(req.URL.Query().Get("end"))
if err != nil {
http.Error(w, fmt.Sprintf("failed to parse end value: %v", err), http.StatusInternalServerError)
return
}
select {
case blockRangesCh <- blockRange{uint64(start), uint64(end)}:
case <-time.After(time.Millisecond * 200):
http.Error(w, "server is busy", http.StatusInternalServerError)
return
}
fmt.Fprintf(w, "added block range to the queue\n")
}
http.HandleFunc("/writeDiff", handler)
logrus.Fatal(http.ListenAndServe(addr, nil))
}
type diffService interface {
WriteStateDiffAt(blockNumber uint64, params gethsd.Params) error
}
func processRanges(sds diffService, param gethsd.Params, blockRangesCh chan blockRange) {
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
for rng := range blockRangesCh {
if rng[1] < rng[0] {
logWithCommand.Fatal("range ending block number needs to be greater than starting block number")
}
logrus.Infof("Writing statediffs from block %d to %d", rng[0], rng[1])
for height := rng[0]; height <= rng[1]; height++ {
err := sds.WriteStateDiffAt(height, param)
if err != nil {
logrus.Errorf("failed to write state diff for range: %v %v", rng, err)
}
}
}
}()
wg.Wait()
} }

76
cmd/write_test.go Normal file
View File

@ -0,0 +1,76 @@
package cmd
import (
"fmt"
"net/http"
"testing"
"time"
gethsd "github.com/ethereum/go-ethereum/statediff"
"github.com/stretchr/testify/require"
)
type mockService struct {
reqCount int
}
func (ms *mockService) WriteStateDiffAt(_ uint64, _ gethsd.Params) error {
ms.reqCount++
return nil
}
func TestProcessRanges(t *testing.T) {
blockRangesCh := make(chan blockRange)
srv := new(mockService)
go func() {
blockRangesCh <- blockRange{uint64(1), uint64(5)}
blockRangesCh <- blockRange{uint64(8), uint64(10)}
blockRangesCh <- blockRange{uint64(6), uint64(7)}
blockRangesCh <- blockRange{uint64(50), uint64(100)}
blockRangesCh <- blockRange{uint64(5), uint64(8)}
close(blockRangesCh)
}()
processRanges(srv, gethsd.Params{}, blockRangesCh)
require.Equal(t, 65, srv.reqCount)
}
func TestHttpEndpoint(t *testing.T) {
addr := ":11111"
queueSize := 5
blockRangesCh := make(chan blockRange, queueSize)
srv := new(mockService)
go startServer(addr, blockRangesCh)
go func() {
br := []blockRange{
{uint64(1), uint64(5)},
{uint64(8), uint64(10)},
{uint64(6), uint64(7)},
{uint64(50), uint64(100)},
{uint64(5), uint64(8)},
// Below request should fail since server has queue size of 5
{uint64(52), uint64(328)},
{uint64(35), uint64(428)},
{uint64(45), uint64(844)},
}
for idx, r := range br {
res, err := http.Get(fmt.Sprintf("http://localhost:11111/writeDiff?start=%d&end=%d", r[0], r[1]))
require.NoError(t, err)
require.NotNil(t, res)
if idx < queueSize {
require.Equal(t, res.StatusCode, 200)
} else {
require.Equal(t, res.StatusCode, 500)
}
require.NoError(t, res.Body.Close())
}
processRanges(srv, gethsd.Params{}, blockRangesCh)
}()
time.Sleep(1 * time.Second)
require.Equal(t, 65, srv.reqCount)
}

1
go.mod
View File

@ -7,6 +7,7 @@ require (
github.com/sirupsen/logrus v1.7.0 github.com/sirupsen/logrus v1.7.0
github.com/spf13/cobra v1.1.1 github.com/spf13/cobra v1.1.1
github.com/spf13/viper v1.7.1 github.com/spf13/viper v1.7.1
github.com/stretchr/testify v1.7.0
github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191 github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191
) )