cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

fixes after tests

Browse Source
This commit is contained in:
Ian Norden
2020-04-20 11:16:08 -05:00
parent eceaa0aecb
commit 43c254b5f6
16 changed files with 214 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/super_node/eth/cleaner.go
+3
View File
@@ -50,6 +50,9 @@ func (c *Cleaner) ResetValidation(rngs [][2]uint64) error {
SET times_validated = 0
WHERE block_number BETWEEN $1 AND $2`
if _, err := tx.Exec(pgStr, rng[0], rng[1]); err != nil {
if err := tx.Rollback(); err != nil {
logrus.Error(err)
}
return err
}
}
pkg/super_node/eth/payload_fetcher.go
+4 -4
View File
@@ -21,15 +21,15 @@ import (
"fmt"
"time"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff"
"github.com/vulcanize/vulcanizedb/pkg/eth/client"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
// BatchClient is an interface to a batch-fetching geth rpc client; created to allow mock insertion
type BatchClient interface {
BatchCallContext(ctx context.Context, batch []client.BatchElem) error
BatchCallContext(ctx context.Context, batch []rpc.BatchElem) error
}
// PayloadFetcher satisfies the PayloadFetcher interface for ethereum
@@ -53,9 +53,9 @@ func NewPayloadFetcher(bc BatchClient, timeout time.Duration) *PayloadFetcher {
// FetchAt fetches the statediff payloads at the given block heights
// Calls StateDiffAt(ctx context.Context, blockNumber uint64) (*Payload, error)
func (fetcher *PayloadFetcher) FetchAt(blockHeights []uint64) ([]shared.RawChainData, error) {
batch := make([]client.BatchElem, 0)
batch := make([]rpc.BatchElem, 0)
for _, height := range blockHeights {
batch = append(batch, client.BatchElem{
batch = append(batch, rpc.BatchElem{
Method: method,
Args: []interface{}{height},
Result: new(statediff.Payload),
pkg/super_node/eth/retriever.go
+3 -16
View File
@@ -20,6 +20,8 @@ import (
"fmt"
"math/big"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/jmoiron/sqlx"
@@ -479,22 +481,7 @@ func (ecr *CIDRetriever) RetrieveGapsInData(validationLevel int) ([]shared.Gap,
if len(heights) == 0 {
return emptyGaps, nil
}
validationGaps := make([]shared.Gap, 0)
start := heights[0]
lastHeight := start
for _, height := range heights[1:] {
if height == lastHeight+1 {
lastHeight = height
continue
}
validationGaps = append(validationGaps, shared.Gap{
Start: start,
Stop: lastHeight,
})
start = height
lastHeight = start
}
return append(emptyGaps, validationGaps...), nil
return append(emptyGaps, utils.MissingHeightsToGaps(heights)...), nil
}
// RetrieveBlockByHash returns all of the CIDs needed to compose an entire block, for a given block hash
pkg/super_node/eth/retriever_test.go
+76 -3
View File
@@ -544,8 +544,16 @@ var _ = Describe("Retriever", func() {
payload4.HeaderCID.BlockNumber = "101"
payload5 := payload4
payload5.HeaderCID.BlockNumber = "102"
payload6 := payload5
payload6.HeaderCID.BlockNumber = "1000"
payload6 := payload4
payload6.HeaderCID.BlockNumber = "103"
payload7 := payload4
payload7.HeaderCID.BlockNumber = "104"
payload8 := payload4
payload8.HeaderCID.BlockNumber = "105"
payload9 := payload4
payload9.HeaderCID.BlockNumber = "106"
payload10 := payload5
payload10.HeaderCID.BlockNumber = "1000"
err := repo.Index(&payload1)
Expect(err).ToNot(HaveOccurred())
err = repo.Index(&payload2)
@@ -558,11 +566,76 @@ var _ = Describe("Retriever", func() {
Expect(err).ToNot(HaveOccurred())
err = repo.Index(&payload6)
Expect(err).ToNot(HaveOccurred())
err = repo.Index(&payload7)
Expect(err).ToNot(HaveOccurred())
err = repo.Index(&payload8)
Expect(err).ToNot(HaveOccurred())
err = repo.Index(&payload9)
Expect(err).ToNot(HaveOccurred())
err = repo.Index(&payload10)
Expect(err).ToNot(HaveOccurred())
gaps, err := retriever.RetrieveGapsInData(1)
Expect(err).ToNot(HaveOccurred())
Expect(len(gaps)).To(Equal(3))
Expect(shared.ListContainsGap(gaps, shared.Gap{Start: 6, Stop: 99})).To(BeTrue())
Expect(shared.ListContainsGap(gaps, shared.Gap{Start: 103, Stop: 999})).To(BeTrue())
Expect(shared.ListContainsGap(gaps, shared.Gap{Start: 107, Stop: 999})).To(BeTrue())
Expect(shared.ListContainsGap(gaps, shared.Gap{Start: 1001, Stop: 1010100})).To(BeTrue())
})
It("Finds validation level gaps", func() {
payload1 := *mocks.MockCIDPayload
payload1.HeaderCID.BlockNumber = "1010101"
payload2 := payload1
payload2.HeaderCID.BlockNumber = "5"
payload3 := payload2
payload3.HeaderCID.BlockNumber = "100"
payload4 := payload3
payload4.HeaderCID.BlockNumber = "101"
payload5 := payload4
payload5.HeaderCID.BlockNumber = "102"
payload6 := payload4
payload6.HeaderCID.BlockNumber = "103"
payload7 := payload4
payload7.HeaderCID.BlockNumber = "104"
payload8 := payload4
payload8.HeaderCID.BlockNumber = "105"
payload9 := payload4
payload9.HeaderCID.BlockNumber = "106"
payload10 := payload5
payload10.HeaderCID.BlockNumber = "1000"
err := repo.Index(&payload1)
Expect(err).ToNot(HaveOccurred())
err = repo.Index(&payload2)
Expect(err).ToNot(HaveOccurred())
err = repo.Index(&payload3)
Expect(err).ToNot(HaveOccurred())
err = repo.Index(&payload4)
Expect(err).ToNot(HaveOccurred())
err = repo.Index(&payload5)
Expect(err).ToNot(HaveOccurred())
err = repo.Index(&payload6)
Expect(err).ToNot(HaveOccurred())
err = repo.Index(&payload7)
Expect(err).ToNot(HaveOccurred())
err = repo.Index(&payload8)
Expect(err).ToNot(HaveOccurred())
err = repo.Index(&payload9)
Expect(err).ToNot(HaveOccurred())
err = repo.Index(&payload10)
Expect(err).ToNot(HaveOccurred())
cleaner := eth.NewCleaner(db)
err = cleaner.ResetValidation([][2]uint64{{101, 102}, {104, 104}})
Expect(err).ToNot(HaveOccurred())
gaps, err := retriever.RetrieveGapsInData(1)
Expect(err).ToNot(HaveOccurred())
Expect(len(gaps)).To(Equal(5))
Expect(shared.ListContainsGap(gaps, shared.Gap{Start: 6, Stop: 99})).To(BeTrue())
Expect(shared.ListContainsGap(gaps, shared.Gap{Start: 101, Stop: 102})).To(BeTrue())
Expect(shared.ListContainsGap(gaps, shared.Gap{Start: 104, Stop: 104})).To(BeTrue())
Expect(shared.ListContainsGap(gaps, shared.Gap{Start: 107, Stop: 999})).To(BeTrue())
Expect(shared.ListContainsGap(gaps, shared.Gap{Start: 1001, Stop: 1010100})).To(BeTrue())
})
})
pkg/super_node/eth/streamer.go
+11 -4
View File
@@ -17,10 +17,12 @@
package eth
import (
"context"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/eth/core"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
@@ -28,13 +30,18 @@ const (
PayloadChanBufferSize = 20000 // the max eth sub buffer size
)
// StreamClient is an interface for subscribing and streaming from geth
type StreamClient interface {
Subscribe(ctx context.Context, namespace string, payloadChan interface{}, args ...interface{}) (*rpc.ClientSubscription, error)
}
// PayloadStreamer satisfies the PayloadStreamer interface for ethereum
type PayloadStreamer struct {
Client core.RPCClient
Client StreamClient
}
// NewPayloadStreamer creates a pointer to a new PayloadStreamer which satisfies the PayloadStreamer interface for ethereum
func NewPayloadStreamer(client core.RPCClient) *PayloadStreamer {
func NewPayloadStreamer(client StreamClient) *PayloadStreamer {
return &PayloadStreamer{
Client: client,
}
@@ -53,5 +60,5 @@ func (ps *PayloadStreamer) Stream(payloadChan chan shared.RawChainData) (shared.
}
}
}()
return ps.Client.Subscribe("statediff", stateDiffChan, "stream")
return ps.Client.Subscribe(context.Background(), "statediff", stateDiffChan, "stream")
}
pkg/super_node/eth/streamer_test.go
+2 -2
View File
@@ -18,14 +18,14 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
var _ = Describe("StateDiff Streamer", func() {
It("subscribes to the geth statediff service", func() {
client := &fakes.MockRPCClient{}
client := &mocks.StreamClient{}
streamer := eth.NewPayloadStreamer(client)
payloadChan := make(chan shared.RawChainData)
_, err := streamer.Stream(payloadChan)