deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

Extract storage diff fetching behind an interface

Browse Source 
- Replaces directly reading from a CSV
- Simplifies testing
- Should hopefully make it easier to plug in other sources for storage
  diffs (e.g. differently formatted CSVs, JSON RPC, etc)
This commit is contained in:
Rob Mulholand
2019-05-01 12:30:36 -05:00
parent c93e6adb27
commit 2d684c5aec
15 changed files with 279 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/shared/watcher/contract_watcher.go
+6 -6
View File
@@ -54,10 +54,10 @@ func (watcher *ContractWatcher) AddTransformers(inits interface{}) error {
watcher.Transformers = append(watcher.Transformers, t)
}
for _, transformer := range watcher.Transformers {
err := transformer.Init()
for _, contractTransformer := range watcher.Transformers {
err := contractTransformer.Init()
if err != nil {
log.Print("Unable to initialize transformer:", transformer.GetConfig().Name, err)
log.Print("Unable to initialize transformer:", contractTransformer.GetConfig().Name, err)
return err
}
}
@@ -65,10 +65,10 @@ func (watcher *ContractWatcher) AddTransformers(inits interface{}) error {
}
func (watcher *ContractWatcher) Execute() error {
for _, transformer := range watcher.Transformers {
err := transformer.Execute()
for _, contractTransformer := range watcher.Transformers {
err := contractTransformer.Execute()
if err != nil {
log.Error("Unable to execute transformer:", transformer.GetConfig().Name, err)
log.Error("Unable to execute transformer:", contractTransformer.GetConfig().Name, err)
return err
}
}
libraries/shared/watcher/event_watcher.go
+2 -2
View File
@@ -37,7 +37,7 @@ type EventWatcher struct {
Transformers []transformer.EventTransformer
BlockChain core.BlockChain
DB *postgres.DB
Fetcher fetcher.LogFetcher
Fetcher fetcher.ILogFetcher
Chunker chunker.Chunker
Addresses []common.Address
Topics []common.Hash
@@ -47,7 +47,7 @@ type EventWatcher struct {
func NewEventWatcher(db *postgres.DB, bc core.BlockChain) EventWatcher {
logChunker := chunker.NewLogChunker()
logFetcher := fetcher.NewFetcher(bc)
logFetcher := fetcher.NewLogFetcher(bc)
transactionSyncer := transactions.NewTransactionsSyncer(db, bc)
return EventWatcher{
BlockChain: bc,
libraries/shared/watcher/storage_watcher.go
+36 -35
View File
@@ -18,33 +18,32 @@ package watcher
import (
"reflect"
"strings"
"github.com/ethereum/go-ethereum/common"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/fs"
)
type StorageWatcher struct {
db *postgres.DB
tailer fs.Tailer
Queue storage.IStorageQueue
Transformers map[common.Address]transformer.StorageTransformer
db *postgres.DB
StorageFetcher fetcher.IStorageFetcher
Queue storage.IStorageQueue
Transformers map[common.Address]transformer.StorageTransformer
}
func NewStorageWatcher(tailer fs.Tailer, db *postgres.DB) StorageWatcher {
func NewStorageWatcher(fetcher fetcher.IStorageFetcher, db *postgres.DB) StorageWatcher {
transformers := make(map[common.Address]transformer.StorageTransformer)
queue := storage.NewStorageQueue(db)
return StorageWatcher{
db: db,
tailer: tailer,
Queue: queue,
Transformers: transformers,
db: db,
StorageFetcher: fetcher,
Queue: queue,
Transformers: transformers,
}
}
@@ -56,34 +55,36 @@ func (watcher StorageWatcher) AddTransformers(initializers []transformer.Storage
}
func (watcher StorageWatcher) Execute() error {
t, tailErr := watcher.tailer.Tail()
if tailErr != nil {
return tailErr
rows := make(chan utils.StorageDiffRow)
errs := make(chan error)
go watcher.StorageFetcher.FetchStorageDiffs(rows, errs)
for {
select {
case row := <-rows:
watcher.processRow(row)
case err := <-errs:
return err
}
}
for line := range t.Lines {
row, parseErr := utils.FromStrings(strings.Split(line.Text, ","))
if parseErr != nil {
return parseErr
}
storageTransformer, ok := watcher.Transformers[row.Contract]
if !ok {
logrus.Warn(utils.ErrContractNotFound{Contract: row.Contract.Hex()}.Error())
continue
}
executeErr := storageTransformer.Execute(row)
if executeErr != nil {
if isKeyNotFound(executeErr) {
queueErr := watcher.Queue.Add(row)
if queueErr != nil {
logrus.Warn(queueErr.Error())
}
} else {
logrus.Warn(executeErr.Error())
}
func (watcher StorageWatcher) processRow(row utils.StorageDiffRow) {
storageTransformer, ok := watcher.Transformers[row.Contract]
if !ok {
// ignore rows from unwatched contracts
return
}
executeErr := storageTransformer.Execute(row)
if executeErr != nil {
if isKeyNotFound(executeErr) {
queueErr := watcher.Queue.Add(row)
if queueErr != nil {
logrus.Warn(queueErr.Error())
}
continue
} else {
logrus.Warn(executeErr.Error())
}
}
return nil
}
func isKeyNotFound(executeErr error) bool {
libraries/shared/watcher/storage_watcher_test.go
+67 -156
View File
@@ -17,15 +17,10 @@
package watcher_test
import (
"errors"
"fmt"
"io/ioutil"
"os"
"strings"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/hpcloud/tail"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
@@ -34,7 +29,6 @@ import (
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/test_config"
)
@@ -43,170 +37,87 @@ var _ = Describe("Storage Watcher", func() {
It("adds transformers", func() {
fakeAddress := common.HexToAddress("0x12345")
fakeTransformer := &mocks.MockStorageTransformer{Address: fakeAddress}
w := watcher.NewStorageWatcher(&fakes.MockTailer{}, test_config.NewTestDB(core.Node{}))
w := watcher.NewStorageWatcher(mocks.NewMockStorageFetcher(), test_config.NewTestDB(test_config.NewTestNode()))
w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
Expect(w.Transformers[fakeAddress]).To(Equal(fakeTransformer))
})
It("reads the tail of the storage diffs file", func() {
mockTailer := fakes.NewMockTailer()
w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
Describe("executing watcher", func() {
var (
mockFetcher *mocks.MockStorageFetcher
mockQueue *mocks.MockStorageQueue
mockTransformer *mocks.MockStorageTransformer
row utils.StorageDiffRow
storageWatcher watcher.StorageWatcher
)
assert(func(err error) {
Expect(err).To(BeNil())
Expect(mockTailer.TailCalled).To(BeTrue())
}, w, mockTailer, []*tail.Line{})
})
BeforeEach(func() {
address := common.HexToAddress("0x0123456789abcdef")
mockFetcher = mocks.NewMockStorageFetcher()
mockQueue = &mocks.MockStorageQueue{}
mockTransformer = &mocks.MockStorageTransformer{Address: address}
row = utils.StorageDiffRow{
Contract: address,
BlockHash: common.HexToHash("0xfedcba9876543210"),
BlockHeight: 0,
StorageKey: common.HexToHash("0xabcdef1234567890"),
StorageValue: common.HexToHash("0x9876543210abcdef"),
}
mockFetcher.RowsToReturn = []utils.StorageDiffRow{row}
storageWatcher = watcher.NewStorageWatcher(mockFetcher, test_config.NewTestDB(test_config.NewTestNode()))
storageWatcher.Queue = mockQueue
storageWatcher.AddTransformers([]transformer.StorageTransformerInitializer{mockTransformer.FakeTransformerInitializer})
})
It("returns error if row parsing fails", func() {
mockTailer := fakes.NewMockTailer()
w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
line := &tail.Line{Text: "oops"}
It("executes transformer for recognized storage row", func() {
err := storageWatcher.Execute()
assert(func(err error) {
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(utils.ErrRowMalformed{Length: 1}))
}, w, mockTailer, []*tail.Line{line})
})
Expect(err).NotTo(HaveOccurred())
Expect(mockTransformer.PassedRow).To(Equal(row))
})
It("logs error if no transformer can parse storage row", func() {
mockTailer := fakes.NewMockTailer()
address := common.HexToAddress("0x12345")
line := getFakeLine(address.Bytes())
w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
tempFile, err := ioutil.TempFile("", "log")
defer os.Remove(tempFile.Name())
Expect(err).NotTo(HaveOccurred())
logrus.SetOutput(tempFile)
It("queues row for later processing if row's key not recognized", func() {
mockTransformer.ExecuteErr = utils.ErrStorageKeyNotFound{}
err := storageWatcher.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(mockQueue.AddCalled).To(BeTrue())
Expect(mockQueue.PassedRow).To(Equal(row))
})
It("logs error if queueing row fails", func() {
mockTransformer.ExecuteErr = utils.ErrStorageKeyNotFound{}
mockQueue.AddError = fakes.FakeError
tempFile, fileErr := ioutil.TempFile("", "log")
Expect(fileErr).NotTo(HaveOccurred())
defer os.Remove(tempFile.Name())
logrus.SetOutput(tempFile)
err := storageWatcher.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(mockQueue.AddCalled).To(BeTrue())
logContent, readErr := ioutil.ReadFile(tempFile.Name())
Expect(readErr).NotTo(HaveOccurred())
Expect(string(logContent)).To(ContainSubstring(fakes.FakeError.Error()))
})
It("logs error if transformer execution fails for reason other than key not found", func() {
mockTransformer.ExecuteErr = fakes.FakeError
tempFile, fileErr := ioutil.TempFile("", "log")
Expect(fileErr).NotTo(HaveOccurred())
defer os.Remove(tempFile.Name())
logrus.SetOutput(tempFile)
err := storageWatcher.Execute()
assert(func(err error) {
Expect(err).NotTo(HaveOccurred())
logContent, readErr := ioutil.ReadFile(tempFile.Name())
Expect(readErr).NotTo(HaveOccurred())
Expect(string(logContent)).To(ContainSubstring(utils.ErrContractNotFound{Contract: address.Hex()}.Error()))
}, w, mockTailer, []*tail.Line{line})
})
It("executes transformer with storage row", func() {
address := []byte{1, 2, 3}
line := getFakeLine(address)
mockTailer := fakes.NewMockTailer()
w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address)}
w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
assert(func(err error) {
Expect(err).To(BeNil())
expectedRow, err := utils.FromStrings(strings.Split(line.Text, ","))
Expect(err).NotTo(HaveOccurred())
Expect(fakeTransformer.PassedRow).To(Equal(expectedRow))
}, w, mockTailer, []*tail.Line{line})
})
Describe("when executing transformer fails", func() {
It("queues row when error is storage key not found", func() {
address := []byte{1, 2, 3}
line := getFakeLine(address)
mockTailer := fakes.NewMockTailer()
w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
mockQueue := &mocks.MockStorageQueue{}
w.Queue = mockQueue
keyNotFoundError := utils.ErrStorageKeyNotFound{Key: "unknown_storage_key"}
fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address), ExecuteErr: keyNotFoundError}
w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
assert(func(err error) {
Expect(err).NotTo(HaveOccurred())
Expect(mockQueue.AddCalled).To(BeTrue())
}, w, mockTailer, []*tail.Line{line})
})
It("logs error if queuing row fails", func() {
address := []byte{1, 2, 3}
line := getFakeLine(address)
mockTailer := fakes.NewMockTailer()
w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
mockQueue := &mocks.MockStorageQueue{}
mockQueue.AddError = fakes.FakeError
w.Queue = mockQueue
keyNotFoundError := utils.ErrStorageKeyNotFound{Key: "unknown_storage_key"}
fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address), ExecuteErr: keyNotFoundError}
w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
tempFile, err := ioutil.TempFile("", "log")
defer os.Remove(tempFile.Name())
Expect(err).NotTo(HaveOccurred())
logrus.SetOutput(tempFile)
assert(func(err error) {
Expect(err).NotTo(HaveOccurred())
Expect(mockQueue.AddCalled).To(BeTrue())
logContent, readErr := ioutil.ReadFile(tempFile.Name())
Expect(readErr).NotTo(HaveOccurred())
Expect(string(logContent)).To(ContainSubstring(fakes.FakeError.Error()))
}, w, mockTailer, []*tail.Line{line})
})
It("logs any other error", func() {
address := []byte{1, 2, 3}
line := getFakeLine(address)
mockTailer := fakes.NewMockTailer()
w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
executionError := errors.New("storage watcher failed attempting to execute transformer")
fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address), ExecuteErr: executionError}
w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
tempFile, err := ioutil.TempFile("", "log")
defer os.Remove(tempFile.Name())
Expect(err).NotTo(HaveOccurred())
logrus.SetOutput(tempFile)
assert(func(err error) {
Expect(err).NotTo(HaveOccurred())
logContent, readErr := ioutil.ReadFile(tempFile.Name())
Expect(readErr).NotTo(HaveOccurred())
Expect(string(logContent)).To(ContainSubstring(executionError.Error()))
}, w, mockTailer, []*tail.Line{line})
Expect(string(logContent)).To(ContainSubstring(fakes.FakeError.Error()))
})
})
})
func assert(assertion func(err error), watcher watcher.StorageWatcher, mockTailer *fakes.MockTailer, lines []*tail.Line) {
errs := make(chan error, 1)
done := make(chan bool, 1)
go execute(watcher, errs, done)
for _, line := range lines {
mockTailer.Lines <- line
}
close(mockTailer.Lines)
select {
case err := <-errs:
assertion(err)
break
case <-done:
assertion(nil)
break
}
}
func execute(w watcher.StorageWatcher, errs chan error, done chan bool) {
err := w.Execute()
if err != nil {
errs <- err
} else {
done <- true
}
}
func getFakeLine(address []byte) *tail.Line {
blockHash := []byte{4, 5, 6}
blockHeight := int64(789)
storageKey := []byte{9, 8, 7}
storageValue := []byte{6, 5, 4}
return &tail.Line{
Text: fmt.Sprintf("%s,%s,%d,%s,%s", common.Bytes2Hex(address), common.Bytes2Hex(blockHash), blockHeight, common.Bytes2Hex(storageKey), common.Bytes2Hex(storageValue)),
Time: time.Time{},
Err: nil,
}
}