cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

Rename StorageDiffRow -> StorageDiff

Browse Source
This commit is contained in:
Elizabeth Engelman 2019-07-29 14:28:20 -05:00
parent 87252f42b9
commit 045d78be25
19 changed files with 207 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
libraries/shared/factories/storage/transformer.go
View File

@ -41,14 +41,14 @@ func (transformer Transformer) ContractAddress() common.Address {
return transformer.Address return transformer.Address
} }
func (transformer Transformer) Execute(row utils.StorageDiffRow) error { func (transformer Transformer) Execute(diff utils.StorageDiff) error {
metadata, lookupErr := transformer.Mappings.Lookup(row.StorageKey) metadata, lookupErr := transformer.Mappings.Lookup(diff.StorageKey)
if lookupErr != nil { if lookupErr != nil {
return lookupErr return lookupErr
} }
value, decodeErr := utils.Decode(row, metadata) value, decodeErr := utils.Decode(diff, metadata)
if decodeErr != nil { if decodeErr != nil {
return decodeErr return decodeErr
} }
return transformer.Repository.Create(row.BlockHeight, row.BlockHash.Hex(), metadata, value) return transformer.Repository.Create(diff.BlockHeight, diff.BlockHash.Hex(), metadata, value)
} }

12
libraries/shared/factories/storage/transformer_test.go
View File

@ -52,7 +52,7 @@ var _ = Describe("Storage transformer", func() {
}) })
It("looks up metadata for storage key", func() { It("looks up metadata for storage key", func() {
t.Execute(utils.StorageDiffRow{}) t.Execute(utils.StorageDiff{})
Expect(mappings.LookupCalled).To(BeTrue()) Expect(mappings.LookupCalled).To(BeTrue())
}) })
@ -60,7 +60,7 @@ var _ = Describe("Storage transformer", func() {
It("returns error if lookup fails", func() { It("returns error if lookup fails", func() {
mappings.LookupErr = fakes.FakeError mappings.LookupErr = fakes.FakeError
err := t.Execute(utils.StorageDiffRow{}) err := t.Execute(utils.StorageDiff{})
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError)) Expect(err).To(MatchError(fakes.FakeError))
@ -72,7 +72,7 @@ var _ = Describe("Storage transformer", func() {
rawValue := common.HexToAddress("0x12345") rawValue := common.HexToAddress("0x12345")
fakeBlockNumber := 123 fakeBlockNumber := 123
fakeBlockHash := "0x67890" fakeBlockHash := "0x67890"
fakeRow := utils.StorageDiffRow{ fakeRow := utils.StorageDiff{
Contract: common.Address{}, Contract: common.Address{},
BlockHash: common.HexToHash(fakeBlockHash), BlockHash: common.HexToHash(fakeBlockHash),
BlockHeight: fakeBlockNumber, BlockHeight: fakeBlockNumber,
@ -95,7 +95,7 @@ var _ = Describe("Storage transformer", func() {
mappings.Metadata = fakeMetadata mappings.Metadata = fakeMetadata
repository.CreateErr = fakes.FakeError repository.CreateErr = fakes.FakeError
err := t.Execute(utils.StorageDiffRow{StorageValue: rawValue.Hash()}) err := t.Execute(utils.StorageDiff{StorageValue: rawValue.Hash()})
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError)) Expect(err).To(MatchError(fakes.FakeError))
@ -120,7 +120,7 @@ var _ = Describe("Storage transformer", func() {
It("passes the decoded data items to the repository", func() { It("passes the decoded data items to the repository", func() {
mappings.Metadata = fakeMetadata mappings.Metadata = fakeMetadata
fakeRow := utils.StorageDiffRow{ fakeRow := utils.StorageDiff{
Contract: common.Address{}, Contract: common.Address{},
BlockHash: common.HexToHash(fakeBlockHash), BlockHash: common.HexToHash(fakeBlockHash),
BlockHeight: fakeBlockNumber, BlockHeight: fakeBlockNumber,
@ -144,7 +144,7 @@ var _ = Describe("Storage transformer", func() {
mappings.Metadata = fakeMetadata mappings.Metadata = fakeMetadata
repository.CreateErr = fakes.FakeError repository.CreateErr = fakes.FakeError
err := t.Execute(utils.StorageDiffRow{StorageValue: rawValue.Hash()}) err := t.Execute(utils.StorageDiff{StorageValue: rawValue.Hash()})
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError)) Expect(err).To(MatchError(fakes.FakeError))

6
libraries/shared/fetcher/csv_tail_storage_fetcher.go
View File

@ -33,18 +33,18 @@ func NewCsvTailStorageFetcher(tailer fs.Tailer) CsvTailStorageFetcher {
return CsvTailStorageFetcher{tailer: tailer} return CsvTailStorageFetcher{tailer: tailer}
} }
func (storageFetcher CsvTailStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffRow, errs chan<- error) { func (storageFetcher CsvTailStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
t, tailErr := storageFetcher.tailer.Tail() t, tailErr := storageFetcher.tailer.Tail()
if tailErr != nil { if tailErr != nil {
errs <- tailErr errs <- tailErr
} }
log.Debug("fetching storage diffs...") log.Debug("fetching storage diffs...")
for line := range t.Lines { for line := range t.Lines {
row, parseErr := utils.FromStrings(strings.Split(line.Text, ",")) diff, parseErr := utils.FromStrings(strings.Split(line.Text, ","))
if parseErr != nil { if parseErr != nil {
errs <- parseErr errs <- parseErr
} else { } else {
out <- row out <- diff
} }
} }
} }

14
libraries/shared/fetcher/csv_tail_storage_fetcher_test.go
View File

@ -35,13 +35,13 @@ var _ = Describe("Csv Tail Storage Fetcher", func() {
var ( var (
errorsChannel chan error errorsChannel chan error
mockTailer *fakes.MockTailer mockTailer *fakes.MockTailer
rowsChannel chan utils.StorageDiffRow diffsChannel chan utils.StorageDiff
storageFetcher fetcher.CsvTailStorageFetcher storageFetcher fetcher.CsvTailStorageFetcher
) )
BeforeEach(func() { BeforeEach(func() {
errorsChannel = make(chan error) errorsChannel = make(chan error)
rowsChannel = make(chan utils.StorageDiffRow) diffsChannel = make(chan utils.StorageDiff)
mockTailer = fakes.NewMockTailer() mockTailer = fakes.NewMockTailer()
storageFetcher = fetcher.NewCsvTailStorageFetcher(mockTailer) storageFetcher = fetcher.NewCsvTailStorageFetcher(mockTailer)
}) })
@ -49,7 +49,7 @@ var _ = Describe("Csv Tail Storage Fetcher", func() {
It("adds error to errors channel if tailing file fails", func(done Done) { It("adds error to errors channel if tailing file fails", func(done Done) {
mockTailer.TailErr = fakes.FakeError mockTailer.TailErr = fakes.FakeError
go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel) go storageFetcher.FetchStorageDiffs(diffsChannel, errorsChannel)
Expect(<-errorsChannel).To(MatchError(fakes.FakeError)) Expect(<-errorsChannel).To(MatchError(fakes.FakeError))
close(done) close(done)
@ -58,24 +58,24 @@ var _ = Describe("Csv Tail Storage Fetcher", func() {
It("adds parsed csv row to rows channel for storage diff", func(done Done) { It("adds parsed csv row to rows channel for storage diff", func(done Done) {
line := getFakeLine() line := getFakeLine()
go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel) go storageFetcher.FetchStorageDiffs(diffsChannel, errorsChannel)
mockTailer.Lines <- line mockTailer.Lines <- line
expectedRow, err := utils.FromStrings(strings.Split(line.Text, ",")) expectedRow, err := utils.FromStrings(strings.Split(line.Text, ","))
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(<-rowsChannel).To(Equal(expectedRow)) Expect(<-diffsChannel).To(Equal(expectedRow))
close(done) close(done)
}) })
It("adds error to errors channel if parsing csv fails", func(done Done) { It("adds error to errors channel if parsing csv fails", func(done Done) {
line := &tail.Line{Text: "invalid"} line := &tail.Line{Text: "invalid"}
go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel) go storageFetcher.FetchStorageDiffs(diffsChannel, errorsChannel)
mockTailer.Lines <- line mockTailer.Lines <- line
Expect(<-errorsChannel).To(HaveOccurred()) Expect(<-errorsChannel).To(HaveOccurred())
select { select {
case <-rowsChannel: case <-diffsChannel:
Fail("value passed to rows channel on error") Fail("value passed to rows channel on error")
default: default:
Succeed() Succeed()

4
libraries/shared/fetcher/geth_rpc_storage_fetcher.go
View File

@ -36,7 +36,7 @@ func NewGethRpcStorageFetcher(streamer streamer.Streamer, statediffPayloadChan c
} }
} }
func (fetcher *GethRpcStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffRow, errs chan<- error) { func (fetcher *GethRpcStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
ethStatediffPayloadChan := fetcher.statediffPayloadChan ethStatediffPayloadChan := fetcher.statediffPayloadChan
clientSubscription, clientSubErr := fetcher.streamer.Stream(ethStatediffPayloadChan) clientSubscription, clientSubErr := fetcher.streamer.Stream(ethStatediffPayloadChan)
if clientSubErr != nil { if clientSubErr != nil {
@ -61,7 +61,7 @@ func (fetcher *GethRpcStorageFetcher) FetchStorageDiffs(out chan<- utils.Storage
logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account", len(account.Storage))) logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account", len(account.Storage)))
for _, storage := range account.Storage { for _, storage := range account.Storage {
logrus.Trace("adding storage diff to out channel") logrus.Trace("adding storage diff to out channel")
out <- utils.StorageDiffRow{ out <- utils.StorageDiff{
Contract: common.BytesToAddress(account.Key), Contract: common.BytesToAddress(account.Key),
BlockHash: stateDiff.BlockHash, BlockHash: stateDiff.BlockHash,
BlockHeight: int(stateDiff.BlockNumber.Int64()), BlockHeight: int(stateDiff.BlockNumber.Int64()),

20
libraries/shared/fetcher/geth_rpc_storage_fetcher_test.go
View File

@ -57,21 +57,21 @@ var _ = Describe("Geth RPC Storage Fetcher", func() {
var streamer MockStoragediffStreamer var streamer MockStoragediffStreamer
var statediffPayloadChan chan statediff.Payload var statediffPayloadChan chan statediff.Payload
var statediffFetcher fetcher.GethRpcStorageFetcher var statediffFetcher fetcher.GethRpcStorageFetcher
var storagediffRowChan chan utils.StorageDiffRow var storagediffChan chan utils.StorageDiff
var errorChan chan error var errorChan chan error
BeforeEach(func() { BeforeEach(func() {
streamer = MockStoragediffStreamer{} streamer = MockStoragediffStreamer{}
statediffPayloadChan = make(chan statediff.Payload, 1) statediffPayloadChan = make(chan statediff.Payload, 1)
statediffFetcher = fetcher.NewGethRpcStorageFetcher(&streamer, statediffPayloadChan) statediffFetcher = fetcher.NewGethRpcStorageFetcher(&streamer, statediffPayloadChan)
storagediffRowChan = make(chan utils.StorageDiffRow) storagediffChan = make(chan utils.StorageDiff)
errorChan = make(chan error) errorChan = make(chan error)
}) })
It("adds errors to error channel if the RPC subscription fails ", func(done Done) { It("adds errors to error channel if the RPC subscription fails ", func(done Done) {
streamer.SetSubscribeError(fakes.FakeError) streamer.SetSubscribeError(fakes.FakeError)
go statediffFetcher.FetchStorageDiffs(storagediffRowChan, errorChan) go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
Expect(<-errorChan).To(MatchError(fakes.FakeError)) Expect(<-errorChan).To(MatchError(fakes.FakeError))
close(done) close(done)
@ -80,7 +80,7 @@ var _ = Describe("Geth RPC Storage Fetcher", func() {
It("streams StatediffPayloads from a Geth RPC subscription", func(done Done) { It("streams StatediffPayloads from a Geth RPC subscription", func(done Done) {
streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload}) streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload})
go statediffFetcher.FetchStorageDiffs(storagediffRowChan, errorChan) go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
streamedPayload := <-statediffPayloadChan streamedPayload := <-statediffPayloadChan
Expect(streamedPayload).To(Equal(test_data.MockStatediffPayload)) Expect(streamedPayload).To(Equal(test_data.MockStatediffPayload))
@ -91,11 +91,11 @@ var _ = Describe("Geth RPC Storage Fetcher", func() {
It("adds parsed statediff payloads to the rows channel", func(done Done) { It("adds parsed statediff payloads to the rows channel", func(done Done) {
streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload}) streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload})
go statediffFetcher.FetchStorageDiffs(storagediffRowChan, errorChan) go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
height := test_data.BlockNumber height := test_data.BlockNumber
intHeight := int(height.Int64()) intHeight := int(height.Int64())
expectedStorageDiffRow := utils.StorageDiffRow{ expectedStorageDiff := utils.StorageDiff{
//this is not the contract address, but the keccak 256 of the address //this is not the contract address, but the keccak 256 of the address
Contract: common.BytesToAddress(test_data.ContractLeafKey[:]), Contract: common.BytesToAddress(test_data.ContractLeafKey[:]),
BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
@ -103,7 +103,7 @@ var _ = Describe("Geth RPC Storage Fetcher", func() {
StorageKey: common.BytesToHash(test_data.StorageKey), StorageKey: common.BytesToHash(test_data.StorageKey),
StorageValue: common.BytesToHash(test_data.StorageValue), StorageValue: common.BytesToHash(test_data.StorageValue),
} }
anotherExpectedStorageDiffRow := utils.StorageDiffRow{ anotherExpectedStorageDiff := utils.StorageDiff{
//this is not the contract address, but the keccak 256 of the address //this is not the contract address, but the keccak 256 of the address
Contract: common.BytesToAddress(test_data.AnotherContractLeafKey[:]), Contract: common.BytesToAddress(test_data.AnotherContractLeafKey[:]),
BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
@ -111,8 +111,8 @@ var _ = Describe("Geth RPC Storage Fetcher", func() {
StorageKey: common.BytesToHash(test_data.StorageKey), StorageKey: common.BytesToHash(test_data.StorageKey),
StorageValue: common.BytesToHash(test_data.StorageValue), StorageValue: common.BytesToHash(test_data.StorageValue),
} }
Expect(<-storagediffRowChan).To(Equal(expectedStorageDiffRow)) Expect(<-storagediffChan).To(Equal(expectedStorageDiff))
Expect(<-storagediffRowChan).To(Equal(anotherExpectedStorageDiffRow)) Expect(<-storagediffChan).To(Equal(anotherExpectedStorageDiff))
close(done) close(done)
}) })
@ -121,7 +121,7 @@ var _ = Describe("Geth RPC Storage Fetcher", func() {
badStatediffPayload := statediff.Payload{} badStatediffPayload := statediff.Payload{}
streamer.SetPayloads([]statediff.Payload{badStatediffPayload}) streamer.SetPayloads([]statediff.Payload{badStatediffPayload})
go statediffFetcher.FetchStorageDiffs(storagediffRowChan, errorChan) go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
Expect(<-errorChan).To(MatchError("EOF")) Expect(<-errorChan).To(MatchError("EOF"))

2
libraries/shared/fetcher/storage_fetcher_interface.go
View File

@ -17,5 +17,5 @@ package fetcher
import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
type IStorageFetcher interface { type IStorageFetcher interface {
FetchStorageDiffs(out chan<- utils.StorageDiffRow, errs chan<- error) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error)
} }

10
libraries/shared/mocks/storage_fetcher.go
View File

@ -19,21 +19,21 @@ package mocks
import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
type MockStorageFetcher struct { type MockStorageFetcher struct {
RowsToReturn []utils.StorageDiffRow DiffsToReturn []utils.StorageDiff
ErrsToReturn []error ErrsToReturn []error
} }
func NewMockStorageFetcher() *MockStorageFetcher { func NewMockStorageFetcher() *MockStorageFetcher {
return &MockStorageFetcher{} return &MockStorageFetcher{}
} }
func (fetcher *MockStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffRow, errs chan<- error) { func (fetcher *MockStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
defer close(out) defer close(out)
defer close(errs) defer close(errs)
for _, err := range fetcher.ErrsToReturn { for _, err := range fetcher.ErrsToReturn {
errs <- err errs <- err
} }
for _, row := range fetcher.RowsToReturn { for _, diff := range fetcher.DiffsToReturn {
out <- row out <- diff
} }
} }

12
libraries/shared/mocks/storage_queue.go
View File

@ -23,16 +23,16 @@ import (
type MockStorageQueue struct { type MockStorageQueue struct {
AddCalled bool AddCalled bool
AddError error AddError error
AddPassedRow utils.StorageDiffRow AddPassedDiff utils.StorageDiff
DeleteErr error DeleteErr error
DeletePassedId int DeletePassedId int
GetAllErr error GetAllErr error
RowsToReturn []utils.StorageDiffRow DiffsToReturn []utils.StorageDiff
} }
func (queue *MockStorageQueue) Add(row utils.StorageDiffRow) error { func (queue *MockStorageQueue) Add(diff utils.StorageDiff) error {
queue.AddCalled = true queue.AddCalled = true
queue.AddPassedRow = row queue.AddPassedDiff = diff
return queue.AddError return queue.AddError
} }
@ -41,6 +41,6 @@ func (queue *MockStorageQueue) Delete(id int) error {
return queue.DeleteErr return queue.DeleteErr
} }
func (queue *MockStorageQueue) GetAll() ([]utils.StorageDiffRow, error) { func (queue *MockStorageQueue) GetAll() ([]utils.StorageDiff, error) {
return queue.RowsToReturn, queue.GetAllErr return queue.DiffsToReturn, queue.GetAllErr
} }

6
libraries/shared/mocks/storage_transformer.go
View File

@ -27,11 +27,11 @@ import (
type MockStorageTransformer struct { type MockStorageTransformer struct {
Address common.Address Address common.Address
ExecuteErr error ExecuteErr error
PassedRow utils.StorageDiffRow PassedDiff utils.StorageDiff
} }
func (transformer *MockStorageTransformer) Execute(row utils.StorageDiffRow) error { func (transformer *MockStorageTransformer) Execute(diff utils.StorageDiff) error {
transformer.PassedRow = row transformer.PassedDiff = diff
return transformer.ExecuteErr return transformer.ExecuteErr
} }

14
libraries/shared/storage/storage_queue.go
View File

@ -22,9 +22,9 @@ import (
) )
type IStorageQueue interface { type IStorageQueue interface {
Add(row utils.StorageDiffRow) error Add(diff utils.StorageDiff) error
Delete(id int) error Delete(id int) error
GetAll() ([]utils.StorageDiffRow, error) GetAll() ([]utils.StorageDiff, error)
} }
type StorageQueue struct { type StorageQueue struct {
@ -35,11 +35,11 @@ func NewStorageQueue(db *postgres.DB) StorageQueue {
return StorageQueue{db: db} return StorageQueue{db: db}
} }
func (queue StorageQueue) Add(row utils.StorageDiffRow) error { func (queue StorageQueue) Add(diff utils.StorageDiff) error {
_, err := queue.db.Exec(`INSERT INTO public.queued_storage (contract, _, err := queue.db.Exec(`INSERT INTO public.queued_storage (contract,
block_hash, block_height, storage_key, storage_value) VALUES block_hash, block_height, storage_key, storage_value) VALUES
($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`, row.Contract.Bytes(), row.BlockHash.Bytes(), ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`, diff.Contract.Bytes(), diff.BlockHash.Bytes(),
row.BlockHeight, row.StorageKey.Bytes(), row.StorageValue.Bytes()) diff.BlockHeight, diff.StorageKey.Bytes(), diff.StorageValue.Bytes())
return err return err
} }
@ -48,8 +48,8 @@ func (queue StorageQueue) Delete(id int) error {
return err return err
} }
func (queue StorageQueue) GetAll() ([]utils.StorageDiffRow, error) { func (queue StorageQueue) GetAll() ([]utils.StorageDiff, error) {
var result []utils.StorageDiffRow var result []utils.StorageDiff
err := queue.db.Select(&result, `SELECT * FROM public.queued_storage`) err := queue.db.Select(&result, `SELECT * FROM public.queued_storage`)
return result, err return result, err
} }

60
libraries/shared/storage/storage_queue_test.go
View File

@ -30,12 +30,12 @@ import (
var _ = Describe("Storage queue", func() { var _ = Describe("Storage queue", func() {
var ( var (
db *postgres.DB db *postgres.DB
row utils.StorageDiffRow diff utils.StorageDiff
queue storage.IStorageQueue queue storage.IStorageQueue
) )
BeforeEach(func() { BeforeEach(func() {
row = utils.StorageDiffRow{ diff = utils.StorageDiff{
Contract: common.HexToAddress("0x123456"), Contract: common.HexToAddress("0x123456"),
BlockHash: common.HexToHash("0x678901"), BlockHash: common.HexToHash("0x678901"),
BlockHeight: 987, BlockHeight: 987,
@ -45,20 +45,20 @@ var _ = Describe("Storage queue", func() {
db = test_config.NewTestDB(test_config.NewTestNode()) db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)
queue = storage.NewStorageQueue(db) queue = storage.NewStorageQueue(db)
addErr := queue.Add(row) addErr := queue.Add(diff)
Expect(addErr).NotTo(HaveOccurred()) Expect(addErr).NotTo(HaveOccurred())
}) })
Describe("Add", func() { Describe("Add", func() {
It("adds a storage row to the db", func() { It("adds a storage diff to the db", func() {
var result utils.StorageDiffRow var result utils.StorageDiff
getErr := db.Get(&result, `SELECT contract, block_hash, block_height, storage_key, storage_value FROM public.queued_storage`) getErr := db.Get(&result, `SELECT contract, block_hash, block_height, storage_key, storage_value FROM public.queued_storage`)
Expect(getErr).NotTo(HaveOccurred()) Expect(getErr).NotTo(HaveOccurred())
Expect(result).To(Equal(row)) Expect(result).To(Equal(diff))
}) })
It("does not duplicate storage rows", func() { It("does not duplicate storage diffs", func() {
addErr := queue.Add(row) addErr := queue.Add(diff)
Expect(addErr).NotTo(HaveOccurred()) Expect(addErr).NotTo(HaveOccurred())
var count int var count int
getErr := db.Get(&count, `SELECT count(*) FROM public.queued_storage`) getErr := db.Get(&count, `SELECT count(*) FROM public.queued_storage`)
@ -67,12 +67,12 @@ var _ = Describe("Storage queue", func() {
}) })
}) })
It("deletes storage row from db", func() { It("deletes storage diff from db", func() {
rows, getErr := queue.GetAll() diffs, getErr := queue.GetAll()
Expect(getErr).NotTo(HaveOccurred()) Expect(getErr).NotTo(HaveOccurred())
Expect(len(rows)).To(Equal(1)) Expect(len(diffs)).To(Equal(1))
err := queue.Delete(rows[0].Id) err := queue.Delete(diffs[0].Id)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
remainingRows, secondGetErr := queue.GetAll() remainingRows, secondGetErr := queue.GetAll()
@ -80,33 +80,33 @@ var _ = Describe("Storage queue", func() {
Expect(len(remainingRows)).To(BeZero()) Expect(len(remainingRows)).To(BeZero())
}) })
It("gets all storage rows from db", func() { It("gets all storage diffs from db", func() {
rowTwo := utils.StorageDiffRow{ diffTwo := utils.StorageDiff{
Contract: common.HexToAddress("0x123456"), Contract: common.HexToAddress("0x123456"),
BlockHash: common.HexToHash("0x678902"), BlockHash: common.HexToHash("0x678902"),
BlockHeight: 988, BlockHeight: 988,
StorageKey: common.HexToHash("0x654322"), StorageKey: common.HexToHash("0x654322"),
StorageValue: common.HexToHash("0x198766"), StorageValue: common.HexToHash("0x198766"),
} }
addErr := queue.Add(rowTwo) addErr := queue.Add(diffTwo)
Expect(addErr).NotTo(HaveOccurred()) Expect(addErr).NotTo(HaveOccurred())
rows, err := queue.GetAll() diffs, err := queue.GetAll()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(len(rows)).To(Equal(2)) Expect(len(diffs)).To(Equal(2))
Expect(rows[0]).NotTo(Equal(rows[1])) Expect(diffs[0]).NotTo(Equal(diffs[1]))
Expect(rows[0].Id).NotTo(BeZero()) Expect(diffs[0].Id).NotTo(BeZero())
Expect(rows[0].Contract).To(Or(Equal(row.Contract), Equal(rowTwo.Contract))) Expect(diffs[0].Contract).To(Or(Equal(diff.Contract), Equal(diffTwo.Contract)))
Expect(rows[0].BlockHash).To(Or(Equal(row.BlockHash), Equal(rowTwo.BlockHash))) Expect(diffs[0].BlockHash).To(Or(Equal(diff.BlockHash), Equal(diffTwo.BlockHash)))
Expect(rows[0].BlockHeight).To(Or(Equal(row.BlockHeight), Equal(rowTwo.BlockHeight))) Expect(diffs[0].BlockHeight).To(Or(Equal(diff.BlockHeight), Equal(diffTwo.BlockHeight)))
Expect(rows[0].StorageKey).To(Or(Equal(row.StorageKey), Equal(rowTwo.StorageKey))) Expect(diffs[0].StorageKey).To(Or(Equal(diff.StorageKey), Equal(diffTwo.StorageKey)))
Expect(rows[0].StorageValue).To(Or(Equal(row.StorageValue), Equal(rowTwo.StorageValue))) Expect(diffs[0].StorageValue).To(Or(Equal(diff.StorageValue), Equal(diffTwo.StorageValue)))
Expect(rows[1].Id).NotTo(BeZero()) Expect(diffs[1].Id).NotTo(BeZero())
Expect(rows[1].Contract).To(Or(Equal(row.Contract), Equal(rowTwo.Contract))) Expect(diffs[1].Contract).To(Or(Equal(diff.Contract), Equal(diffTwo.Contract)))
Expect(rows[1].BlockHash).To(Or(Equal(row.BlockHash), Equal(rowTwo.BlockHash))) Expect(diffs[1].BlockHash).To(Or(Equal(diff.BlockHash), Equal(diffTwo.BlockHash)))
Expect(rows[1].BlockHeight).To(Or(Equal(row.BlockHeight), Equal(rowTwo.BlockHeight))) Expect(diffs[1].BlockHeight).To(Or(Equal(diff.BlockHeight), Equal(diffTwo.BlockHeight)))
Expect(rows[1].StorageKey).To(Or(Equal(row.StorageKey), Equal(rowTwo.StorageKey))) Expect(diffs[1].StorageKey).To(Or(Equal(diff.StorageKey), Equal(diffTwo.StorageKey)))
Expect(rows[1].StorageValue).To(Or(Equal(row.StorageValue), Equal(rowTwo.StorageValue))) Expect(diffs[1].StorageValue).To(Or(Equal(diff.StorageValue), Equal(diffTwo.StorageValue)))
}) })
}) })

14
libraries/shared/storage/utils/decoder.go
View File

@ -27,20 +27,20 @@ const (
bitsPerByte = 8 bitsPerByte = 8
) )
func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, error) { func Decode(diff StorageDiff, metadata StorageValueMetadata) (interface{}, error) {
switch metadata.Type { switch metadata.Type {
case Uint256: case Uint256:
return decodeInteger(row.StorageValue.Bytes()), nil return decodeInteger(diff.StorageValue.Bytes()), nil
case Uint48: case Uint48:
return decodeInteger(row.StorageValue.Bytes()), nil return decodeInteger(diff.StorageValue.Bytes()), nil
case Uint128: case Uint128:
return decodeInteger(row.StorageValue.Bytes()), nil return decodeInteger(diff.StorageValue.Bytes()), nil
case Address: case Address:
return decodeAddress(row.StorageValue.Bytes()), nil return decodeAddress(diff.StorageValue.Bytes()), nil
case Bytes32: case Bytes32:
return row.StorageValue.Hex(), nil return diff.StorageValue.Hex(), nil
case PackedSlot: case PackedSlot:
return decodePackedSlot(row.StorageValue.Bytes(), metadata.PackedTypes), nil return decodePackedSlot(diff.StorageValue.Bytes(), metadata.PackedTypes), nil
default: default:
panic(fmt.Sprintf("can't decode unknown type: %d", metadata.Type)) panic(fmt.Sprintf("can't decode unknown type: %d", metadata.Type))
} }

30
libraries/shared/storage/utils/decoder_test.go
View File

@ -29,10 +29,10 @@ import (
var _ = Describe("Storage decoder", func() { var _ = Describe("Storage decoder", func() {
It("decodes uint256", func() { It("decodes uint256", func() {
fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000539") fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000539")
row := utils.StorageDiffRow{StorageValue: fakeInt} diff := utils.StorageDiff{StorageValue: fakeInt}
metadata := utils.StorageValueMetadata{Type: utils.Uint256} metadata := utils.StorageValueMetadata{Type: utils.Uint256}
result, err := utils.Decode(row, metadata) result, err := utils.Decode(diff, metadata)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String())) Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
@ -40,10 +40,10 @@ var _ = Describe("Storage decoder", func() {
It("decodes uint128", func() { It("decodes uint128", func() {
fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000011123") fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000011123")
row := utils.StorageDiffRow{StorageValue: fakeInt} diff := utils.StorageDiff{StorageValue: fakeInt}
metadata := utils.StorageValueMetadata{Type: utils.Uint128} metadata := utils.StorageValueMetadata{Type: utils.Uint128}
result, err := utils.Decode(row, metadata) result, err := utils.Decode(diff, metadata)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String())) Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
@ -51,10 +51,10 @@ var _ = Describe("Storage decoder", func() {
It("decodes uint48", func() { It("decodes uint48", func() {
fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000123") fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000123")
row := utils.StorageDiffRow{StorageValue: fakeInt} diff := utils.StorageDiff{StorageValue: fakeInt}
metadata := utils.StorageValueMetadata{Type: utils.Uint48} metadata := utils.StorageValueMetadata{Type: utils.Uint48}
result, err := utils.Decode(row, metadata) result, err := utils.Decode(diff, metadata)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String())) Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
@ -62,10 +62,10 @@ var _ = Describe("Storage decoder", func() {
It("decodes address", func() { It("decodes address", func() {
fakeAddress := common.HexToAddress("0x12345") fakeAddress := common.HexToAddress("0x12345")
row := utils.StorageDiffRow{StorageValue: fakeAddress.Hash()} diff := utils.StorageDiff{StorageValue: fakeAddress.Hash()}
metadata := utils.StorageValueMetadata{Type: utils.Address} metadata := utils.StorageValueMetadata{Type: utils.Address}
result, err := utils.Decode(row, metadata) result, err := utils.Decode(diff, metadata)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(result).To(Equal(fakeAddress.Hex())) Expect(result).To(Equal(fakeAddress.Hex()))
@ -75,7 +75,7 @@ var _ = Describe("Storage decoder", func() {
It("decodes uint48 items", func() { It("decodes uint48 items", func() {
//this is a real storage data example //this is a real storage data example
packedStorage := common.HexToHash("000000000000000000000000000000000000000000000002a300000000002a30") packedStorage := common.HexToHash("000000000000000000000000000000000000000000000002a300000000002a30")
row := utils.StorageDiffRow{StorageValue: packedStorage} diff := utils.StorageDiff{StorageValue: packedStorage}
packedTypes := map[int]utils.ValueType{} packedTypes := map[int]utils.ValueType{}
packedTypes[0] = utils.Uint48 packedTypes[0] = utils.Uint48
packedTypes[1] = utils.Uint48 packedTypes[1] = utils.Uint48
@ -85,7 +85,7 @@ var _ = Describe("Storage decoder", func() {
PackedTypes: packedTypes, PackedTypes: packedTypes,
} }
result, err := utils.Decode(row, metadata) result, err := utils.Decode(diff, metadata)
decodedValues := result.(map[int]string) decodedValues := result.(map[int]string)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
@ -99,7 +99,7 @@ var _ = Describe("Storage decoder", func() {
packedStorageHex := "0000000A5D1AFFFFFFFFFFFE00000009F3C600000002A300000000002A30" packedStorageHex := "0000000A5D1AFFFFFFFFFFFE00000009F3C600000002A300000000002A30"
packedStorage := common.HexToHash(packedStorageHex) packedStorage := common.HexToHash(packedStorageHex)
row := utils.StorageDiffRow{StorageValue: packedStorage} diff := utils.StorageDiff{StorageValue: packedStorage}
packedTypes := map[int]utils.ValueType{} packedTypes := map[int]utils.ValueType{}
packedTypes[0] = utils.Uint48 packedTypes[0] = utils.Uint48
packedTypes[1] = utils.Uint48 packedTypes[1] = utils.Uint48
@ -112,7 +112,7 @@ var _ = Describe("Storage decoder", func() {
PackedTypes: packedTypes, PackedTypes: packedTypes,
} }
result, err := utils.Decode(row, metadata) result, err := utils.Decode(diff, metadata)
decodedValues := result.(map[int]string) decodedValues := result.(map[int]string)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
@ -129,7 +129,7 @@ var _ = Describe("Storage decoder", func() {
packedStorageHex := "000000038D7EA4C67FF8E502B6730000" + packedStorageHex := "000000038D7EA4C67FF8E502B6730000" +
"0000000000000000AB54A98CEB1F0AD2" "0000000000000000AB54A98CEB1F0AD2"
packedStorage := common.HexToHash(packedStorageHex) packedStorage := common.HexToHash(packedStorageHex)
row := utils.StorageDiffRow{StorageValue: packedStorage} diff := utils.StorageDiff{StorageValue: packedStorage}
packedTypes := map[int]utils.ValueType{} packedTypes := map[int]utils.ValueType{}
packedTypes[0] = utils.Uint128 packedTypes[0] = utils.Uint128
packedTypes[1] = utils.Uint128 packedTypes[1] = utils.Uint128
@ -139,7 +139,7 @@ var _ = Describe("Storage decoder", func() {
PackedTypes: packedTypes, PackedTypes: packedTypes,
} }
result, err := utils.Decode(row, metadata) result, err := utils.Decode(diff, metadata)
decodedValues := result.(map[int]string) decodedValues := result.(map[int]string)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
@ -151,7 +151,7 @@ var _ = Describe("Storage decoder", func() {
//TODO: replace with real data when available //TODO: replace with real data when available
addressHex := "0000000000000000000000000000000000012345" addressHex := "0000000000000000000000000000000000012345"
packedStorage := common.HexToHash("00000002a300" + "000000002a30" + addressHex) packedStorage := common.HexToHash("00000002a300" + "000000002a30" + addressHex)
row := utils.StorageDiffRow{StorageValue: packedStorage} row := utils.StorageDiff{StorageValue: packedStorage}
packedTypes := map[int]utils.ValueType{} packedTypes := map[int]utils.ValueType{}
packedTypes[0] = utils.Address packedTypes[0] = utils.Address
packedTypes[1] = utils.Uint48 packedTypes[1] = utils.Uint48

10
libraries/shared/storage/utils/row.go → libraries/shared/storage/utils/diff.go
View File

@ -24,7 +24,7 @@ import (
const ExpectedRowLength = 5 const ExpectedRowLength = 5
type StorageDiffRow struct { type StorageDiff struct {
Id int Id int
Contract common.Address Contract common.Address
BlockHash common.Hash `db:"block_hash"` BlockHash common.Hash `db:"block_hash"`
@ -33,15 +33,15 @@ type StorageDiffRow struct {
StorageValue common.Hash `db:"storage_value"` StorageValue common.Hash `db:"storage_value"`
} }
func FromStrings(csvRow []string) (StorageDiffRow, error) { func FromStrings(csvRow []string) (StorageDiff, error) {
if len(csvRow) != ExpectedRowLength { if len(csvRow) != ExpectedRowLength {
return StorageDiffRow{}, ErrRowMalformed{Length: len(csvRow)} return StorageDiff{}, ErrRowMalformed{Length: len(csvRow)}
} }
height, err := strconv.Atoi(csvRow[2]) height, err := strconv.Atoi(csvRow[2])
if err != nil { if err != nil {
return StorageDiffRow{}, err return StorageDiff{}, err
} }
return StorageDiffRow{ return StorageDiff{
Contract: common.HexToAddress(csvRow[0]), Contract: common.HexToAddress(csvRow[0]),
BlockHash: common.HexToHash(csvRow[1]), BlockHash: common.HexToHash(csvRow[1]),
BlockHeight: height, BlockHeight: height,

0
libraries/shared/storage/utils/row_test.go → libraries/shared/storage/utils/diff_test.go
View File

2
libraries/shared/transformer/storage_transformer.go
View File

@ -24,7 +24,7 @@ import (
) )
type StorageTransformer interface { type StorageTransformer interface {
Execute(row utils.StorageDiffRow) error Execute(diff utils.StorageDiff) error
ContractAddress() common.Address ContractAddress() common.Address
} }

36
libraries/shared/watcher/storage_watcher.go
View File

@ -63,15 +63,15 @@ func (storageWatcher StorageWatcher) AddTransformers(initializers []transformer.
} }
} }
func (storageWatcher StorageWatcher) Execute(rows chan utils.StorageDiffRow, errs chan error, queueRecheckInterval time.Duration) { func (storageWatcher StorageWatcher) Execute(diffsChan chan utils.StorageDiff, errsChan chan error, queueRecheckInterval time.Duration) {
ticker := time.NewTicker(queueRecheckInterval) ticker := time.NewTicker(queueRecheckInterval)
go storageWatcher.StorageFetcher.FetchStorageDiffs(rows, errs) go storageWatcher.StorageFetcher.FetchStorageDiffs(diffsChan, errsChan)
for { for {
select { select {
case fetchErr := <-errs: case fetchErr := <-errsChan:
logrus.Warn(fmt.Sprintf("error fetching storage diffs: %s", fetchErr)) logrus.Warn(fmt.Sprintf("error fetching storage diffs: %s", fetchErr))
case row := <-rows: case diff := <-diffsChan:
storageWatcher.processRow(row) storageWatcher.processRow(diff)
case <-ticker.C: case <-ticker.C:
storageWatcher.processQueue() storageWatcher.processQueue()
} }
@ -96,16 +96,16 @@ func (storageWatcher StorageWatcher) getTransformer(contractAddress common.Addre
return nil, false return nil, false
} }
func (storageWatcher StorageWatcher) processRow(row utils.StorageDiffRow) { func (storageWatcher StorageWatcher) processRow(diff utils.StorageDiff) {
storageTransformer, ok := storageWatcher.getTransformer(row.Contract) storageTransformer, ok := storageWatcher.getTransformer(diff.Contract)
if !ok { if !ok {
logrus.Debug("ignoring a row from an unwatched contract") logrus.Debug("ignoring a diff from an unwatched contract")
return return
} }
executeErr := storageTransformer.Execute(row) executeErr := storageTransformer.Execute(diff)
if executeErr != nil { if executeErr != nil {
logrus.Warn(fmt.Sprintf("error executing storage transformer: %s", executeErr)) logrus.Warn(fmt.Sprintf("error executing storage transformer: %s", executeErr))
queueErr := storageWatcher.Queue.Add(row) queueErr := storageWatcher.Queue.Add(diff)
if queueErr != nil { if queueErr != nil {
logrus.Warn(fmt.Sprintf("error queueing storage diff: %s", queueErr)) logrus.Warn(fmt.Sprintf("error queueing storage diff: %s", queueErr))
} }
@ -113,20 +113,20 @@ func (storageWatcher StorageWatcher) processRow(row utils.StorageDiffRow) {
} }
func (storageWatcher StorageWatcher) processQueue() { func (storageWatcher StorageWatcher) processQueue() {
rows, fetchErr := storageWatcher.Queue.GetAll() diffs, fetchErr := storageWatcher.Queue.GetAll()
if fetchErr != nil { if fetchErr != nil {
logrus.Warn(fmt.Sprintf("error getting queued storage: %s", fetchErr)) logrus.Warn(fmt.Sprintf("error getting queued storage: %s", fetchErr))
} }
for _, row := range rows { for _, diff := range diffs {
storageTransformer, ok := storageWatcher.getTransformer(row.Contract) storageTransformer, ok := storageWatcher.getTransformer(diff.Contract)
if !ok { if !ok {
// delete row from queue if address no longer watched // delete diff from queue if address no longer watched
storageWatcher.deleteRow(row.Id) storageWatcher.deleteRow(diff.Id)
continue continue
} }
executeErr := storageTransformer.Execute(row) executeErr := storageTransformer.Execute(diff)
if executeErr == nil { if executeErr == nil {
storageWatcher.deleteRow(row.Id) storageWatcher.deleteRow(diff.Id)
} }
} }
} }
@ -134,7 +134,7 @@ func (storageWatcher StorageWatcher) processQueue() {
func (storageWatcher StorageWatcher) deleteRow(id int) { func (storageWatcher StorageWatcher) deleteRow(id int) {
deleteErr := storageWatcher.Queue.Delete(id) deleteErr := storageWatcher.Queue.Delete(id)
if deleteErr != nil { if deleteErr != nil {
logrus.Warn(fmt.Sprintf("error deleting persisted row from queue: %s", deleteErr)) logrus.Warn(fmt.Sprintf("error deleting persisted diff from queue: %s", deleteErr))
} }
} }

154
libraries/shared/watcher/storage_watcher_test.go
View File

@ -52,20 +52,20 @@ var _ = Describe("Storage Watcher", func() {
mockFetcher *mocks.MockStorageFetcher mockFetcher *mocks.MockStorageFetcher
mockQueue *mocks.MockStorageQueue mockQueue *mocks.MockStorageQueue
mockTransformer *mocks.MockStorageTransformer mockTransformer *mocks.MockStorageTransformer
csvRow utils.StorageDiffRow csvDiff utils.StorageDiff
gethRow utils.StorageDiffRow gethDiff utils.StorageDiff
rows chan utils.StorageDiffRow diffs chan utils.StorageDiff
storageWatcher watcher.StorageWatcher storageWatcher watcher.StorageWatcher
) )
BeforeEach(func() { BeforeEach(func() {
errs = make(chan error) errs = make(chan error)
rows = make(chan utils.StorageDiffRow) diffs = make(chan utils.StorageDiff)
address := common.HexToAddress("0x0123456789abcdef") address := common.HexToAddress("0x0123456789abcdef")
mockFetcher = mocks.NewMockStorageFetcher() mockFetcher = mocks.NewMockStorageFetcher()
mockQueue = &mocks.MockStorageQueue{} mockQueue = &mocks.MockStorageQueue{}
mockTransformer = &mocks.MockStorageTransformer{Address: address} mockTransformer = &mocks.MockStorageTransformer{Address: address}
csvRow = utils.StorageDiffRow{ csvDiff = utils.StorageDiff{
Id: 1337, Id: 1337,
Contract: address, Contract: address,
BlockHash: common.HexToHash("0xfedcba9876543210"), BlockHash: common.HexToHash("0xfedcba9876543210"),
@ -73,7 +73,7 @@ var _ = Describe("Storage Watcher", func() {
StorageKey: common.HexToHash("0xabcdef1234567890"), StorageKey: common.HexToHash("0xabcdef1234567890"),
StorageValue: common.HexToHash("0x9876543210abcdef"), StorageValue: common.HexToHash("0x9876543210abcdef"),
} }
gethRow = utils.StorageDiffRow{ gethDiff = utils.StorageDiff{
Id: 1338, Id: 1338,
Contract: common.BytesToAddress(crypto.Keccak256(address[:])), Contract: common.BytesToAddress(crypto.Keccak256(address[:])),
BlockHash: common.HexToHash("0xfedcba9876543210"), BlockHash: common.HexToHash("0xfedcba9876543210"),
@ -93,7 +93,7 @@ var _ = Describe("Storage Watcher", func() {
defer os.Remove(tempFile.Name()) defer os.Remove(tempFile.Name())
logrus.SetOutput(tempFile) logrus.SetOutput(tempFile)
go storageWatcher.Execute(rows, errs, time.Hour) go storageWatcher.Execute(diffs, errs, time.Hour)
Eventually(func() (string, error) { Eventually(func() (string, error) {
logContent, err := ioutil.ReadFile(tempFile.Name()) logContent, err := ioutil.ReadFile(tempFile.Name())
@ -105,37 +105,37 @@ var _ = Describe("Storage Watcher", func() {
Describe("transforming new storage diffs from csv", func() { Describe("transforming new storage diffs from csv", func() {
Describe("where diff source is a csv file", func() { Describe("where diff source is a csv file", func() {
BeforeEach(func() { BeforeEach(func() {
mockFetcher.RowsToReturn = []utils.StorageDiffRow{csvRow} mockFetcher.DiffsToReturn = []utils.StorageDiff{csvDiff}
storageWatcher = watcher.NewStorageWatcher(mockFetcher, test_config.NewTestDB(test_config.NewTestNode())) storageWatcher = watcher.NewStorageWatcher(mockFetcher, test_config.NewTestDB(test_config.NewTestNode()))
storageWatcher.Queue = mockQueue storageWatcher.Queue = mockQueue
storageWatcher.AddTransformers([]transformer.StorageTransformerInitializer{mockTransformer.FakeTransformerInitializer}) storageWatcher.AddTransformers([]transformer.StorageTransformerInitializer{mockTransformer.FakeTransformerInitializer})
}) })
It("executes transformer for recognized storage row", func(done Done) { It("executes transformer for recognized storage diff", func(done Done) {
go storageWatcher.Execute(rows, errs, time.Hour) go storageWatcher.Execute(diffs, errs, time.Hour)
Eventually(func() utils.StorageDiffRow { Eventually(func() utils.StorageDiff {
return mockTransformer.PassedRow return mockTransformer.PassedDiff
}).Should(Equal(csvRow)) }).Should(Equal(csvDiff))
close(done) close(done)
}) })
It("queues row for later processing if transformer execution fails", func(done Done) { It("queues diff for later processing if transformer execution fails", func(done Done) {
mockTransformer.ExecuteErr = fakes.FakeError mockTransformer.ExecuteErr = fakes.FakeError
go storageWatcher.Execute(rows, errs, time.Hour) go storageWatcher.Execute(diffs, errs, time.Hour)
Expect(<-errs).To(BeNil()) Expect(<-errs).To(BeNil())
Eventually(func() bool { Eventually(func() bool {
return mockQueue.AddCalled return mockQueue.AddCalled
}).Should(BeTrue()) }).Should(BeTrue())
Eventually(func() utils.StorageDiffRow { Eventually(func() utils.StorageDiff {
return mockQueue.AddPassedRow return mockQueue.AddPassedDiff
}).Should(Equal(csvRow)) }).Should(Equal(csvDiff))
close(done) close(done)
}) })
It("logs error if queueing row fails", func(done Done) { It("logs error if queueing diff fails", func(done Done) {
mockTransformer.ExecuteErr = utils.ErrStorageKeyNotFound{} mockTransformer.ExecuteErr = utils.ErrStorageKeyNotFound{}
mockQueue.AddError = fakes.FakeError mockQueue.AddError = fakes.FakeError
tempFile, fileErr := ioutil.TempFile("", "log") tempFile, fileErr := ioutil.TempFile("", "log")
@ -143,7 +143,7 @@ var _ = Describe("Storage Watcher", func() {
defer os.Remove(tempFile.Name()) defer os.Remove(tempFile.Name())
logrus.SetOutput(tempFile) logrus.SetOutput(tempFile)
go storageWatcher.Execute(rows, errs, time.Hour) go storageWatcher.Execute(diffs, errs, time.Hour)
Eventually(func() bool { Eventually(func() bool {
return mockQueue.AddCalled return mockQueue.AddCalled
@ -158,38 +158,38 @@ var _ = Describe("Storage Watcher", func() {
Describe("where diff source is geth RPC pub sub", func() { Describe("where diff source is geth RPC pub sub", func() {
BeforeEach(func() { BeforeEach(func() {
mockFetcher.RowsToReturn = []utils.StorageDiffRow{gethRow} mockFetcher.DiffsToReturn = []utils.StorageDiff{gethDiff}
storageWatcher = watcher.NewStorageWatcher(mockFetcher, test_config.NewTestDB(test_config.NewTestNode())) storageWatcher = watcher.NewStorageWatcher(mockFetcher, test_config.NewTestDB(test_config.NewTestNode()))
storageWatcher.SetStorageDiffSource("geth") storageWatcher.SetStorageDiffSource("geth")
storageWatcher.Queue = mockQueue storageWatcher.Queue = mockQueue
storageWatcher.AddTransformers([]transformer.StorageTransformerInitializer{mockTransformer.FakeTransformerInitializer}) storageWatcher.AddTransformers([]transformer.StorageTransformerInitializer{mockTransformer.FakeTransformerInitializer})
}) })
It("executes transformer for recognized storage row", func(done Done) { It("executes transformer for recognized storage diff", func(done Done) {
go storageWatcher.Execute(rows, errs, time.Hour) go storageWatcher.Execute(diffs, errs, time.Hour)
Eventually(func() utils.StorageDiffRow { Eventually(func() utils.StorageDiff {
return mockTransformer.PassedRow return mockTransformer.PassedDiff
}).Should(Equal(gethRow)) }).Should(Equal(gethDiff))
close(done) close(done)
}) })
It("queues row for later processing if transformer execution fails", func(done Done) { It("queues diff for later processing if transformer execution fails", func(done Done) {
mockTransformer.ExecuteErr = fakes.FakeError mockTransformer.ExecuteErr = fakes.FakeError
go storageWatcher.Execute(rows, errs, time.Hour) go storageWatcher.Execute(diffs, errs, time.Hour)
Expect(<-errs).To(BeNil()) Expect(<-errs).To(BeNil())
Eventually(func() bool { Eventually(func() bool {
return mockQueue.AddCalled return mockQueue.AddCalled
}).Should(BeTrue()) }).Should(BeTrue())
Eventually(func() utils.StorageDiffRow { Eventually(func() utils.StorageDiff {
return mockQueue.AddPassedRow return mockQueue.AddPassedDiff
}).Should(Equal(gethRow)) }).Should(Equal(gethDiff))
close(done) close(done)
}) })
It("logs error if queueing row fails", func(done Done) { It("logs error if queueing diff fails", func(done Done) {
mockTransformer.ExecuteErr = utils.ErrStorageKeyNotFound{} mockTransformer.ExecuteErr = utils.ErrStorageKeyNotFound{}
mockQueue.AddError = fakes.FakeError mockQueue.AddError = fakes.FakeError
tempFile, fileErr := ioutil.TempFile("", "log") tempFile, fileErr := ioutil.TempFile("", "log")
@ -197,7 +197,7 @@ var _ = Describe("Storage Watcher", func() {
defer os.Remove(tempFile.Name()) defer os.Remove(tempFile.Name())
logrus.SetOutput(tempFile) logrus.SetOutput(tempFile)
go storageWatcher.Execute(rows, errs, time.Hour) go storageWatcher.Execute(diffs, errs, time.Hour)
Eventually(func() bool { Eventually(func() bool {
return mockQueue.AddCalled return mockQueue.AddCalled
@ -214,38 +214,38 @@ var _ = Describe("Storage Watcher", func() {
Describe("transforming queued storage diffs", func() { Describe("transforming queued storage diffs", func() {
Describe("where diff source is a csv file", func() { Describe("where diff source is a csv file", func() {
BeforeEach(func() { BeforeEach(func() {
mockQueue.RowsToReturn = []utils.StorageDiffRow{csvRow} mockQueue.DiffsToReturn = []utils.StorageDiff{csvDiff}
storageWatcher = watcher.NewStorageWatcher(mockFetcher, test_config.NewTestDB(test_config.NewTestNode())) storageWatcher = watcher.NewStorageWatcher(mockFetcher, test_config.NewTestDB(test_config.NewTestNode()))
storageWatcher.Queue = mockQueue storageWatcher.Queue = mockQueue
storageWatcher.AddTransformers([]transformer.StorageTransformerInitializer{mockTransformer.FakeTransformerInitializer}) storageWatcher.AddTransformers([]transformer.StorageTransformerInitializer{mockTransformer.FakeTransformerInitializer})
}) })
It("executes transformer for storage row", func(done Done) { It("executes transformer for storage diff", func(done Done) {
go storageWatcher.Execute(rows, errs, time.Nanosecond) go storageWatcher.Execute(diffs, errs, time.Nanosecond)
Eventually(func() utils.StorageDiffRow { Eventually(func() utils.StorageDiff {
return mockTransformer.PassedRow return mockTransformer.PassedDiff
}).Should(Equal(csvRow)) }).Should(Equal(csvDiff))
close(done) close(done)
}) })
It("deletes row from queue if transformer execution successful", func(done Done) { It("deletes diff from queue if transformer execution successful", func(done Done) {
go storageWatcher.Execute(rows, errs, time.Nanosecond) go storageWatcher.Execute(diffs, errs, time.Nanosecond)
Eventually(func() int { Eventually(func() int {
return mockQueue.DeletePassedId return mockQueue.DeletePassedId
}).Should(Equal(csvRow.Id)) }).Should(Equal(csvDiff.Id))
close(done) close(done)
}) })
It("logs error if deleting persisted row fails", func(done Done) { It("logs error if deleting persisted diff fails", func(done Done) {
mockQueue.DeleteErr = fakes.FakeError mockQueue.DeleteErr = fakes.FakeError
tempFile, fileErr := ioutil.TempFile("", "log") tempFile, fileErr := ioutil.TempFile("", "log")
Expect(fileErr).NotTo(HaveOccurred()) Expect(fileErr).NotTo(HaveOccurred())
defer os.Remove(tempFile.Name()) defer os.Remove(tempFile.Name())
logrus.SetOutput(tempFile) logrus.SetOutput(tempFile)
go storageWatcher.Execute(rows, errs, time.Nanosecond) go storageWatcher.Execute(diffs, errs, time.Nanosecond)
Eventually(func() (string, error) { Eventually(func() (string, error) {
logContent, err := ioutil.ReadFile(tempFile.Name()) logContent, err := ioutil.ReadFile(tempFile.Name())
@ -254,34 +254,34 @@ var _ = Describe("Storage Watcher", func() {
close(done) close(done)
}) })
It("deletes obsolete row from queue if contract not recognized", func(done Done) { It("deletes obsolete diff from queue if contract not recognized", func(done Done) {
obsoleteRow := utils.StorageDiffRow{ obsoleteDiff := utils.StorageDiff{
Id: csvRow.Id + 1, Id: csvDiff.Id + 1,
Contract: common.HexToAddress("0xfedcba9876543210"), Contract: common.HexToAddress("0xfedcba9876543210"),
} }
mockQueue.RowsToReturn = []utils.StorageDiffRow{obsoleteRow} mockQueue.DiffsToReturn = []utils.StorageDiff{obsoleteDiff}
go storageWatcher.Execute(rows, errs, time.Nanosecond) go storageWatcher.Execute(diffs, errs, time.Nanosecond)
Eventually(func() int { Eventually(func() int {
return mockQueue.DeletePassedId return mockQueue.DeletePassedId
}).Should(Equal(obsoleteRow.Id)) }).Should(Equal(obsoleteDiff.Id))
close(done) close(done)
}) })
It("logs error if deleting obsolete row fails", func(done Done) { It("logs error if deleting obsolete diff fails", func(done Done) {
obsoleteRow := utils.StorageDiffRow{ obsoleteDiff := utils.StorageDiff{
Id: csvRow.Id + 1, Id: csvDiff.Id + 1,
Contract: common.HexToAddress("0xfedcba9876543210"), Contract: common.HexToAddress("0xfedcba9876543210"),
} }
mockQueue.RowsToReturn = []utils.StorageDiffRow{obsoleteRow} mockQueue.DiffsToReturn = []utils.StorageDiff{obsoleteDiff}
mockQueue.DeleteErr = fakes.FakeError mockQueue.DeleteErr = fakes.FakeError
tempFile, fileErr := ioutil.TempFile("", "log") tempFile, fileErr := ioutil.TempFile("", "log")
Expect(fileErr).NotTo(HaveOccurred()) Expect(fileErr).NotTo(HaveOccurred())
defer os.Remove(tempFile.Name()) defer os.Remove(tempFile.Name())
logrus.SetOutput(tempFile) logrus.SetOutput(tempFile)
go storageWatcher.Execute(rows, errs, time.Nanosecond) go storageWatcher.Execute(diffs, errs, time.Nanosecond)
Eventually(func() (string, error) { Eventually(func() (string, error) {
logContent, err := ioutil.ReadFile(tempFile.Name()) logContent, err := ioutil.ReadFile(tempFile.Name())
@ -293,39 +293,39 @@ var _ = Describe("Storage Watcher", func() {
Describe("where diff source is geth RPC pub sub", func() { Describe("where diff source is geth RPC pub sub", func() {
BeforeEach(func() { BeforeEach(func() {
mockQueue.RowsToReturn = []utils.StorageDiffRow{gethRow} mockQueue.DiffsToReturn = []utils.StorageDiff{gethDiff}
storageWatcher = watcher.NewStorageWatcher(mockFetcher, test_config.NewTestDB(test_config.NewTestNode())) storageWatcher = watcher.NewStorageWatcher(mockFetcher, test_config.NewTestDB(test_config.NewTestNode()))
storageWatcher.Queue = mockQueue storageWatcher.Queue = mockQueue
storageWatcher.SetStorageDiffSource("geth") storageWatcher.SetStorageDiffSource("geth")
storageWatcher.AddTransformers([]transformer.StorageTransformerInitializer{mockTransformer.FakeTransformerInitializer}) storageWatcher.AddTransformers([]transformer.StorageTransformerInitializer{mockTransformer.FakeTransformerInitializer})
}) })
It("executes transformer for storage row", func(done Done) { It("executes transformer for storage diff", func(done Done) {
go storageWatcher.Execute(rows, errs, time.Nanosecond) go storageWatcher.Execute(diffs, errs, time.Nanosecond)
Eventually(func() utils.StorageDiffRow { Eventually(func() utils.StorageDiff {
return mockTransformer.PassedRow return mockTransformer.PassedDiff
}).Should(Equal(gethRow)) }).Should(Equal(gethDiff))
close(done) close(done)
}) })
It("deletes row from queue if transformer execution successful", func(done Done) { It("deletes diff from queue if transformer execution successful", func(done Done) {
go storageWatcher.Execute(rows, errs, time.Nanosecond) go storageWatcher.Execute(diffs, errs, time.Nanosecond)
Eventually(func() int { Eventually(func() int {
return mockQueue.DeletePassedId return mockQueue.DeletePassedId
}).Should(Equal(gethRow.Id)) }).Should(Equal(gethDiff.Id))
close(done) close(done)
}) })
It("logs error if deleting persisted row fails", func(done Done) { It("logs error if deleting persisted diff fails", func(done Done) {
mockQueue.DeleteErr = fakes.FakeError mockQueue.DeleteErr = fakes.FakeError
tempFile, fileErr := ioutil.TempFile("", "log") tempFile, fileErr := ioutil.TempFile("", "log")
Expect(fileErr).NotTo(HaveOccurred()) Expect(fileErr).NotTo(HaveOccurred())
defer os.Remove(tempFile.Name()) defer os.Remove(tempFile.Name())
logrus.SetOutput(tempFile) logrus.SetOutput(tempFile)
go storageWatcher.Execute(rows, errs, time.Nanosecond) go storageWatcher.Execute(diffs, errs, time.Nanosecond)
Eventually(func() (string, error) { Eventually(func() (string, error) {
logContent, err := ioutil.ReadFile(tempFile.Name()) logContent, err := ioutil.ReadFile(tempFile.Name())
@ -334,34 +334,34 @@ var _ = Describe("Storage Watcher", func() {
close(done) close(done)
}) })
It("deletes obsolete row from queue if contract not recognized", func(done Done) { It("deletes obsolete diff from queue if contract not recognized", func(done Done) {
obsoleteRow := utils.StorageDiffRow{ obsoleteDiff := utils.StorageDiff{
Id: gethRow.Id + 1, Id: gethDiff.Id + 1,
Contract: common.HexToAddress("0xfedcba9876543210"), Contract: common.HexToAddress("0xfedcba9876543210"),
} }
mockQueue.RowsToReturn = []utils.StorageDiffRow{obsoleteRow} mockQueue.DiffsToReturn = []utils.StorageDiff{obsoleteDiff}
go storageWatcher.Execute(rows, errs, time.Nanosecond) go storageWatcher.Execute(diffs, errs, time.Nanosecond)
Eventually(func() int { Eventually(func() int {
return mockQueue.DeletePassedId return mockQueue.DeletePassedId
}).Should(Equal(obsoleteRow.Id)) }).Should(Equal(obsoleteDiff.Id))
close(done) close(done)
}) })
It("logs error if deleting obsolete row fails", func(done Done) { It("logs error if deleting obsolete diff fails", func(done Done) {
obsoleteRow := utils.StorageDiffRow{ obsoleteDiff := utils.StorageDiff{
Id: gethRow.Id + 1, Id: gethDiff.Id + 1,
Contract: common.HexToAddress("0xfedcba9876543210"), Contract: common.HexToAddress("0xfedcba9876543210"),
} }
mockQueue.RowsToReturn = []utils.StorageDiffRow{obsoleteRow} mockQueue.DiffsToReturn = []utils.StorageDiff{obsoleteDiff}
mockQueue.DeleteErr = fakes.FakeError mockQueue.DeleteErr = fakes.FakeError
tempFile, fileErr := ioutil.TempFile("", "log") tempFile, fileErr := ioutil.TempFile("", "log")
Expect(fileErr).NotTo(HaveOccurred()) Expect(fileErr).NotTo(HaveOccurred())
defer os.Remove(tempFile.Name()) defer os.Remove(tempFile.Name())
logrus.SetOutput(tempFile) logrus.SetOutput(tempFile)
go storageWatcher.Execute(rows, errs, time.Nanosecond) go storageWatcher.Execute(diffs, errs, time.Nanosecond)
Eventually(func() (string, error) { Eventually(func() (string, error) {
logContent, err := ioutil.ReadFile(tempFile.Name()) logContent, err := ioutil.ReadFile(tempFile.Name())