diff --git a/cmd/composeAndExecute.go b/cmd/composeAndExecute.go index cb244e17..f1e4bf95 100644 --- a/cmd/composeAndExecute.go +++ b/cmd/composeAndExecute.go @@ -17,19 +17,19 @@ package cmd import ( - "github.com/ethereum/go-ethereum/statediff" - "github.com/vulcanize/vulcanizedb/libraries/shared/fetcher" - "github.com/vulcanize/vulcanizedb/libraries/shared/streamer" - "github.com/vulcanize/vulcanizedb/pkg/fs" "os" "plugin" syn "sync" "time" + "github.com/ethereum/go-ethereum/statediff" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/vulcanize/vulcanizedb/libraries/shared/fetcher" + "github.com/vulcanize/vulcanizedb/libraries/shared/streamer" "github.com/vulcanize/vulcanizedb/libraries/shared/watcher" + "github.com/vulcanize/vulcanizedb/pkg/fs" p2 "github.com/vulcanize/vulcanizedb/pkg/plugin" "github.com/vulcanize/vulcanizedb/pkg/plugin/helpers" "github.com/vulcanize/vulcanizedb/utils" diff --git a/cmd/contractWatcher.go b/cmd/contractWatcher.go index 2e2a8814..11238869 100644 --- a/cmd/contractWatcher.go +++ b/cmd/contractWatcher.go @@ -18,13 +18,13 @@ package cmd import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/config" "time" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" st "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" + "github.com/vulcanize/vulcanizedb/pkg/config" ft "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer" ht "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/transformer" "github.com/vulcanize/vulcanizedb/utils" diff --git a/cmd/execute.go b/cmd/execute.go index c68b7e08..2f2c31ea 100644 --- a/cmd/execute.go +++ b/cmd/execute.go @@ -18,21 +18,21 @@ package cmd import ( "fmt" - "github.com/ethereum/go-ethereum/statediff" - "github.com/vulcanize/vulcanizedb/libraries/shared/fetcher" - "github.com/vulcanize/vulcanizedb/libraries/shared/streamer" - "github.com/vulcanize/vulcanizedb/pkg/fs" "plugin" syn "sync" "time" + "github.com/ethereum/go-ethereum/statediff" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/vulcanize/vulcanizedb/libraries/shared/constants" + "github.com/vulcanize/vulcanizedb/libraries/shared/fetcher" storageUtils "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" + "github.com/vulcanize/vulcanizedb/libraries/shared/streamer" "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" "github.com/vulcanize/vulcanizedb/libraries/shared/watcher" + "github.com/vulcanize/vulcanizedb/pkg/fs" "github.com/vulcanize/vulcanizedb/utils" ) diff --git a/libraries/shared/chunker/chunker_suite_test.go b/libraries/shared/chunker/chunker_suite_test.go index 0c5d4562..b2189eef 100644 --- a/libraries/shared/chunker/chunker_suite_test.go +++ b/libraries/shared/chunker/chunker_suite_test.go @@ -19,10 +19,11 @@ package chunker_test import ( "testing" + "io/ioutil" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" log "github.com/sirupsen/logrus" - "io/ioutil" ) func TestFactories(t *testing.T) { diff --git a/libraries/shared/chunker/log_chunker.go b/libraries/shared/chunker/log_chunker.go index 38edd5af..67771ab6 100644 --- a/libraries/shared/chunker/log_chunker.go +++ b/libraries/shared/chunker/log_chunker.go @@ -17,10 +17,11 @@ package chunker import ( + "strings" + "github.com/ethereum/go-ethereum/common" "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" "github.com/vulcanize/vulcanizedb/pkg/core" - "strings" ) type Chunker interface { diff --git a/libraries/shared/factories/event/factories_suite_test.go b/libraries/shared/factories/event/factories_suite_test.go index f2687b75..d205475e 100644 --- a/libraries/shared/factories/event/factories_suite_test.go +++ b/libraries/shared/factories/event/factories_suite_test.go @@ -19,10 +19,11 @@ package event_test import ( "testing" + "io/ioutil" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" log "github.com/sirupsen/logrus" - "io/ioutil" ) func TestFactories(t *testing.T) { diff --git a/libraries/shared/factories/event/transformer_test.go b/libraries/shared/factories/event/transformer_test.go index 0b452409..dad7d138 100644 --- a/libraries/shared/factories/event/transformer_test.go +++ b/libraries/shared/factories/event/transformer_test.go @@ -17,6 +17,8 @@ package event_test import ( + "math/rand" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/libraries/shared/factories/event" @@ -25,7 +27,6 @@ import ( "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" - "math/rand" ) var _ = Describe("Transformer", func() { diff --git a/libraries/shared/fetcher/csv_tail_storage_fetcher.go b/libraries/shared/fetcher/csv_tail_storage_fetcher.go index 01dc3caa..2205ef38 100644 --- a/libraries/shared/fetcher/csv_tail_storage_fetcher.go +++ b/libraries/shared/fetcher/csv_tail_storage_fetcher.go @@ -17,10 +17,11 @@ package fetcher import ( + "strings" + "github.com/sirupsen/logrus" "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" "github.com/vulcanize/vulcanizedb/pkg/fs" - "strings" ) type CsvTailStorageFetcher struct { diff --git a/libraries/shared/fetcher/csv_tail_storage_fetcher_test.go b/libraries/shared/fetcher/csv_tail_storage_fetcher_test.go index eaca0eb3..cf0f06f5 100644 --- a/libraries/shared/fetcher/csv_tail_storage_fetcher_test.go +++ b/libraries/shared/fetcher/csv_tail_storage_fetcher_test.go @@ -18,6 +18,9 @@ package fetcher_test import ( "fmt" + "strings" + "time" + "github.com/ethereum/go-ethereum/common" "github.com/hpcloud/tail" . "github.com/onsi/ginkgo" @@ -25,8 +28,6 @@ import ( "github.com/vulcanize/vulcanizedb/libraries/shared/fetcher" "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" "github.com/vulcanize/vulcanizedb/pkg/fakes" - "strings" - "time" ) var _ = Describe("Csv Tail Storage Fetcher", func() { diff --git a/libraries/shared/fetcher/fetcher_suite_test.go b/libraries/shared/fetcher/fetcher_suite_test.go index e1bb832a..691fb502 100644 --- a/libraries/shared/fetcher/fetcher_suite_test.go +++ b/libraries/shared/fetcher/fetcher_suite_test.go @@ -19,10 +19,11 @@ package fetcher_test import ( "testing" + "io/ioutil" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" log "github.com/sirupsen/logrus" - "io/ioutil" ) func TestFactories(t *testing.T) { diff --git a/libraries/shared/logs/delegator.go b/libraries/shared/logs/delegator.go index 79b398ad..35b31601 100644 --- a/libraries/shared/logs/delegator.go +++ b/libraries/shared/logs/delegator.go @@ -18,6 +18,7 @@ package logs import ( "errors" + "github.com/sirupsen/logrus" "github.com/vulcanize/vulcanizedb/libraries/shared/chunker" "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" diff --git a/libraries/shared/logs/delegator_test.go b/libraries/shared/logs/delegator_test.go index 43813be8..9a56e831 100644 --- a/libraries/shared/logs/delegator_test.go +++ b/libraries/shared/logs/delegator_test.go @@ -17,6 +17,8 @@ package logs_test import ( + "strings" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" @@ -27,7 +29,6 @@ import ( "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" - "strings" ) var _ = Describe("Log delegator", func() { diff --git a/libraries/shared/logs/extractor.go b/libraries/shared/logs/extractor.go index a28802ae..89f5dec6 100644 --- a/libraries/shared/logs/extractor.go +++ b/libraries/shared/logs/extractor.go @@ -18,6 +18,7 @@ package logs import ( "errors" + "github.com/ethereum/go-ethereum/common" "github.com/sirupsen/logrus" "github.com/vulcanize/vulcanizedb/libraries/shared/constants" diff --git a/libraries/shared/logs/extractor_test.go b/libraries/shared/logs/extractor_test.go index f3b623d2..954baf29 100644 --- a/libraries/shared/logs/extractor_test.go +++ b/libraries/shared/logs/extractor_test.go @@ -17,6 +17,8 @@ package logs_test import ( + "math/rand" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" @@ -27,7 +29,6 @@ import ( "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" - "math/rand" ) var _ = Describe("Log extractor", func() { diff --git a/libraries/shared/logs/logs_suite_test.go b/libraries/shared/logs/logs_suite_test.go index 5227bac7..9a04f74f 100644 --- a/libraries/shared/logs/logs_suite_test.go +++ b/libraries/shared/logs/logs_suite_test.go @@ -17,10 +17,11 @@ package logs_test import ( - "github.com/sirupsen/logrus" "io/ioutil" "testing" + "github.com/sirupsen/logrus" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) diff --git a/libraries/shared/repository/repository_suite_test.go b/libraries/shared/repository/repository_suite_test.go index d934f9f8..0868f7fc 100644 --- a/libraries/shared/repository/repository_suite_test.go +++ b/libraries/shared/repository/repository_suite_test.go @@ -19,10 +19,11 @@ package repository_test import ( "testing" + "io/ioutil" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" log "github.com/sirupsen/logrus" - "io/ioutil" ) func TestFactories(t *testing.T) { diff --git a/libraries/shared/storage/backfill.go b/libraries/shared/storage/backfill.go index bde07288..a0aaadef 100644 --- a/libraries/shared/storage/backfill.go +++ b/libraries/shared/storage/backfill.go @@ -31,18 +31,22 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/geth/client" ) +// IBackFiller is the backfilling interface type IBackFiller interface { BackFill(bfa BackFillerArgs) (map[common.Hash][]utils.StorageDiff, error) } +// BatchClient is an interface to a batch-fetching geth rpc client; created to allow mock insertion type BatchClient interface { BatchCall(batch []client.BatchElem) error } +// BackFiller is the backfilling struct type BackFiller struct { client BatchClient } +// BackFillerArgs are used to pass configuration params to the backfiller type BackFillerArgs struct { // mapping of hashed addresses to a list of the storage key hashes we want to collect at that address WantedStorage map[common.Hash][]common.Hash @@ -52,6 +56,7 @@ type BackFillerArgs struct { const method = "statediff_stateDiffAt" +// NewStorageBackFiller returns a IBackFiller func NewStorageBackFiller(bc BatchClient) IBackFiller { return &BackFiller{ client: bc, diff --git a/libraries/shared/storage/backfill_test.go b/libraries/shared/storage/backfill_test.go index 7f243ff4..7dd24f8d 100644 --- a/libraries/shared/storage/backfill_test.go +++ b/libraries/shared/storage/backfill_test.go @@ -33,12 +33,12 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/geth/client" ) -type MockClient struct { +type mockClient struct { MappedStateDiffAt map[uint64][]byte - MappedErrors map[uint64]error } -func (mc *MockClient) SetReturnDiffAt(height uint64, diffPayload statediff.Payload) error { +// SetReturnDiffAt method to set what statediffs the mock client returns +func (mc *mockClient) SetReturnDiffAt(height uint64, diffPayload statediff.Payload) error { if mc.MappedStateDiffAt == nil { mc.MappedStateDiffAt = make(map[uint64][]byte) } @@ -50,15 +50,9 @@ func (mc *MockClient) SetReturnDiffAt(height uint64, diffPayload statediff.Paylo return nil } -func (mc *MockClient) SetReturnErrorAt(height uint64, err error) { - if mc.MappedErrors == nil { - mc.MappedErrors = make(map[uint64]error) - } - mc.MappedErrors[height] = err -} - -func (mc *MockClient) BatchCall(batch []client.BatchElem) error { - if mc.MappedStateDiffAt == nil || mc.MappedErrors == nil { +// BatchCall mockClient method to simulate batch call to geth +func (mc *mockClient) BatchCall(batch []client.BatchElem) error { + if mc.MappedStateDiffAt == nil { return errors.New("mockclient needs to be initialized with statediff payloads and errors") } for _, batchElem := range batch { @@ -73,22 +67,25 @@ func (mc *MockClient) BatchCall(batch []client.BatchElem) error { if err != nil { return err } - batchElem.Error = mc.MappedErrors[blockHeight] } return nil } var _ = Describe("BackFiller", func() { Describe("BackFill", func() { - It("Batch calls statediff_stateDiffAt", func() { - mc := new(MockClient) + var ( + mc *mockClient + backFiller storage.IBackFiller + ) + BeforeEach(func() { + mc = new(mockClient) setDiffAtErr1 := mc.SetReturnDiffAt(test_data.BlockNumber.Uint64(), test_data.MockStatediffPayload) Expect(setDiffAtErr1).ToNot(HaveOccurred()) setDiffAtErr2 := mc.SetReturnDiffAt(test_data.BlockNumber2.Uint64(), test_data.MockStatediffPayload2) Expect(setDiffAtErr2).ToNot(HaveOccurred()) - mc.SetReturnErrorAt(test_data.BlockNumber.Uint64(), nil) - mc.SetReturnErrorAt(test_data.BlockNumber2.Uint64(), nil) - backFiller := storage.NewStorageBackFiller(mc) + backFiller = storage.NewStorageBackFiller(mc) + }) + It("Batch calls statediff_stateDiffAt", func() { backFillArgs := storage.BackFillerArgs{ WantedStorage: map[common.Hash][]common.Hash{ test_data.ContractLeafKey: {common.BytesToHash(test_data.StorageKey)}, @@ -127,14 +124,6 @@ var _ = Describe("BackFiller", func() { }) It("Only returns storage for provided addresses (address hashes)", func() { - mc := new(MockClient) - setDiffAtErr1 := mc.SetReturnDiffAt(test_data.BlockNumber.Uint64(), test_data.MockStatediffPayload) - Expect(setDiffAtErr1).ToNot(HaveOccurred()) - setDiffAtErr2 := mc.SetReturnDiffAt(test_data.BlockNumber2.Uint64(), test_data.MockStatediffPayload2) - Expect(setDiffAtErr2).ToNot(HaveOccurred()) - mc.SetReturnErrorAt(test_data.BlockNumber.Uint64(), nil) - mc.SetReturnErrorAt(test_data.BlockNumber2.Uint64(), nil) - backFiller := storage.NewStorageBackFiller(mc) backFillArgs := storage.BackFillerArgs{ WantedStorage: map[common.Hash][]common.Hash{ test_data.ContractLeafKey: {common.BytesToHash(test_data.StorageKey)}, @@ -151,14 +140,6 @@ var _ = Describe("BackFiller", func() { }) It("Only returns storage for provided storage keys", func() { - mc := new(MockClient) - setDiffAtErr1 := mc.SetReturnDiffAt(test_data.BlockNumber.Uint64(), test_data.MockStatediffPayload) - Expect(setDiffAtErr1).ToNot(HaveOccurred()) - setDiffAtErr2 := mc.SetReturnDiffAt(test_data.BlockNumber2.Uint64(), test_data.MockStatediffPayload2) - Expect(setDiffAtErr2).ToNot(HaveOccurred()) - mc.SetReturnErrorAt(test_data.BlockNumber.Uint64(), nil) - mc.SetReturnErrorAt(test_data.BlockNumber2.Uint64(), nil) - backFiller := storage.NewStorageBackFiller(mc) backFillArgs := storage.BackFillerArgs{ WantedStorage: map[common.Hash][]common.Hash{ test_data.ContractLeafKey: nil, diff --git a/libraries/shared/storage/storage_suite_test.go b/libraries/shared/storage/storage_suite_test.go index a24f35d3..c22b8dba 100644 --- a/libraries/shared/storage/storage_suite_test.go +++ b/libraries/shared/storage/storage_suite_test.go @@ -19,10 +19,11 @@ package storage_test import ( "testing" + "io/ioutil" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" log "github.com/sirupsen/logrus" - "io/ioutil" ) func TestFactories(t *testing.T) { diff --git a/libraries/shared/storage/utils/diff_test.go b/libraries/shared/storage/utils/diff_test.go index 2de496ac..1c530a7c 100644 --- a/libraries/shared/storage/utils/diff_test.go +++ b/libraries/shared/storage/utils/diff_test.go @@ -17,6 +17,9 @@ package utils_test import ( + "math/big" + "math/rand" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff" @@ -25,8 +28,6 @@ import ( "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" "github.com/vulcanize/vulcanizedb/libraries/shared/test_data" "github.com/vulcanize/vulcanizedb/pkg/fakes" - "math/big" - "math/rand" ) var _ = Describe("Storage row parsing", func() { diff --git a/libraries/shared/storage/utils/utils_suite_test.go b/libraries/shared/storage/utils/utils_suite_test.go index f77d6860..51a53ac6 100644 --- a/libraries/shared/storage/utils/utils_suite_test.go +++ b/libraries/shared/storage/utils/utils_suite_test.go @@ -17,10 +17,11 @@ package utils_test import ( - "github.com/sirupsen/logrus" "io/ioutil" "testing" + "github.com/sirupsen/logrus" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) diff --git a/libraries/shared/test_data/generic.go b/libraries/shared/test_data/generic.go index 43d3ba12..07be69b8 100644 --- a/libraries/shared/test_data/generic.go +++ b/libraries/shared/test_data/generic.go @@ -17,13 +17,14 @@ package test_data import ( + "math/rand" + "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/vulcanize/vulcanizedb/libraries/shared/factories/event" "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" - "math/rand" - "time" ) var startingBlockNumber = rand.Int63() diff --git a/libraries/shared/watcher/event_watcher.go b/libraries/shared/watcher/event_watcher.go index d6028c5c..32fd873a 100644 --- a/libraries/shared/watcher/event_watcher.go +++ b/libraries/shared/watcher/event_watcher.go @@ -17,6 +17,8 @@ package watcher import ( + "time" + "github.com/sirupsen/logrus" "github.com/vulcanize/vulcanizedb/libraries/shared/chunker" "github.com/vulcanize/vulcanizedb/libraries/shared/constants" @@ -27,7 +29,6 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "time" ) const NoNewDataPause = time.Second * 7 diff --git a/libraries/shared/watcher/event_watcher_test.go b/libraries/shared/watcher/event_watcher_test.go index 78221db9..9315684b 100644 --- a/libraries/shared/watcher/event_watcher_test.go +++ b/libraries/shared/watcher/event_watcher_test.go @@ -18,6 +18,7 @@ package watcher_test import ( "errors" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/libraries/shared/constants" diff --git a/libraries/shared/watcher/storage_watcher.go b/libraries/shared/watcher/storage_watcher.go index 38467c43..76a99e95 100644 --- a/libraries/shared/watcher/storage_watcher.go +++ b/libraries/shared/watcher/storage_watcher.go @@ -18,6 +18,8 @@ package watcher import ( "fmt" + "time" + "github.com/ethereum/go-ethereum/common" "github.com/sirupsen/logrus" "github.com/vulcanize/vulcanizedb/libraries/shared/fetcher" @@ -25,7 +27,6 @@ import ( "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "time" ) type IStorageWatcher interface { diff --git a/libraries/shared/watcher/storage_watcher_test.go b/libraries/shared/watcher/storage_watcher_test.go index 7cc9a115..391dfd6c 100644 --- a/libraries/shared/watcher/storage_watcher_test.go +++ b/libraries/shared/watcher/storage_watcher_test.go @@ -17,6 +17,10 @@ package watcher_test import ( + "io/ioutil" + "os" + "time" + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -27,9 +31,6 @@ import ( "github.com/vulcanize/vulcanizedb/libraries/shared/watcher" "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/test_config" - "io/ioutil" - "os" - "time" ) var _ = Describe("Storage Watcher", func() { diff --git a/libraries/shared/watcher/watcher_suite_test.go b/libraries/shared/watcher/watcher_suite_test.go index 9685a721..f480cc3a 100644 --- a/libraries/shared/watcher/watcher_suite_test.go +++ b/libraries/shared/watcher/watcher_suite_test.go @@ -17,10 +17,11 @@ package watcher_test import ( - log "github.com/sirupsen/logrus" "io/ioutil" "testing" + log "github.com/sirupsen/logrus" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" )