From 90e67d6da9f97fadc22ae98affcad6840ed70aae Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Sun, 10 Feb 2019 16:42:41 -0600 Subject: [PATCH] move storage mappings pkg into shared --- cmd/composeAndExecute.go | 29 +++++---- cmd/root.go | 1 + libraries/shared/mocks/storage_transformer.go | 2 +- .../storage_repository.go | 25 ++------ libraries/shared/storage/mappings.go | 61 +++++++++++++++++++ .../shared/{ => storage}/utils/decoder.go | 0 .../{ => storage}/utils/decoder_test.go | 2 +- .../shared/{ => storage}/utils/errors.go | 0 libraries/shared/{ => storage}/utils/row.go | 0 .../shared/{ => storage}/utils/row_test.go | 2 +- .../{ => storage}/utils/shared_suite_test.go | 0 libraries/shared/{ => storage}/utils/value.go | 0 .../shared/transformer/storage_transformer.go | 2 +- libraries/shared/watcher/storage_watcher.go | 2 +- .../shared/watcher/storage_watcher_test.go | 2 +- 15 files changed, 91 insertions(+), 37 deletions(-) rename libraries/shared/{mocks => repository}/storage_repository.go (53%) create mode 100644 libraries/shared/storage/mappings.go rename libraries/shared/{ => storage}/utils/decoder.go (100%) rename libraries/shared/{ => storage}/utils/decoder_test.go (96%) rename libraries/shared/{ => storage}/utils/errors.go (100%) rename libraries/shared/{ => storage}/utils/row.go (100%) rename libraries/shared/{ => storage}/utils/row_test.go (96%) rename libraries/shared/{ => storage}/utils/shared_suite_test.go (100%) rename libraries/shared/{ => storage}/utils/value.go (100%) diff --git a/cmd/composeAndExecute.go b/cmd/composeAndExecute.go index 1d31171b..09172fe5 100644 --- a/cmd/composeAndExecute.go +++ b/cmd/composeAndExecute.go @@ -18,6 +18,7 @@ package cmd import ( "errors" "fmt" + "github.com/vulcanize/vulcanizedb/libraries/shared/constants" "log" "os" "plugin" @@ -151,18 +152,18 @@ func composeAndExecute() { // Use WaitGroup to wait on both goroutines var wg syn.WaitGroup if len(ethEventInitializers) > 0 { - w := watcher.NewWatcher(&db, blockChain) - w.AddTransformers(ethEventInitializers) + ew := watcher.NewEventWatcher(&db, blockChain) + ew.AddTransformers(ethEventInitializers) wg.Add(1) - go watchEthEvents(&w, &wg) + go watchEthEvents(&ew, &wg) } if len(ethStorageInitializers) > 0 { tailer := fs.FileTailer{Path: storageDiffsPath} - w := watcher.NewStorageWatcher(tailer, &db) - w.AddTransformers(ethStorageInitializers) + sw := watcher.NewStorageWatcher(tailer, &db) + sw.AddTransformers(ethStorageInitializers) wg.Add(1) - go watchEthStorage(&w, &wg) + go watchEthStorage(&sw, &wg) } wg.Wait() } @@ -173,17 +174,23 @@ type Exporter interface { func init() { rootCmd.AddCommand(composeAndExecuteCmd) - composeAndExecuteCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start transformer execution from") + composeAndExecuteCmd.Flags().BoolVar(&recheckHeadersArg, "recheckHeaders", false, "checks headers that are already checked for each transformer.") } -func watchEthEvents(w *watcher.Watcher, wg *syn.WaitGroup) { +func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) { defer wg.Done() // Execute over the TransformerInitializer set using the watcher - fmt.Println("executing transformers") + fmt.Println("executing event transformers") + var recheck constants.TransformerExecution + if recheckHeadersArg { + recheck = constants.HeaderRecheck + } else { + recheck = constants.HeaderMissing + } ticker := time.NewTicker(pollingInterval) defer ticker.Stop() for range ticker.C { - err := w.Execute() + err := w.Execute(recheck) if err != nil { // TODO Handle watcher errors in composeAndExecute } @@ -193,7 +200,7 @@ func watchEthEvents(w *watcher.Watcher, wg *syn.WaitGroup) { func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) { defer wg.Done() // Execute over the TransformerInitializer set using the watcher - fmt.Println("executing transformers") + fmt.Println("executing storage transformers") ticker := time.NewTicker(pollingInterval) defer ticker.Stop() for range ticker.C { diff --git a/cmd/root.go b/cmd/root.go index 0cfafd6e..b7b4d993 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -45,6 +45,7 @@ var ( storageDiffsPath string syncAll bool endingBlockNumber int64 + recheckHeadersArg bool ) const ( diff --git a/libraries/shared/mocks/storage_transformer.go b/libraries/shared/mocks/storage_transformer.go index f5a778b2..047a9cc3 100644 --- a/libraries/shared/mocks/storage_transformer.go +++ b/libraries/shared/mocks/storage_transformer.go @@ -19,8 +19,8 @@ package mocks import ( "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" - "github.com/vulcanize/vulcanizedb/libraries/shared/utils" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) diff --git a/libraries/shared/mocks/storage_repository.go b/libraries/shared/repository/storage_repository.go similarity index 53% rename from libraries/shared/mocks/storage_repository.go rename to libraries/shared/repository/storage_repository.go index 035adf16..379362e3 100644 --- a/libraries/shared/mocks/storage_repository.go +++ b/libraries/shared/repository/storage_repository.go @@ -14,29 +14,14 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package mocks +package repository import ( - "github.com/vulcanize/vulcanizedb/libraries/shared/utils" + "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -type MockStorageRepository struct { - CreateErr error - PassedBlockNumber int - PassedBlockHash string - PassedMetadata utils.StorageValueMetadata - PassedValue interface{} -} - -func (repository *MockStorageRepository) Create(blockNumber int, blockHash string, metadata utils.StorageValueMetadata, value interface{}) error { - repository.PassedBlockNumber = blockNumber - repository.PassedBlockHash = blockHash - repository.PassedMetadata = metadata - repository.PassedValue = value - return repository.CreateErr -} - -func (*MockStorageRepository) SetDB(db *postgres.DB) { - panic("implement me") +type StorageRepository interface { + Create(blockNumber int, blockHash string, metadata utils.StorageValueMetadata, value interface{}) error + SetDB(db *postgres.DB) } diff --git a/libraries/shared/storage/mappings.go b/libraries/shared/storage/mappings.go new file mode 100644 index 00000000..fb52edd9 --- /dev/null +++ b/libraries/shared/storage/mappings.go @@ -0,0 +1,61 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package storage + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + + "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type Mappings interface { + Lookup(key common.Hash) (utils.StorageValueMetadata, error) + SetDB(db *postgres.DB) +} + +const ( + IndexZero = "0000000000000000000000000000000000000000000000000000000000000000" + IndexOne = "0000000000000000000000000000000000000000000000000000000000000001" + IndexTwo = "0000000000000000000000000000000000000000000000000000000000000002" + IndexThree = "0000000000000000000000000000000000000000000000000000000000000003" + IndexFour = "0000000000000000000000000000000000000000000000000000000000000004" + IndexFive = "0000000000000000000000000000000000000000000000000000000000000005" + IndexSix = "0000000000000000000000000000000000000000000000000000000000000006" + IndexSeven = "0000000000000000000000000000000000000000000000000000000000000007" +) + +func GetMapping(indexOnContract, key string) common.Hash { + keyBytes := common.FromHex("0x" + key + indexOnContract) + encoded := crypto.Keccak256(keyBytes) + return common.BytesToHash(encoded) +} + +func GetNestedMapping(indexOnContract, primaryKey, secondaryKey string) common.Hash { + primaryMappingIndex := crypto.Keccak256(common.FromHex(primaryKey + indexOnContract)) + secondaryMappingIndex := crypto.Keccak256(common.FromHex(secondaryKey), primaryMappingIndex) + return common.BytesToHash(secondaryMappingIndex) +} + +func GetIncrementedKey(original common.Hash, incrementBy int64) common.Hash { + originalMappingAsInt := original.Big() + incremented := big.NewInt(0).Add(originalMappingAsInt, big.NewInt(incrementBy)) + return common.BytesToHash(incremented.Bytes()) +} diff --git a/libraries/shared/utils/decoder.go b/libraries/shared/storage/utils/decoder.go similarity index 100% rename from libraries/shared/utils/decoder.go rename to libraries/shared/storage/utils/decoder.go diff --git a/libraries/shared/utils/decoder_test.go b/libraries/shared/storage/utils/decoder_test.go similarity index 96% rename from libraries/shared/utils/decoder_test.go rename to libraries/shared/storage/utils/decoder_test.go index 7bd7238e..28721d50 100644 --- a/libraries/shared/utils/decoder_test.go +++ b/libraries/shared/storage/utils/decoder_test.go @@ -23,7 +23,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/libraries/shared/utils" + "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" ) var _ = Describe("Storage decoder", func() { diff --git a/libraries/shared/utils/errors.go b/libraries/shared/storage/utils/errors.go similarity index 100% rename from libraries/shared/utils/errors.go rename to libraries/shared/storage/utils/errors.go diff --git a/libraries/shared/utils/row.go b/libraries/shared/storage/utils/row.go similarity index 100% rename from libraries/shared/utils/row.go rename to libraries/shared/storage/utils/row.go diff --git a/libraries/shared/utils/row_test.go b/libraries/shared/storage/utils/row_test.go similarity index 96% rename from libraries/shared/utils/row_test.go rename to libraries/shared/storage/utils/row_test.go index c1ae1e7d..261085c3 100644 --- a/libraries/shared/utils/row_test.go +++ b/libraries/shared/storage/utils/row_test.go @@ -21,7 +21,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/libraries/shared/utils" + "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" ) var _ = Describe("Storage row parsing", func() { diff --git a/libraries/shared/utils/shared_suite_test.go b/libraries/shared/storage/utils/shared_suite_test.go similarity index 100% rename from libraries/shared/utils/shared_suite_test.go rename to libraries/shared/storage/utils/shared_suite_test.go diff --git a/libraries/shared/utils/value.go b/libraries/shared/storage/utils/value.go similarity index 100% rename from libraries/shared/utils/value.go rename to libraries/shared/storage/utils/value.go diff --git a/libraries/shared/transformer/storage_transformer.go b/libraries/shared/transformer/storage_transformer.go index 19203314..22ca9378 100644 --- a/libraries/shared/transformer/storage_transformer.go +++ b/libraries/shared/transformer/storage_transformer.go @@ -19,7 +19,7 @@ package transformer import ( "github.com/ethereum/go-ethereum/common" - "github.com/vulcanize/vulcanizedb/libraries/shared/utils" + "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) diff --git a/libraries/shared/watcher/storage_watcher.go b/libraries/shared/watcher/storage_watcher.go index ac554701..64dd31e7 100644 --- a/libraries/shared/watcher/storage_watcher.go +++ b/libraries/shared/watcher/storage_watcher.go @@ -23,8 +23,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/sirupsen/logrus" + "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" - "github.com/vulcanize/vulcanizedb/libraries/shared/utils" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/fs" ) diff --git a/libraries/shared/watcher/storage_watcher_test.go b/libraries/shared/watcher/storage_watcher_test.go index 6376250f..4a073be3 100644 --- a/libraries/shared/watcher/storage_watcher_test.go +++ b/libraries/shared/watcher/storage_watcher_test.go @@ -31,8 +31,8 @@ import ( "github.com/sirupsen/logrus" "github.com/vulcanize/vulcanizedb/libraries/shared/mocks" + "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" - "github.com/vulcanize/vulcanizedb/libraries/shared/utils" "github.com/vulcanize/vulcanizedb/libraries/shared/watcher" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes"