diff --git a/cmd/composeAndExecute.go b/cmd/composeAndExecute.go
index 1d31171b..09172fe5 100644
--- a/cmd/composeAndExecute.go
+++ b/cmd/composeAndExecute.go
@@ -18,6 +18,7 @@ package cmd
import (
"errors"
"fmt"
+ "github.com/vulcanize/vulcanizedb/libraries/shared/constants"
"log"
"os"
"plugin"
@@ -151,18 +152,18 @@ func composeAndExecute() {
// Use WaitGroup to wait on both goroutines
var wg syn.WaitGroup
if len(ethEventInitializers) > 0 {
- w := watcher.NewWatcher(&db, blockChain)
- w.AddTransformers(ethEventInitializers)
+ ew := watcher.NewEventWatcher(&db, blockChain)
+ ew.AddTransformers(ethEventInitializers)
wg.Add(1)
- go watchEthEvents(&w, &wg)
+ go watchEthEvents(&ew, &wg)
}
if len(ethStorageInitializers) > 0 {
tailer := fs.FileTailer{Path: storageDiffsPath}
- w := watcher.NewStorageWatcher(tailer, &db)
- w.AddTransformers(ethStorageInitializers)
+ sw := watcher.NewStorageWatcher(tailer, &db)
+ sw.AddTransformers(ethStorageInitializers)
wg.Add(1)
- go watchEthStorage(&w, &wg)
+ go watchEthStorage(&sw, &wg)
}
wg.Wait()
}
@@ -173,17 +174,23 @@ type Exporter interface {
func init() {
rootCmd.AddCommand(composeAndExecuteCmd)
- composeAndExecuteCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start transformer execution from")
+ composeAndExecuteCmd.Flags().BoolVar(&recheckHeadersArg, "recheckHeaders", false, "checks headers that are already checked for each transformer.")
}
-func watchEthEvents(w *watcher.Watcher, wg *syn.WaitGroup) {
+func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
defer wg.Done()
// Execute over the TransformerInitializer set using the watcher
- fmt.Println("executing transformers")
+ fmt.Println("executing event transformers")
+ var recheck constants.TransformerExecution
+ if recheckHeadersArg {
+ recheck = constants.HeaderRecheck
+ } else {
+ recheck = constants.HeaderMissing
+ }
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
for range ticker.C {
- err := w.Execute()
+ err := w.Execute(recheck)
if err != nil {
// TODO Handle watcher errors in composeAndExecute
}
@@ -193,7 +200,7 @@ func watchEthEvents(w *watcher.Watcher, wg *syn.WaitGroup) {
func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
defer wg.Done()
// Execute over the TransformerInitializer set using the watcher
- fmt.Println("executing transformers")
+ fmt.Println("executing storage transformers")
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
for range ticker.C {
diff --git a/cmd/root.go b/cmd/root.go
index 0cfafd6e..b7b4d993 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -45,6 +45,7 @@ var (
storageDiffsPath string
syncAll bool
endingBlockNumber int64
+ recheckHeadersArg bool
)
const (
diff --git a/libraries/shared/mocks/storage_transformer.go b/libraries/shared/mocks/storage_transformer.go
index f5a778b2..047a9cc3 100644
--- a/libraries/shared/mocks/storage_transformer.go
+++ b/libraries/shared/mocks/storage_transformer.go
@@ -19,8 +19,8 @@ package mocks
import (
"github.com/ethereum/go-ethereum/common"
+ "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
- "github.com/vulcanize/vulcanizedb/libraries/shared/utils"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
diff --git a/libraries/shared/mocks/storage_repository.go b/libraries/shared/repository/storage_repository.go
similarity index 53%
rename from libraries/shared/mocks/storage_repository.go
rename to libraries/shared/repository/storage_repository.go
index 035adf16..379362e3 100644
--- a/libraries/shared/mocks/storage_repository.go
+++ b/libraries/shared/repository/storage_repository.go
@@ -14,29 +14,14 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-package mocks
+package repository
import (
- "github.com/vulcanize/vulcanizedb/libraries/shared/utils"
+ "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
-type MockStorageRepository struct {
- CreateErr error
- PassedBlockNumber int
- PassedBlockHash string
- PassedMetadata utils.StorageValueMetadata
- PassedValue interface{}
-}
-
-func (repository *MockStorageRepository) Create(blockNumber int, blockHash string, metadata utils.StorageValueMetadata, value interface{}) error {
- repository.PassedBlockNumber = blockNumber
- repository.PassedBlockHash = blockHash
- repository.PassedMetadata = metadata
- repository.PassedValue = value
- return repository.CreateErr
-}
-
-func (*MockStorageRepository) SetDB(db *postgres.DB) {
- panic("implement me")
+type StorageRepository interface {
+ Create(blockNumber int, blockHash string, metadata utils.StorageValueMetadata, value interface{}) error
+ SetDB(db *postgres.DB)
}
diff --git a/libraries/shared/storage/mappings.go b/libraries/shared/storage/mappings.go
new file mode 100644
index 00000000..fb52edd9
--- /dev/null
+++ b/libraries/shared/storage/mappings.go
@@ -0,0 +1,61 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package storage
+
+import (
+ "math/big"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/crypto"
+
+ "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+ "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type Mappings interface {
+ Lookup(key common.Hash) (utils.StorageValueMetadata, error)
+ SetDB(db *postgres.DB)
+}
+
+const (
+ IndexZero = "0000000000000000000000000000000000000000000000000000000000000000"
+ IndexOne = "0000000000000000000000000000000000000000000000000000000000000001"
+ IndexTwo = "0000000000000000000000000000000000000000000000000000000000000002"
+ IndexThree = "0000000000000000000000000000000000000000000000000000000000000003"
+ IndexFour = "0000000000000000000000000000000000000000000000000000000000000004"
+ IndexFive = "0000000000000000000000000000000000000000000000000000000000000005"
+ IndexSix = "0000000000000000000000000000000000000000000000000000000000000006"
+ IndexSeven = "0000000000000000000000000000000000000000000000000000000000000007"
+)
+
+func GetMapping(indexOnContract, key string) common.Hash {
+ keyBytes := common.FromHex("0x" + key + indexOnContract)
+ encoded := crypto.Keccak256(keyBytes)
+ return common.BytesToHash(encoded)
+}
+
+func GetNestedMapping(indexOnContract, primaryKey, secondaryKey string) common.Hash {
+ primaryMappingIndex := crypto.Keccak256(common.FromHex(primaryKey + indexOnContract))
+ secondaryMappingIndex := crypto.Keccak256(common.FromHex(secondaryKey), primaryMappingIndex)
+ return common.BytesToHash(secondaryMappingIndex)
+}
+
+func GetIncrementedKey(original common.Hash, incrementBy int64) common.Hash {
+ originalMappingAsInt := original.Big()
+ incremented := big.NewInt(0).Add(originalMappingAsInt, big.NewInt(incrementBy))
+ return common.BytesToHash(incremented.Bytes())
+}
diff --git a/libraries/shared/utils/decoder.go b/libraries/shared/storage/utils/decoder.go
similarity index 100%
rename from libraries/shared/utils/decoder.go
rename to libraries/shared/storage/utils/decoder.go
diff --git a/libraries/shared/utils/decoder_test.go b/libraries/shared/storage/utils/decoder_test.go
similarity index 96%
rename from libraries/shared/utils/decoder_test.go
rename to libraries/shared/storage/utils/decoder_test.go
index 7bd7238e..28721d50 100644
--- a/libraries/shared/utils/decoder_test.go
+++ b/libraries/shared/storage/utils/decoder_test.go
@@ -23,7 +23,7 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
- "github.com/vulcanize/vulcanizedb/libraries/shared/utils"
+ "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
)
var _ = Describe("Storage decoder", func() {
diff --git a/libraries/shared/utils/errors.go b/libraries/shared/storage/utils/errors.go
similarity index 100%
rename from libraries/shared/utils/errors.go
rename to libraries/shared/storage/utils/errors.go
diff --git a/libraries/shared/utils/row.go b/libraries/shared/storage/utils/row.go
similarity index 100%
rename from libraries/shared/utils/row.go
rename to libraries/shared/storage/utils/row.go
diff --git a/libraries/shared/utils/row_test.go b/libraries/shared/storage/utils/row_test.go
similarity index 96%
rename from libraries/shared/utils/row_test.go
rename to libraries/shared/storage/utils/row_test.go
index c1ae1e7d..261085c3 100644
--- a/libraries/shared/utils/row_test.go
+++ b/libraries/shared/storage/utils/row_test.go
@@ -21,7 +21,7 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
- "github.com/vulcanize/vulcanizedb/libraries/shared/utils"
+ "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
)
var _ = Describe("Storage row parsing", func() {
diff --git a/libraries/shared/utils/shared_suite_test.go b/libraries/shared/storage/utils/shared_suite_test.go
similarity index 100%
rename from libraries/shared/utils/shared_suite_test.go
rename to libraries/shared/storage/utils/shared_suite_test.go
diff --git a/libraries/shared/utils/value.go b/libraries/shared/storage/utils/value.go
similarity index 100%
rename from libraries/shared/utils/value.go
rename to libraries/shared/storage/utils/value.go
diff --git a/libraries/shared/transformer/storage_transformer.go b/libraries/shared/transformer/storage_transformer.go
index 19203314..22ca9378 100644
--- a/libraries/shared/transformer/storage_transformer.go
+++ b/libraries/shared/transformer/storage_transformer.go
@@ -19,7 +19,7 @@ package transformer
import (
"github.com/ethereum/go-ethereum/common"
- "github.com/vulcanize/vulcanizedb/libraries/shared/utils"
+ "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
diff --git a/libraries/shared/watcher/storage_watcher.go b/libraries/shared/watcher/storage_watcher.go
index ac554701..64dd31e7 100644
--- a/libraries/shared/watcher/storage_watcher.go
+++ b/libraries/shared/watcher/storage_watcher.go
@@ -23,8 +23,8 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/sirupsen/logrus"
+ "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
- "github.com/vulcanize/vulcanizedb/libraries/shared/utils"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/fs"
)
diff --git a/libraries/shared/watcher/storage_watcher_test.go b/libraries/shared/watcher/storage_watcher_test.go
index 6376250f..4a073be3 100644
--- a/libraries/shared/watcher/storage_watcher_test.go
+++ b/libraries/shared/watcher/storage_watcher_test.go
@@ -31,8 +31,8 @@ import (
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
+ "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
- "github.com/vulcanize/vulcanizedb/libraries/shared/utils"
"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"