move storage mappings pkg into shared

cmd/composeAndExecute.go

@@ -18,6 +18,7 @@ package cmd
 import (
 	"errors"
 	"fmt"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
 	"log"
 	"os"
 	"plugin"
@@ -151,18 +152,18 @@ func composeAndExecute() {
 	// Use WaitGroup to wait on both goroutines
 	var wg syn.WaitGroup
 	if len(ethEventInitializers) > 0 {
-		w := watcher.NewWatcher(&db, blockChain)
+		ew := watcher.NewEventWatcher(&db, blockChain)
-		w.AddTransformers(ethEventInitializers)
+		ew.AddTransformers(ethEventInitializers)
 		wg.Add(1)
-		go watchEthEvents(&w, &wg)
+		go watchEthEvents(&ew, &wg)
 	}
 
 	if len(ethStorageInitializers) > 0 {
 		tailer := fs.FileTailer{Path: storageDiffsPath}
-		w := watcher.NewStorageWatcher(tailer, &db)
+		sw := watcher.NewStorageWatcher(tailer, &db)
-		w.AddTransformers(ethStorageInitializers)
+		sw.AddTransformers(ethStorageInitializers)
 		wg.Add(1)
-		go watchEthStorage(&w, &wg)
+		go watchEthStorage(&sw, &wg)
 	}
 	wg.Wait()
 }
@@ -173,17 +174,23 @@ type Exporter interface {
 
 func init() {
 	rootCmd.AddCommand(composeAndExecuteCmd)
-	composeAndExecuteCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start transformer execution from")
+	composeAndExecuteCmd.Flags().BoolVar(&recheckHeadersArg, "recheckHeaders", false, "checks headers that are already checked for each transformer.")
 }
 
-func watchEthEvents(w *watcher.Watcher, wg *syn.WaitGroup) {
+func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
 	defer wg.Done()
 	// Execute over the TransformerInitializer set using the watcher
-	fmt.Println("executing transformers")
+	fmt.Println("executing event transformers")
+	var recheck constants.TransformerExecution
+	if recheckHeadersArg {
+		recheck = constants.HeaderRecheck
+	} else {
+		recheck = constants.HeaderMissing
+	}
 	ticker := time.NewTicker(pollingInterval)
 	defer ticker.Stop()
 	for range ticker.C {
-		err := w.Execute()
+		err := w.Execute(recheck)
 		if err != nil {
 			// TODO Handle watcher errors in composeAndExecute
 		}
@@ -193,7 +200,7 @@ func watchEthEvents(w *watcher.Watcher, wg *syn.WaitGroup) {
 func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
 	defer wg.Done()
 	// Execute over the TransformerInitializer set using the watcher
-	fmt.Println("executing transformers")
+	fmt.Println("executing storage transformers")
 	ticker := time.NewTicker(pollingInterval)
 	defer ticker.Stop()
 	for range ticker.C {

cmd/root.go

@@ -45,6 +45,7 @@ var (
 	storageDiffsPath    string
 	syncAll             bool
 	endingBlockNumber   int64
+	recheckHeadersArg   bool
 )
 
 const (

libraries/shared/mocks/storage_transformer.go

@@ -19,8 +19,8 @@ package mocks
 import (
 	"github.com/ethereum/go-ethereum/common"
 
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 

libraries/shared/repository/storage_repository.go

@@ -14,29 +14,14 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package mocks
+package repository
 
 import (
-	"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
-type MockStorageRepository struct {
+type StorageRepository interface {
-	CreateErr         error
+	Create(blockNumber int, blockHash string, metadata utils.StorageValueMetadata, value interface{}) error
-	PassedBlockNumber int
+	SetDB(db *postgres.DB)
-	PassedBlockHash   string
-	PassedMetadata    utils.StorageValueMetadata
-	PassedValue       interface{}
-}
-
-func (repository *MockStorageRepository) Create(blockNumber int, blockHash string, metadata utils.StorageValueMetadata, value interface{}) error {
-	repository.PassedBlockNumber = blockNumber
-	repository.PassedBlockHash = blockHash
-	repository.PassedMetadata = metadata
-	repository.PassedValue = value
-	return repository.CreateErr
-}
-
-func (*MockStorageRepository) SetDB(db *postgres.DB) {
-	panic("implement me")
 }

libraries/shared/storage/mappings.go

@@ -0,0 +1,61 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type Mappings interface {
+	Lookup(key common.Hash) (utils.StorageValueMetadata, error)
+	SetDB(db *postgres.DB)
+}
+
+const (
+	IndexZero  = "0000000000000000000000000000000000000000000000000000000000000000"
+	IndexOne   = "0000000000000000000000000000000000000000000000000000000000000001"
+	IndexTwo   = "0000000000000000000000000000000000000000000000000000000000000002"
+	IndexThree = "0000000000000000000000000000000000000000000000000000000000000003"
+	IndexFour  = "0000000000000000000000000000000000000000000000000000000000000004"
+	IndexFive  = "0000000000000000000000000000000000000000000000000000000000000005"
+	IndexSix   = "0000000000000000000000000000000000000000000000000000000000000006"
+	IndexSeven = "0000000000000000000000000000000000000000000000000000000000000007"
+)
+
+func GetMapping(indexOnContract, key string) common.Hash {
+	keyBytes := common.FromHex("0x" + key + indexOnContract)
+	encoded := crypto.Keccak256(keyBytes)
+	return common.BytesToHash(encoded)
+}
+
+func GetNestedMapping(indexOnContract, primaryKey, secondaryKey string) common.Hash {
+	primaryMappingIndex := crypto.Keccak256(common.FromHex(primaryKey + indexOnContract))
+	secondaryMappingIndex := crypto.Keccak256(common.FromHex(secondaryKey), primaryMappingIndex)
+	return common.BytesToHash(secondaryMappingIndex)
+}
+
+func GetIncrementedKey(original common.Hash, incrementBy int64) common.Hash {
+	originalMappingAsInt := original.Big()
+	incremented := big.NewInt(0).Add(originalMappingAsInt, big.NewInt(incrementBy))
+	return common.BytesToHash(incremented.Bytes())
+}

libraries/shared/storage/utils/decoder_test.go

@@ -23,7 +23,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 )
 
 var _ = Describe("Storage decoder", func() {

libraries/shared/storage/utils/row_test.go

@@ -21,7 +21,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 )
 
 var _ = Describe("Storage row parsing", func() {

libraries/shared/transformer/storage_transformer.go

@@ -19,7 +19,7 @@ package transformer
 import (
 	"github.com/ethereum/go-ethereum/common"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 

libraries/shared/watcher/storage_watcher.go

@@ -23,8 +23,8 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/sirupsen/logrus"
 
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/fs"
 )

libraries/shared/watcher/storage_watcher_test.go

@@ -31,8 +31,8 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"