remove pkg/transformers; extract shared files needed to

libraries/shared; work on automated db migration management

libraries/shared/chunker/log_chunker.go

@@ -0,0 +1,72 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package chunker
+
+import (
+	"strings"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+
+	shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+)
+
+type Chunker interface {
+	AddConfigs(transformerConfigs []shared_t.TransformerConfig)
+	ChunkLogs(logs []types.Log) map[string][]types.Log
+}
+
+type LogChunker struct {
+	AddressToNames map[string][]string
+	NameToTopic0   map[string]common.Hash
+}
+
+// Returns a new log chunker with initialised maps.
+// Needs to have configs added with `AddConfigs` to consider logs for the respective transformer.
+func NewLogChunker() *LogChunker {
+	return &LogChunker{
+		AddressToNames: map[string][]string{},
+		NameToTopic0:   map[string]common.Hash{},
+	}
+}
+
+// Configures the chunker by adding more addreses and topics to consider.
+func (chunker *LogChunker) AddConfigs(transformerConfigs []shared_t.TransformerConfig) {
+	for _, config := range transformerConfigs {
+		for _, address := range config.ContractAddresses {
+			var lowerCaseAddress = strings.ToLower(address)
+			chunker.AddressToNames[lowerCaseAddress] = append(chunker.AddressToNames[lowerCaseAddress], config.TransformerName)
+			chunker.NameToTopic0[config.TransformerName] = common.HexToHash(config.Topic)
+		}
+	}
+}
+
+// Goes through an array of logs, associating relevant logs (matching addresses and topic) with transformers
+func (chunker *LogChunker) ChunkLogs(logs []types.Log) map[string][]types.Log {
+	chunks := map[string][]types.Log{}
+	for _, log := range logs {
+		// Topic0 is not unique to each transformer, also need to consider the contract address
+		relevantTransformers := chunker.AddressToNames[strings.ToLower(log.Address.String())]
+
+		for _, transformer := range relevantTransformers {
+			if chunker.NameToTopic0[transformer] == log.Topics[0] {
+				chunks[transformer] = append(chunks[transformer], log)
+			}
+		}
+	}
+	return chunks
+}

libraries/shared/chunker/log_chunker_test.go

@@ -0,0 +1,152 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package chunker_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	chunk "github.com/vulcanize/vulcanizedb/libraries/shared/chunker"
+	shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+)
+
+var _ = Describe("Log chunker", func() {
+	var (
+		configs []shared_t.TransformerConfig
+		chunker *chunk.LogChunker
+	)
+
+	BeforeEach(func() {
+		configA := shared_t.TransformerConfig{
+			TransformerName:   "TransformerA",
+			ContractAddresses: []string{"0x00000000000000000000000000000000000000A1", "0x00000000000000000000000000000000000000A2"},
+			Topic:             "0xA",
+		}
+		configB := shared_t.TransformerConfig{
+			TransformerName:   "TransformerB",
+			ContractAddresses: []string{"0x00000000000000000000000000000000000000B1"},
+			Topic:             "0xB",
+		}
+
+		configC := shared_t.TransformerConfig{
+			TransformerName:   "TransformerC",
+			ContractAddresses: []string{"0x00000000000000000000000000000000000000A2"},
+			Topic:             "0xC",
+		}
+
+		configs = []shared_t.TransformerConfig{configA, configB, configC}
+		chunker = chunk.NewLogChunker()
+		chunker.AddConfigs(configs)
+	})
+
+	Describe("initialisation", func() {
+		It("creates lookup maps correctly", func() {
+			Expect(chunker.AddressToNames).To(Equal(map[string][]string{
+				"0x00000000000000000000000000000000000000a1": []string{"TransformerA"},
+				"0x00000000000000000000000000000000000000a2": []string{"TransformerA", "TransformerC"},
+				"0x00000000000000000000000000000000000000b1": []string{"TransformerB"},
+			}))
+
+			Expect(chunker.NameToTopic0).To(Equal(map[string]common.Hash{
+				"TransformerA": common.HexToHash("0xA"),
+				"TransformerB": common.HexToHash("0xB"),
+				"TransformerC": common.HexToHash("0xC"),
+			}))
+		})
+	})
+
+	Describe("AddConfigs", func() {
+		It("can add more configs later", func() {
+			configD := shared_t.TransformerConfig{
+				TransformerName:   "TransformerD",
+				ContractAddresses: []string{"0x000000000000000000000000000000000000000D"},
+				Topic:             "0xD",
+			}
+			chunker.AddConfigs([]shared_t.TransformerConfig{configD})
+
+			Expect(chunker.AddressToNames).To(ContainElement([]string{"TransformerD"}))
+			Expect(chunker.NameToTopic0).To(ContainElement(common.HexToHash("0xD")))
+		})
+
+		It("lower cases address", func() {
+			configD := shared_t.TransformerConfig{
+				TransformerName:   "TransformerD",
+				ContractAddresses: []string{"0x000000000000000000000000000000000000000D"},
+				Topic:             "0xD",
+			}
+			chunker.AddConfigs([]shared_t.TransformerConfig{configD})
+
+			Expect(chunker.AddressToNames["0x000000000000000000000000000000000000000d"]).To(Equal([]string{"TransformerD"}))
+		})
+	})
+
+	Describe("ChunkLogs", func() {
+		It("only associates logs with relevant topic0 and address to transformers", func() {
+			logs := []types.Log{log1, log2, log3, log4, log5}
+			chunks := chunker.ChunkLogs(logs)
+
+			Expect(chunks["TransformerA"]).To(And(ContainElement(log1), ContainElement(log4)))
+			Expect(chunks["TransformerB"]).To(BeEmpty())
+			Expect(chunks["TransformerC"]).To(ContainElement(log5))
+		})
+	})
+})
+
+var (
+	// Match TransformerA
+	log1 = types.Log{
+		Address: common.HexToAddress("0xA1"),
+		Topics: []common.Hash{
+			common.HexToHash("0xA"),
+			common.HexToHash("0xLogTopic1"),
+		},
+	}
+	// Match TransformerA address, but not topic0
+	log2 = types.Log{
+		Address: common.HexToAddress("0xA1"),
+		Topics: []common.Hash{
+			common.HexToHash("0xB"),
+			common.HexToHash("0xLogTopic2"),
+		},
+	}
+	// Match TransformerA topic, but TransformerB address
+	log3 = types.Log{
+		Address: common.HexToAddress("0xB1"),
+		Topics: []common.Hash{
+			common.HexToHash("0xA"),
+			common.HexToHash("0xLogTopic3"),
+		},
+	}
+	// Match TransformerA, with the other address
+	log4 = types.Log{
+		Address: common.HexToAddress("0xA2"),
+		Topics: []common.Hash{
+			common.HexToHash("0xA"),
+			common.HexToHash("0xLogTopic4"),
+		},
+	}
+	// Match TransformerC, which shares address with TransformerA
+	log5 = types.Log{
+		Address: common.HexToAddress("0xA2"),
+		Topics: []common.Hash{
+			common.HexToHash("0xC"),
+			common.HexToHash("0xLogTopic5"),
+		},
+	}
+)

libraries/shared/constants/checked_headers.go

@@ -0,0 +1,25 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package constants
+
+type TransformerExecution bool
+
+const (
+	HeaderRecheck    TransformerExecution = true
+	HeaderMissing    TransformerExecution = false
+	RecheckHeaderCap                      = "4"
+)

libraries/shared/constants/data.go

@@ -14,4 +14,9 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package transformer_test
+package constants
+
+var DataItemLength = 32
+
+// TODO Grab this from DB, since it can change through governance
+var TTL = int64(10800) // 60 * 60 * 3 == 10800 seconds == 3 hours

libraries/shared/fetcher/log_fetcher.go

@@ -0,0 +1,58 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package fetcher
+
+import (
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+)
+
+type LogFetcher interface {
+	FetchLogs(contractAddresses []common.Address, topics []common.Hash, missingHeader core.Header) ([]types.Log, error)
+}
+
+type Fetcher struct {
+	blockChain core.BlockChain
+}
+
+func NewFetcher(blockchain core.BlockChain) *Fetcher {
+	return &Fetcher{
+		blockChain: blockchain,
+	}
+}
+
+// Checks all topic0s, on all addresses, fetching matching logs for the given header
+func (fetcher Fetcher) FetchLogs(addresses []common.Address, topic0s []common.Hash, header core.Header) ([]types.Log, error) {
+	blockHash := common.HexToHash(header.Hash)
+	query := ethereum.FilterQuery{
+		BlockHash: &blockHash,
+		Addresses: addresses,
+		// Search for _any_ of the topics in topic0 position; see docs on `FilterQuery`
+		Topics: [][]common.Hash{topic0s},
+	}
+
+	logs, err := fetcher.blockChain.GetEthLogsWithCustomQuery(query)
+	if err != nil {
+		// TODO review aggregate fetching error handling
+		return []types.Log{}, err
+	}
+
+	return logs, nil
+}

libraries/shared/fetcher/log_fetcher_test.go

@@ -0,0 +1,70 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package fetcher_test
+
+import (
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	fetch "github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+)
+
+var _ = Describe("Fetcher", func() {
+	Describe("FetchLogs", func() {
+		It("fetches logs based on the given query", func() {
+			blockChain := fakes.NewMockBlockChain()
+			fetcher := fetch.NewFetcher(blockChain)
+			header := fakes.FakeHeader
+
+			addresses := []common.Address{
+				common.HexToAddress("0xfakeAddress"),
+				common.HexToAddress("0xanotherFakeAddress"),
+			}
+
+			topicZeros := []common.Hash{common.BytesToHash([]byte{1, 2, 3, 4, 5})}
+
+			_, err := fetcher.FetchLogs(addresses, topicZeros, header)
+
+			address1 := common.HexToAddress("0xfakeAddress")
+			address2 := common.HexToAddress("0xanotherFakeAddress")
+			Expect(err).NotTo(HaveOccurred())
+
+			blockHash := common.HexToHash(header.Hash)
+			expectedQuery := ethereum.FilterQuery{
+				BlockHash: &blockHash,
+				Addresses: []common.Address{address1, address2},
+				Topics:    [][]common.Hash{topicZeros},
+			}
+			blockChain.AssertGetEthLogsWithCustomQueryCalledWith(expectedQuery)
+		})
+
+		It("returns an error if fetching the logs fails", func() {
+			blockChain := fakes.NewMockBlockChain()
+			blockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError)
+			fetcher := fetch.NewFetcher(blockChain)
+
+			_, err := fetcher.FetchLogs([]common.Address{}, []common.Hash{}, core.Header{})
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+	})
+})

libraries/shared/mocks/mock_watcher_repository.go

@@ -0,0 +1,69 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type MockWatcherRepository struct {
+	ReturnCheckedColumnNames    []string
+	GetCheckedColumnNamesError  error
+	GetCheckedColumnNamesCalled bool
+
+	ReturnNotCheckedSQL       string
+	CreateNotCheckedSQLCalled bool
+
+	ReturnMissingHeaders []core.Header
+	MissingHeadersError  error
+	MissingHeadersCalled bool
+}
+
+func (repository *MockWatcherRepository) GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
+	repository.GetCheckedColumnNamesCalled = true
+	if repository.GetCheckedColumnNamesError != nil {
+		return []string{}, repository.GetCheckedColumnNamesError
+	}
+
+	return repository.ReturnCheckedColumnNames, nil
+}
+
+func (repository *MockWatcherRepository) SetCheckedColumnNames(checkedColumnNames []string) {
+	repository.ReturnCheckedColumnNames = checkedColumnNames
+}
+
+func (repository *MockWatcherRepository) CreateNotCheckedSQL(boolColumns []string) string {
+	repository.CreateNotCheckedSQLCalled = true
+	return repository.ReturnNotCheckedSQL
+}
+
+func (repository *MockWatcherRepository) SetNotCheckedSQL(notCheckedSql string) {
+	repository.ReturnNotCheckedSQL = notCheckedSql
+}
+
+func (repository *MockWatcherRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) {
+	if repository.MissingHeadersError != nil {
+		return []core.Header{}, repository.MissingHeadersError
+	}
+	repository.MissingHeadersCalled = true
+	return repository.ReturnMissingHeaders, nil
+}
+
+func (repository *MockWatcherRepository) SetMissingHeaders(headers []core.Header) {
+	repository.ReturnMissingHeaders = headers
+}

libraries/shared/mocks/storage_repository.go

@@ -0,0 +1,42 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type MockStorageRepository struct {
+	CreateErr         error
+	PassedBlockNumber int
+	PassedBlockHash   string
+	PassedMetadata    utils.StorageValueMetadata
+	PassedValue       interface{}
+}
+
+func (repository *MockStorageRepository) Create(blockNumber int, blockHash string, metadata utils.StorageValueMetadata, value interface{}) error {
+	repository.PassedBlockNumber = blockNumber
+	repository.PassedBlockHash = blockHash
+	repository.PassedMetadata = metadata
+	repository.PassedValue = value
+	return repository.CreateErr
+}
+
+func (*MockStorageRepository) SetDB(db *postgres.DB) {
+	panic("implement me")
+}

libraries/shared/mocks/storage_transformer.go

@@ -0,0 +1,44 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type MockStorageTransformer struct {
+	Address    common.Address
+	ExecuteErr error
+	PassedRow  utils.StorageDiffRow
+}
+
+func (transformer *MockStorageTransformer) Execute(row utils.StorageDiffRow) error {
+	transformer.PassedRow = row
+	return transformer.ExecuteErr
+}
+
+func (transformer *MockStorageTransformer) ContractAddress() common.Address {
+	return transformer.Address
+}
+
+func (transformer *MockStorageTransformer) FakeTransformerInitializer(db *postgres.DB) transformer.StorageTransformer {
+	return transformer
+}

libraries/shared/mocks/transformer.go

@@ -0,0 +1,62 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/ethereum/go-ethereum/core/types"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
+	shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type MockTransformer struct {
+	ExecuteWasCalled bool
+	ExecuteError     error
+	PassedLogs       []types.Log
+	PassedHeader     core.Header
+	config           shared_t.TransformerConfig
+}
+
+func (mh *MockTransformer) Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error {
+	if mh.ExecuteError != nil {
+		return mh.ExecuteError
+	}
+	mh.ExecuteWasCalled = true
+	mh.PassedLogs = logs
+	mh.PassedHeader = header
+	return nil
+}
+
+func (mh *MockTransformer) GetConfig() shared_t.TransformerConfig {
+	return mh.config
+}
+
+func (mh *MockTransformer) SetTransformerConfig(config shared_t.TransformerConfig) {
+	mh.config = config
+}
+
+func (mh *MockTransformer) FakeTransformerInitializer(db *postgres.DB) shared_t.EventTransformer {
+	return mh
+}
+
+var FakeTransformerConfig = shared_t.TransformerConfig{
+	TransformerName:   "FakeTransformer",
+	ContractAddresses: []string{"FakeAddress"},
+	Topic:             "FakeTopic",
+}

libraries/shared/repository/repository.go

@@ -0,0 +1,192 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package repository
+
+import (
+	"bytes"
+	"database/sql"
+	"database/sql/driver"
+	"fmt"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+func GetOrCreateIlk(ilk string, db *postgres.DB) (int, error) {
+	var ilkID int
+	err := db.Get(&ilkID, `SELECT id FROM maker.ilks WHERE ilk = $1`, ilk)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			insertErr := db.QueryRow(`INSERT INTO maker.ilks (ilk) VALUES ($1) RETURNING id`, ilk).Scan(&ilkID)
+			return ilkID, insertErr
+		}
+	}
+	return ilkID, err
+}
+
+func GetOrCreateIlkInTransaction(ilk string, tx *sql.Tx) (int, error) {
+	var ilkID int
+	err := tx.QueryRow(`SELECT id FROM maker.ilks WHERE ilk = $1`, ilk).Scan(&ilkID)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			insertErr := tx.QueryRow(`INSERT INTO maker.ilks (ilk) VALUES ($1) RETURNING id`, ilk).Scan(&ilkID)
+			return ilkID, insertErr
+		}
+	}
+	return ilkID, err
+}
+
+func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn string) error {
+	_, err := db.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
+		VALUES ($1, $2)
+		ON CONFLICT (header_id) DO
+			UPDATE SET `+checkedHeadersColumn+` = checked_headers.`+checkedHeadersColumn+` + 1`, headerID, 1)
+	return err
+}
+
+func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersColumn string) error {
+	_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
+		VALUES ($1, $2)
+		ON CONFLICT (header_id) DO
+			UPDATE SET `+checkedHeadersColumn+` = checked_headers.`+checkedHeadersColumn+` + 1`, headerID, 1)
+	return err
+}
+
+// Treats a header as missing if it's not in the headers table, or not checked for some log type
+func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) {
+	var result []core.Header
+	var query string
+	var err error
+
+	if endingBlockNumber == -1 {
+		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
+				LEFT JOIN checked_headers on headers.id = header_id
+				WHERE (header_id ISNULL OR ` + notCheckedSQL + `)
+				AND headers.block_number >= $1
+				AND headers.eth_node_fingerprint = $2`
+		err = db.Select(&result, query, startingBlockNumber, db.Node.ID)
+	} else {
+		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
+				LEFT JOIN checked_headers on headers.id = header_id
+				WHERE (header_id ISNULL OR ` + notCheckedSQL + `)
+				AND headers.block_number >= $1
+				AND headers.block_number <= $2
+				AND headers.eth_node_fingerprint = $3`
+		err = db.Select(&result, query, startingBlockNumber, endingBlockNumber, db.Node.ID)
+	}
+
+	return result, err
+}
+
+func RecheckHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, checkedHeadersColumn string) ([]core.Header, error) {
+	var result []core.Header
+	var query string
+	var err error
+
+	if endingBlockNumber == -1 {
+		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
+				LEFT JOIN checked_headers on headers.id = header_id
+				WHERE ` + checkedHeadersColumn + ` between 1 and ` + constants.RecheckHeaderCap + `
+				AND headers.block_number >= $1
+				AND headers.eth_node_fingerprint = $2`
+		err = db.Select(&result, query, startingBlockNumber, db.Node.ID)
+	} else {
+		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
+				LEFT JOIN checked_headers on headers.id = header_id
+				WHERE ` + checkedHeadersColumn + ` between 1 and ` + constants.RecheckHeaderCap + `
+				AND headers.block_number >= $1
+				AND headers.block_number <= $2
+				AND headers.eth_node_fingerprint = $3`
+		err = db.Select(&result, query, startingBlockNumber, endingBlockNumber, db.Node.ID)
+	}
+
+	return result, err
+}
+
+func GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
+	// Query returns `[]driver.Value`, nullable polymorphic interface
+	var queryResult []driver.Value
+	columnNamesQuery :=
+		`SELECT column_name FROM information_schema.columns
+		WHERE table_schema = 'public'
+			AND table_name = 'checked_headers'
+			AND column_name ~ '_checked';`
+
+	err := db.Select(&queryResult, columnNamesQuery)
+	if err != nil {
+		return []string{}, err
+	}
+
+	// Transform column names from `driver.Value` to strings
+	var columnNames []string
+	for _, result := range queryResult {
+		if columnName, ok := result.(string); ok {
+			columnNames = append(columnNames, columnName)
+		} else {
+			return []string{}, fmt.Errorf("incorrect value for checked_headers column name")
+		}
+	}
+
+	return columnNames, nil
+}
+
+// Builds a SQL string that checks if any column value is 0, given the column names.
+// Defaults to FALSE when no columns are provided.
+// Ex: ["columnA", "columnB"] => "NOT (columnA!=0 AND columnB!=0)"
+//     [] => "FALSE"
+func CreateNotCheckedSQL(boolColumns []string, recheckHeaders constants.TransformerExecution) string {
+
+	var result bytes.Buffer
+
+	if len(boolColumns) == 0 {
+		return "FALSE"
+	}
+
+	result.WriteString("NOT (")
+
+	// Loop excluding last column name
+	for _, column := range boolColumns[:len(boolColumns)-1] {
+
+		if recheckHeaders {
+			result.WriteString(fmt.Sprintf("%v>=%s AND ", column, constants.RecheckHeaderCap))
+		} else {
+			result.WriteString(fmt.Sprintf("%v!=0 AND ", column))
+		}
+	}
+
+	// No trailing "OR" for the last column name
+	if recheckHeaders {
+		result.WriteString(fmt.Sprintf("%v>=%s)", boolColumns[len(boolColumns)-1], constants.RecheckHeaderCap))
+	} else {
+		result.WriteString(fmt.Sprintf("%v!=0)", boolColumns[len(boolColumns)-1]))
+
+	}
+
+	return result.String()
+}
+
+func GetTicInTx(headerID int64, tx *sql.Tx) (int64, error) {
+	var blockTimestamp int64
+	err := tx.QueryRow(`SELECT block_timestamp FROM public.headers WHERE id = $1;`, headerID).Scan(&blockTimestamp)
+	if err != nil {
+		return 0, err
+	}
+
+	tic := blockTimestamp + constants.TTL
+	return tic, nil
+}

libraries/shared/repository/repository_utility_test.go

@@ -0,0 +1,183 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package repository_test
+
+import (
+	"fmt"
+	"math/rand"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
+	shared "github.com/vulcanize/vulcanizedb/libraries/shared/repository"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Repository utilities", func() {
+	Describe("MissingHeaders", func() {
+		var (
+			db                       *postgres.DB
+			headerRepository         datastore.HeaderRepository
+			startingBlockNumber      int64
+			endingBlockNumber        int64
+			eventSpecificBlockNumber int64
+			blockNumbers             []int64
+			headerIDs                []int64
+			notCheckedSQL            string
+			err                      error
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(test_config.NewTestNode())
+			test_config.CleanTestDB(db)
+			headerRepository = repositories.NewHeaderRepository(db)
+
+			columnNames, err := shared.GetCheckedColumnNames(db)
+			Expect(err).NotTo(HaveOccurred())
+			notCheckedSQL = shared.CreateNotCheckedSQL(columnNames, constants.HeaderMissing)
+
+			startingBlockNumber = rand.Int63()
+			eventSpecificBlockNumber = startingBlockNumber + 1
+			endingBlockNumber = startingBlockNumber + 2
+			outOfRangeBlockNumber := endingBlockNumber + 1
+
+			blockNumbers = []int64{startingBlockNumber, eventSpecificBlockNumber, endingBlockNumber, outOfRangeBlockNumber}
+
+			headerIDs = []int64{}
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+		})
+
+		It("only treats headers as checked if the event specific logs have been checked", func() {
+			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(3))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
+		})
+
+		It("only returns headers associated with the current node", func() {
+			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
+			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
+			for _, n := range blockNumbers {
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10))
+				Expect(err).NotTo(HaveOccurred())
+			}
+
+			Expect(err).NotTo(HaveOccurred())
+			nodeOneMissingHeaders, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(nodeOneMissingHeaders)).To(Equal(3))
+			Expect(nodeOneMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
+			Expect(nodeOneMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
+			Expect(nodeOneMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
+
+			nodeTwoMissingHeaders, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber+10, dbTwo, notCheckedSQL)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(nodeTwoMissingHeaders)).To(Equal(3))
+			Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
+			Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
+		})
+	})
+
+	Describe("GetCheckedColumnNames", func() {
+		It("gets the column names from checked_headers", func() {
+			db := test_config.NewTestDB(test_config.NewTestNode())
+			test_config.CleanTestDB(db)
+			expectedColumnNames := getExpectedColumnNames()
+			actualColumnNames, err := shared.GetCheckedColumnNames(db)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(actualColumnNames).To(Equal(expectedColumnNames))
+		})
+	})
+
+	Describe("CreateNotCheckedSQL", func() {
+		It("generates a correct SQL string for one column", func() {
+			columns := []string{"columnA"}
+			expected := "NOT (columnA!=0)"
+			actual := shared.CreateNotCheckedSQL(columns, constants.HeaderMissing)
+			Expect(actual).To(Equal(expected))
+		})
+
+		It("generates a correct SQL string for several columns", func() {
+			columns := []string{"columnA", "columnB"}
+			expected := "NOT (columnA!=0 AND columnB!=0)"
+			actual := shared.CreateNotCheckedSQL(columns, constants.HeaderMissing)
+			Expect(actual).To(Equal(expected))
+		})
+
+		It("defaults to FALSE when there are no columns", func() {
+			expected := "FALSE"
+			actual := shared.CreateNotCheckedSQL([]string{}, constants.HeaderMissing)
+			Expect(actual).To(Equal(expected))
+		})
+
+		It("generates a correct SQL string for rechecking headers", func() {
+			columns := []string{"columnA", "columnB"}
+			expected := fmt.Sprintf("NOT (columnA>=%s AND columnB>=%s)", constants.RecheckHeaderCap, constants.RecheckHeaderCap)
+			actual := shared.CreateNotCheckedSQL(columns, constants.HeaderRecheck)
+			Expect(actual).To(Equal(expected))
+		})
+	})
+})
+
+func getExpectedColumnNames() []string {
+	return []string{
+		"price_feeds_checked",
+		"flip_kick_checked",
+		"frob_checked",
+		"tend_checked",
+		"bite_checked",
+		"dent_checked",
+		"pit_file_debt_ceiling_checked",
+		"pit_file_ilk_checked",
+		"vat_init_checked",
+		"drip_file_ilk_checked",
+		"drip_file_repo_checked",
+		"drip_file_vow_checked",
+		"deal_checked",
+		"drip_drip_checked",
+		"cat_file_chop_lump_checked",
+		"cat_file_flip_checked",
+		"cat_file_pit_vow_checked",
+		"flop_kick_checked",
+		"vat_move_checked",
+		"vat_fold_checked",
+		"vat_heal_checked",
+		"vat_toll_checked",
+		"vat_tune_checked",
+		"vat_grab_checked",
+		"vat_flux_checked",
+		"vat_slip_checked",
+		"vow_flog_checked",
+		"flap_kick_checked",
+	}
+}

libraries/shared/transformer/event_transformer.go

@@ -1,5 +1,18 @@
-// Auto-gen this code for different transformer interfaces/configs
-// based on config file to allow for more modularity
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package transformer
 
@@ -8,16 +21,16 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 
+	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
 )
 
-type Transformer interface {
+type EventTransformer interface {
 	Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error
 	GetConfig() TransformerConfig
 }
 
-type TransformerInitializer func(db *postgres.DB) Transformer
+type TransformerInitializer func(db *postgres.DB) EventTransformer
 
 type TransformerConfig struct {
 	TransformerName     string

libraries/shared/transformer/storage_transformer.go

@@ -0,0 +1,31 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package transformer
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type StorageTransformer interface {
+	Execute(row utils.StorageDiffRow) error
+	ContractAddress() common.Address
+}
+
+type StorageTransformerInitializer func(db *postgres.DB) StorageTransformer

libraries/shared/utils/decoder.go

@@ -0,0 +1,46 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package utils
+
+import (
+	"fmt"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, error) {
+	switch metadata.Type {
+	case Uint256:
+		return decodeUint256(row.StorageValue.Bytes()), nil
+	case Address:
+		return decodeAddress(row.StorageValue.Bytes()), nil
+	case Bytes32:
+		return row.StorageValue.Hex(), nil
+	default:
+		panic(fmt.Sprintf("can't decode unknown type: %d", metadata.Type))
+	}
+}
+
+func decodeUint256(raw []byte) string {
+	n := big.NewInt(0).SetBytes(raw)
+	return n.String()
+}
+
+func decodeAddress(raw []byte) string {
+	return common.BytesToAddress(raw).Hex()
+}

libraries/shared/utils/decoder_test.go

@@ -0,0 +1,51 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package utils_test
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
+)
+
+var _ = Describe("Storage decoder", func() {
+	It("decodes uint256", func() {
+		fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000539")
+		row := utils.StorageDiffRow{StorageValue: fakeInt}
+		metadata := utils.StorageValueMetadata{Type: utils.Uint256}
+
+		result, err := utils.Decode(row, metadata)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
+	})
+
+	It("decodes address", func() {
+		fakeAddress := common.HexToAddress("0x12345")
+		row := utils.StorageDiffRow{StorageValue: fakeAddress.Hash()}
+		metadata := utils.StorageValueMetadata{Type: utils.Address}
+
+		result, err := utils.Decode(row, metadata)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result).To(Equal(fakeAddress.Hex()))
+	})
+})

libraries/shared/utils/errors.go

@@ -0,0 +1,53 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package utils
+
+import (
+	"fmt"
+)
+
+type ErrContractNotFound struct {
+	Contract string
+}
+
+func (e ErrContractNotFound) Error() string {
+	return fmt.Sprintf("transformer not found for contract: %s", e.Contract)
+}
+
+type ErrMetadataMalformed struct {
+	MissingData Key
+}
+
+func (e ErrMetadataMalformed) Error() string {
+	return fmt.Sprintf("storage metadata malformed: missing %s", e.MissingData)
+}
+
+type ErrRowMalformed struct {
+	Length int
+}
+
+func (e ErrRowMalformed) Error() string {
+	return fmt.Sprintf("storage row malformed: length %d, expected %d", e.Length, ExpectedRowLength)
+}
+
+type ErrStorageKeyNotFound struct {
+	Key string
+}
+
+func (e ErrStorageKeyNotFound) Error() string {
+	return fmt.Sprintf("unknown storage key: %s", e.Key)
+}

libraries/shared/utils/row.go

@@ -0,0 +1,50 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package utils
+
+import (
+	"strconv"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+const ExpectedRowLength = 5
+
+type StorageDiffRow struct {
+	Contract     common.Address
+	BlockHash    common.Hash `db:"block_hash"`
+	BlockHeight  int         `db:"block_height"`
+	StorageKey   common.Hash `db:"storage_key"`
+	StorageValue common.Hash `db:"storage_value"`
+}
+
+func FromStrings(csvRow []string) (StorageDiffRow, error) {
+	if len(csvRow) != ExpectedRowLength {
+		return StorageDiffRow{}, ErrRowMalformed{Length: len(csvRow)}
+	}
+	height, err := strconv.Atoi(csvRow[2])
+	if err != nil {
+		return StorageDiffRow{}, err
+	}
+	return StorageDiffRow{
+		Contract:     common.HexToAddress(csvRow[0]),
+		BlockHash:    common.HexToHash(csvRow[1]),
+		BlockHeight:  height,
+		StorageKey:   common.HexToHash(csvRow[3]),
+		StorageValue: common.HexToHash(csvRow[4]),
+	}, nil
+}

libraries/shared/utils/row_test.go

@@ -0,0 +1,58 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package utils_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
+)
+
+var _ = Describe("Storage row parsing", func() {
+	It("converts an array of strings to a row struct", func() {
+		contract := "0x123"
+		blockHash := "0x456"
+		blockHeight := "789"
+		storageKey := "0x987"
+		storageValue := "0x654"
+		data := []string{contract, blockHash, blockHeight, storageKey, storageValue}
+
+		result, err := utils.FromStrings(data)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.Contract).To(Equal(common.HexToAddress(contract)))
+		Expect(result.BlockHash).To(Equal(common.HexToHash(blockHash)))
+		Expect(result.BlockHeight).To(Equal(789))
+		Expect(result.StorageKey).To(Equal(common.HexToHash(storageKey)))
+		Expect(result.StorageValue).To(Equal(common.HexToHash(storageValue)))
+	})
+
+	It("returns an error if row is missing data", func() {
+		_, err := utils.FromStrings([]string{"0x123"})
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(utils.ErrRowMalformed{Length: 1}))
+	})
+
+	It("returns error if block height malformed", func() {
+		_, err := utils.FromStrings([]string{"", "", "", "", ""})
+
+		Expect(err).To(HaveOccurred())
+	})
+})

libraries/shared/utils/shared_suite_test.go

@@ -14,11 +14,11 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package transformer_test
+package utils_test
 
 import (
+	"github.com/sirupsen/logrus"
 	"io/ioutil"
-	"log"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
@@ -27,9 +27,9 @@ import (
 
 func TestShared(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Shared Transformer Suite")
+	RunSpecs(t, "Shared Suite")
 }
 
 var _ = BeforeSuite(func() {
-	log.SetOutput(ioutil.Discard)
+	logrus.SetOutput(ioutil.Discard)
 })

libraries/shared/utils/value.go

@@ -0,0 +1,47 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package utils
+
+type ValueType int
+
+const (
+	Uint256 ValueType = iota
+	Bytes32
+	Address
+)
+
+type Key string
+
+const (
+	Ilk  Key = "ilk"
+	Guy  Key = "guy"
+	Flip Key = "flip"
+)
+
+type StorageValueMetadata struct {
+	Name string
+	Keys map[Key]string
+	Type ValueType
+}
+
+func GetStorageValueMetadata(name string, keys map[Key]string, t ValueType) StorageValueMetadata {
+	return StorageValueMetadata{
+		Name: name,
+		Keys: keys,
+		Type: t,
+	}
+}

libraries/shared/watcher/event_watcher.go

@@ -22,26 +22,28 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	log "github.com/sirupsen/logrus"
 
+	chunk "github.com/vulcanize/vulcanizedb/libraries/shared/chunker"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
+	fetch "github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	repo "github.com/vulcanize/vulcanizedb/libraries/shared/repository"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
-	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
 )
 
 type EventWatcher struct {
-	Transformers  []transformer.Transformer
+	Transformers  []transformer.EventTransformer
 	DB            *postgres.DB
-	Fetcher       shared.LogFetcher
-	Chunker       shared.Chunker
+	Fetcher       fetch.LogFetcher
+	Chunker       chunk.Chunker
 	Addresses     []common.Address
 	Topics        []common.Hash
 	StartingBlock *int64
 }
 
 func NewEventWatcher(db *postgres.DB, bc core.BlockChain) EventWatcher {
-	chunker := shared.NewLogChunker()
-	fetcher := shared.NewFetcher(bc)
+	chunker := chunk.NewLogChunker()
+	fetcher := fetch.NewFetcher(bc)
 	return EventWatcher{
 		DB:      db,
 		Fetcher: fetcher,
@@ -83,13 +85,13 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti
 		return fmt.Errorf("No transformers added to watcher")
 	}
 
-	checkedColumnNames, err := shared.GetCheckedColumnNames(watcher.DB)
+	checkedColumnNames, err := repo.GetCheckedColumnNames(watcher.DB)
 	if err != nil {
 		return err
 	}
-	notCheckedSQL := shared.CreateNotCheckedSQL(checkedColumnNames, recheckHeaders)
+	notCheckedSQL := repo.CreateNotCheckedSQL(checkedColumnNames, recheckHeaders)
 
-	missingHeaders, err := shared.MissingHeaders(*watcher.StartingBlock, -1, watcher.DB, notCheckedSQL)
+	missingHeaders, err := repo.MissingHeaders(*watcher.StartingBlock, -1, watcher.DB, notCheckedSQL)
 	if err != nil {
 		log.Error("Fetching of missing headers failed in watcher!")
 		return err

libraries/shared/watcher/event_watcher_test.go

@@ -25,14 +25,14 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
-	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
-	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 

libraries/shared/watcher/storage_watcher.go

@@ -14,17 +14,18 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package shared
+package watcher
 
 import (
+	"strings"
+	"reflect"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/sirupsen/logrus"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage"
-	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
-	"reflect"
-	"strings"
 
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/fs"
 )
 
@@ -32,11 +33,11 @@ type StorageWatcher struct {
 	db           *postgres.DB
 	tailer       fs.Tailer
 	Queue        IStorageQueue
-	Transformers map[common.Address]storage.Transformer
+	Transformers map[common.Address]transformer.StorageTransformer
 }
 
 func NewStorageWatcher(tailer fs.Tailer, db *postgres.DB) StorageWatcher {
-	transformers := make(map[common.Address]storage.Transformer)
+	transformers := make(map[common.Address]transformer.StorageTransformer)
 	queue := NewStorageQueue(db)
 	return StorageWatcher{
 		db:           db,
@@ -46,7 +47,7 @@ func NewStorageWatcher(tailer fs.Tailer, db *postgres.DB) StorageWatcher {
 	}
 }
 
-func (watcher StorageWatcher) AddTransformers(initializers []storage.TransformerInitializer) {
+func (watcher StorageWatcher) AddTransformers(initializers []transformer.StorageTransformerInitializer) {
 	for _, initializer := range initializers {
 		transformer := initializer(watcher.db)
 		watcher.Transformers[transformer.ContractAddress()] = transformer
@@ -59,13 +60,13 @@ func (watcher StorageWatcher) Execute() error {
 		return tailErr
 	}
 	for line := range t.Lines {
-		row, parseErr := shared.FromStrings(strings.Split(line.Text, ","))
+		row, parseErr := utils.FromStrings(strings.Split(line.Text, ","))
 		if parseErr != nil {
 			return parseErr
 		}
 		transformer, ok := watcher.Transformers[row.Contract]
 		if !ok {
-			logrus.Warn(shared.ErrContractNotFound{Contract: row.Contract.Hex()}.Error())
+			logrus.Warn(utils.ErrContractNotFound{Contract: row.Contract.Hex()}.Error())
 			continue
 		}
 		executeErr := transformer.Execute(row)
@@ -85,5 +86,5 @@ func (watcher StorageWatcher) Execute() error {
 }
 
 func isKeyNotFound(executeErr error) bool {
-	return reflect.TypeOf(executeErr) == reflect.TypeOf(shared.ErrStorageKeyNotFound{})
+	return reflect.TypeOf(executeErr) == reflect.TypeOf(utils.ErrStorageKeyNotFound{})
 }

libraries/shared/watcher/storage_watcher_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package shared_test
+package watcher_test
 
 import (
 	"errors"
@@ -30,12 +30,12 @@ import (
 	. "github.com/onsi/gomega"
 	"github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
-	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage"
-	shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
-	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
@@ -43,39 +43,39 @@ var _ = Describe("Storage Watcher", func() {
 	It("adds transformers", func() {
 		fakeAddress := common.HexToAddress("0x12345")
 		fakeTransformer := &mocks.MockStorageTransformer{Address: fakeAddress}
-		watcher := shared.NewStorageWatcher(&fakes.MockTailer{}, test_config.NewTestDB(core.Node{}))
+		w := watcher.NewStorageWatcher(&fakes.MockTailer{}, test_config.NewTestDB(core.Node{}))
 
-		watcher.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
+		w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
 
-		Expect(watcher.Transformers[fakeAddress]).To(Equal(fakeTransformer))
+		Expect(w.Transformers[fakeAddress]).To(Equal(fakeTransformer))
 	})
 
 	It("reads the tail of the storage diffs file", func() {
 		mockTailer := fakes.NewMockTailer()
-		watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
+		w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
 
 		assert(func(err error) {
 			Expect(err).To(BeNil())
 			Expect(mockTailer.TailCalled).To(BeTrue())
-		}, watcher, mockTailer, []*tail.Line{})
+		}, w, mockTailer, []*tail.Line{})
 	})
 
 	It("returns error if row parsing fails", func() {
 		mockTailer := fakes.NewMockTailer()
-		watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
+		w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
 		line := &tail.Line{Text: "oops"}
 
 		assert(func(err error) {
 			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(shared2.ErrRowMalformed{Length: 1}))
-		}, watcher, mockTailer, []*tail.Line{line})
+			Expect(err).To(MatchError(utils.ErrRowMalformed{Length: 1}))
+		}, w, mockTailer, []*tail.Line{line})
 	})
 
 	It("logs error if no transformer can parse storage row", func() {
 		mockTailer := fakes.NewMockTailer()
 		address := common.HexToAddress("0x12345")
 		line := getFakeLine(address.Bytes())
-		watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
+		w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
 		tempFile, err := ioutil.TempFile("", "log")
 		defer os.Remove(tempFile.Name())
 		Expect(err).NotTo(HaveOccurred())
@@ -85,24 +85,24 @@ var _ = Describe("Storage Watcher", func() {
 			Expect(err).NotTo(HaveOccurred())
 			logContent, readErr := ioutil.ReadFile(tempFile.Name())
 			Expect(readErr).NotTo(HaveOccurred())
-			Expect(string(logContent)).To(ContainSubstring(shared2.ErrContractNotFound{Contract: address.Hex()}.Error()))
-		}, watcher, mockTailer, []*tail.Line{line})
+			Expect(string(logContent)).To(ContainSubstring(utils.ErrContractNotFound{Contract: address.Hex()}.Error()))
+		}, w, mockTailer, []*tail.Line{line})
 	})
 
 	It("executes transformer with storage row", func() {
 		address := []byte{1, 2, 3}
 		line := getFakeLine(address)
 		mockTailer := fakes.NewMockTailer()
-		watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
+		w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
 		fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address)}
-		watcher.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
+		w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
 
 		assert(func(err error) {
 			Expect(err).To(BeNil())
-			expectedRow, err := shared2.FromStrings(strings.Split(line.Text, ","))
+			expectedRow, err := utils.FromStrings(strings.Split(line.Text, ","))
 			Expect(err).NotTo(HaveOccurred())
 			Expect(fakeTransformer.PassedRow).To(Equal(expectedRow))
-		}, watcher, mockTailer, []*tail.Line{line})
+		}, w, mockTailer, []*tail.Line{line})
 	})
 
 	Describe("when executing transformer fails", func() {
@@ -110,30 +110,30 @@ var _ = Describe("Storage Watcher", func() {
 			address := []byte{1, 2, 3}
 			line := getFakeLine(address)
 			mockTailer := fakes.NewMockTailer()
-			watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
+			w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
 			mockQueue := &mocks.MockStorageQueue{}
-			watcher.Queue = mockQueue
-			keyNotFoundError := shared2.ErrStorageKeyNotFound{Key: "unknown_storage_key"}
+			w.Queue = mockQueue
+			keyNotFoundError := utils.ErrStorageKeyNotFound{Key: "unknown_storage_key"}
 			fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address), ExecuteErr: keyNotFoundError}
-			watcher.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
+			w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
 
 			assert(func(err error) {
 				Expect(err).NotTo(HaveOccurred())
 				Expect(mockQueue.AddCalled).To(BeTrue())
-			}, watcher, mockTailer, []*tail.Line{line})
+			}, w, mockTailer, []*tail.Line{line})
 		})
 
 		It("logs error if queuing row fails", func() {
 			address := []byte{1, 2, 3}
 			line := getFakeLine(address)
 			mockTailer := fakes.NewMockTailer()
-			watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
+			w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
 			mockQueue := &mocks.MockStorageQueue{}
 			mockQueue.AddError = fakes.FakeError
-			watcher.Queue = mockQueue
-			keyNotFoundError := shared2.ErrStorageKeyNotFound{Key: "unknown_storage_key"}
+			w.Queue = mockQueue
+			keyNotFoundError := utils.ErrStorageKeyNotFound{Key: "unknown_storage_key"}
 			fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address), ExecuteErr: keyNotFoundError}
-			watcher.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
+			w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
 			tempFile, err := ioutil.TempFile("", "log")
 			defer os.Remove(tempFile.Name())
 			Expect(err).NotTo(HaveOccurred())
@@ -145,17 +145,17 @@ var _ = Describe("Storage Watcher", func() {
 				logContent, readErr := ioutil.ReadFile(tempFile.Name())
 				Expect(readErr).NotTo(HaveOccurred())
 				Expect(string(logContent)).To(ContainSubstring(fakes.FakeError.Error()))
-			}, watcher, mockTailer, []*tail.Line{line})
+			}, w, mockTailer, []*tail.Line{line})
 		})
 
 		It("logs any other error", func() {
 			address := []byte{1, 2, 3}
 			line := getFakeLine(address)
 			mockTailer := fakes.NewMockTailer()
-			watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
+			w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
 			executionError := errors.New("storage watcher failed attempting to execute transformer")
 			fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address), ExecuteErr: executionError}
-			watcher.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
+			w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
 			tempFile, err := ioutil.TempFile("", "log")
 			defer os.Remove(tempFile.Name())
 			Expect(err).NotTo(HaveOccurred())
@@ -166,12 +166,12 @@ var _ = Describe("Storage Watcher", func() {
 				logContent, readErr := ioutil.ReadFile(tempFile.Name())
 				Expect(readErr).NotTo(HaveOccurred())
 				Expect(string(logContent)).To(ContainSubstring(executionError.Error()))
-			}, watcher, mockTailer, []*tail.Line{line})
+			}, w, mockTailer, []*tail.Line{line})
 		})
 	})
 })
 
-func assert(assertion func(err error), watcher shared.StorageWatcher, mockTailer *fakes.MockTailer, lines []*tail.Line) {
+func assert(assertion func(err error), watcher watcher.StorageWatcher, mockTailer *fakes.MockTailer, lines []*tail.Line) {
 	errs := make(chan error, 1)
 	done := make(chan bool, 1)
 	go execute(watcher, errs, done)
@@ -190,8 +190,8 @@ func assert(assertion func(err error), watcher shared.StorageWatcher, mockTailer
 	}
 }
 
-func execute(watcher shared.StorageWatcher, errs chan error, done chan bool) {
-	err := watcher.Execute()
+func execute(w watcher.StorageWatcher, errs chan error, done chan bool) {
+	err := w.Execute()
 	if err != nil {
 		errs <- err
 	} else {