rebase; extract factories and the mocks they are dependent on to

libraries/shared; adjust omni test_helpers to drop and recreate
checked_headers table to avoid reaching 1600 column limit after repeated
tests (dropping columns doesn't actually remove them from contributing
to that limit)

libraries/shared/chunker/chunker_suite_test.go

@@ -0,0 +1,35 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package chunker_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	log "github.com/sirupsen/logrus"
+	"io/ioutil"
+)
+
+func TestFactories(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Shared Chunker Suite")
+}
+
+var _ = BeforeSuite(func() {
+	log.SetOutput(ioutil.Discard)
+})

libraries/shared/factories/converter.go

@@ -0,0 +1,24 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package factories
+
+import "github.com/ethereum/go-ethereum/core/types"
+
+type Converter interface {
+	ToEntities(contractAbi string, ethLog []types.Log) ([]interface{}, error)
+	ToModels([]interface{}) ([]interface{}, error)
+}

libraries/shared/factories/factories_suite_test.go

@@ -0,0 +1,35 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package factories_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	log "github.com/sirupsen/logrus"
+	"io/ioutil"
+)
+
+func TestFactories(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Factories Suite")
+}
+
+var _ = BeforeSuite(func() {
+	log.SetOutput(ioutil.Discard)
+})

libraries/shared/factories/log_note_converter.go

@@ -0,0 +1,23 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package factories
+
+import "github.com/ethereum/go-ethereum/core/types"
+
+type LogNoteConverter interface {
+	ToModels(ethLog []types.Log) ([]interface{}, error)
+}

libraries/shared/factories/log_note_transformer.go

@@ -0,0 +1,73 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package factories
+
+import (
+	"github.com/ethereum/go-ethereum/core/types"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type LogNoteTransformer struct {
+	Config     transformer.TransformerConfig
+	Converter  LogNoteConverter
+	Repository Repository
+}
+
+func (tr LogNoteTransformer) NewLogNoteTransformer(db *postgres.DB) transformer.EventTransformer {
+	tr.Repository.SetDB(db)
+	return tr
+}
+
+func (tr LogNoteTransformer) Execute(logs []types.Log, header core.Header, recheckedHeader constants.TransformerExecution) error {
+	transformerName := tr.Config.TransformerName
+
+	// No matching logs, mark the header as checked for this type of logs
+	if len(logs) < 1 {
+		err := tr.Repository.MarkHeaderChecked(header.Id)
+		if err != nil {
+			log.Printf("Error marking header as checked in %v: %v", transformerName, err)
+			return err
+		}
+		return nil
+	}
+
+	models, err := tr.Converter.ToModels(logs)
+	if err != nil {
+		log.Printf("Error converting logs in %v: %v", transformerName, err)
+		return err
+	}
+
+	err = tr.Repository.Create(header.Id, models)
+	if err != nil {
+		log.Printf("Error persisting %v record: %v", transformerName, err)
+		return err
+	}
+	return nil
+}
+
+func (tr LogNoteTransformer) GetName() string {
+	return tr.Config.TransformerName
+}
+
+func (tr LogNoteTransformer) GetConfig() transformer.TransformerConfig {
+	return tr.Config
+}

libraries/shared/factories/log_note_transformer_test.go

@@ -0,0 +1,126 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package factories_test
+
+import (
+	"math/rand"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/factories"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+)
+
+var _ = Describe("LogNoteTransformer", func() {
+	var (
+		repository mocks.MockRepository
+		converter  mocks.MockLogNoteConverter
+		headerOne  core.Header
+		t          transformer.EventTransformer
+		model      test_data.GenericModel
+		config     = test_data.GenericTestConfig
+		logs       = test_data.GenericTestLogs
+	)
+
+	BeforeEach(func() {
+		repository = mocks.MockRepository{}
+		converter = mocks.MockLogNoteConverter{}
+		t = factories.LogNoteTransformer{
+			Config:     config,
+			Converter:  &converter,
+			Repository: &repository,
+		}.NewLogNoteTransformer(nil)
+
+		headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()}
+	})
+
+	It("sets the database", func() {
+		Expect(repository.SetDbCalled).To(BeTrue())
+	})
+
+	It("marks header checked if no logs are provided", func() {
+		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
+
+		Expect(err).NotTo(HaveOccurred())
+		repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id)
+	})
+
+	It("doesn't attempt to convert or persist an empty collection when there are no logs", func() {
+		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(converter.ToModelsCalledCounter).To(Equal(0))
+		Expect(repository.CreateCalledCounter).To(Equal(0))
+	})
+
+	It("does not call repository.MarkCheckedHeader when there are logs", func() {
+		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+
+		Expect(err).NotTo(HaveOccurred())
+		repository.AssertMarkHeaderCheckedNotCalled()
+	})
+
+	It("returns error if marking header checked returns err", func() {
+		repository.SetMarkHeaderCheckedError(fakes.FakeError)
+
+		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("converts matching logs to models", func() {
+		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(converter.PassedLogs).To(Equal(logs))
+	})
+
+	It("returns error if converter returns error", func() {
+		converter.SetConverterError(fakes.FakeError)
+
+		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("persists the model", func() {
+		converter.SetReturnModels([]interface{}{model})
+		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.PassedHeaderID).To(Equal(headerOne.Id))
+		Expect(repository.PassedModels).To(Equal([]interface{}{model}))
+	})
+
+	It("returns error if repository returns error for create", func() {
+		repository.SetCreateError(fakes.FakeError)
+
+		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+})

libraries/shared/factories/repository.go

@@ -0,0 +1,30 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package factories
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type Repository interface {
+	Create(headerID int64, models []interface{}) error
+	MarkHeaderChecked(headerID int64) error
+	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
+	RecheckHeaders(startingBlockNumber, endingBlockNUmber int64) ([]core.Header, error)
+	SetDB(db *postgres.DB)
+}

libraries/shared/factories/storage/storage_suite_test.go

@@ -0,0 +1,29 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestStorage(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Storage Factories Suite")
+}

libraries/shared/factories/storage/transformer.go

@@ -0,0 +1,55 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type Transformer struct {
+	Address    common.Address
+	Mappings   storage.Mappings
+	Repository repository.StorageRepository
+}
+
+func (transformer Transformer) NewTransformer(db *postgres.DB) transformer.StorageTransformer {
+	transformer.Mappings.SetDB(db)
+	transformer.Repository.SetDB(db)
+	return transformer
+}
+
+func (transformer Transformer) ContractAddress() common.Address {
+	return transformer.Address
+}
+
+func (transformer Transformer) Execute(row utils.StorageDiffRow) error {
+	metadata, lookupErr := transformer.Mappings.Lookup(row.StorageKey)
+	if lookupErr != nil {
+		return lookupErr
+	}
+	value, decodeErr := utils.Decode(row, metadata)
+	if decodeErr != nil {
+		return decodeErr
+	}
+	return transformer.Repository.Create(row.BlockHeight, row.BlockHash.Hex(), metadata, value)
+}

libraries/shared/factories/storage/transformer_test.go

@@ -0,0 +1,103 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/storage"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+)
+
+var _ = Describe("Storage transformer", func() {
+	var (
+		mappings   *mocks.MockMappings
+		repository *mocks.MockStorageRepository
+		t          storage.Transformer
+	)
+
+	BeforeEach(func() {
+		mappings = &mocks.MockMappings{}
+		repository = &mocks.MockStorageRepository{}
+		t = storage.Transformer{
+			Address:    common.Address{},
+			Mappings:   mappings,
+			Repository: repository,
+		}
+	})
+
+	It("returns the contract address being watched", func() {
+		fakeAddress := common.HexToAddress("0x12345")
+		t.Address = fakeAddress
+
+		Expect(t.ContractAddress()).To(Equal(fakeAddress))
+	})
+
+	It("looks up metadata for storage key", func() {
+		t.Execute(utils.StorageDiffRow{})
+
+		Expect(mappings.LookupCalled).To(BeTrue())
+	})
+
+	It("returns error if lookup fails", func() {
+		mappings.LookupErr = fakes.FakeError
+
+		err := t.Execute(utils.StorageDiffRow{})
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("creates storage row with decoded data", func() {
+		fakeMetadata := utils.StorageValueMetadata{Type: utils.Address}
+		mappings.Metadata = fakeMetadata
+		rawValue := common.HexToAddress("0x12345")
+		fakeBlockNumber := 123
+		fakeBlockHash := "0x67890"
+		fakeRow := utils.StorageDiffRow{
+			Contract:     common.Address{},
+			BlockHash:    common.HexToHash(fakeBlockHash),
+			BlockHeight:  fakeBlockNumber,
+			StorageKey:   common.Hash{},
+			StorageValue: rawValue.Hash(),
+		}
+
+		err := t.Execute(fakeRow)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.PassedBlockNumber).To(Equal(fakeBlockNumber))
+		Expect(repository.PassedBlockHash).To(Equal(common.HexToHash(fakeBlockHash).Hex()))
+		Expect(repository.PassedMetadata).To(Equal(fakeMetadata))
+		Expect(repository.PassedValue.(string)).To(Equal(rawValue.Hex()))
+	})
+
+	It("returns error if creating row fails", func() {
+		rawValue := common.HexToAddress("0x12345")
+		fakeMetadata := utils.StorageValueMetadata{Type: utils.Address}
+		mappings.Metadata = fakeMetadata
+		repository.CreateErr = fakes.FakeError
+
+		err := t.Execute(utils.StorageDiffRow{StorageValue: rawValue.Hash()})
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+})

libraries/shared/factories/transformer.go

@@ -0,0 +1,80 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package factories
+
+import (
+	"github.com/ethereum/go-ethereum/core/types"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type Transformer struct {
+	Config     transformer.TransformerConfig
+	Converter  Converter
+	Repository Repository
+}
+
+func (transformer Transformer) NewTransformer(db *postgres.DB) transformer.EventTransformer {
+	transformer.Repository.SetDB(db)
+	return transformer
+}
+
+func (transformer Transformer) Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error {
+	transformerName := transformer.Config.TransformerName
+	config := transformer.Config
+
+	if len(logs) < 1 {
+		err := transformer.Repository.MarkHeaderChecked(header.Id)
+		if err != nil {
+			log.Printf("Error marking header as checked in %v: %v", transformerName, err)
+			return err
+		}
+		return nil
+	}
+
+	entities, err := transformer.Converter.ToEntities(config.ContractAbi, logs)
+	if err != nil {
+		log.Printf("Error converting logs to entities in %v: %v", transformerName, err)
+		return err
+	}
+
+	models, err := transformer.Converter.ToModels(entities)
+	if err != nil {
+		log.Printf("Error converting entities to models in %v: %v", transformerName, err)
+		return err
+	}
+
+	err = transformer.Repository.Create(header.Id, models)
+	if err != nil {
+		log.Printf("Error persisting %v record: %v", transformerName, err)
+		return err
+	}
+
+	return nil
+}
+
+func (transformer Transformer) GetName() string {
+	return transformer.Config.TransformerName
+}
+
+func (transformer Transformer) GetConfig() transformer.TransformerConfig {
+	return transformer.Config
+}

libraries/shared/factories/transformer_test.go

@@ -0,0 +1,147 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package factories_test
+
+import (
+	"math/rand"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/factories"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+)
+
+var _ = Describe("Transformer", func() {
+	var (
+		repository mocks.MockRepository
+		converter  mocks.MockConverter
+		t          transformer.EventTransformer
+		headerOne  core.Header
+		config     = test_data.GenericTestConfig
+		logs       = test_data.GenericTestLogs
+	)
+
+	BeforeEach(func() {
+		repository = mocks.MockRepository{}
+		converter = mocks.MockConverter{}
+
+		t = factories.Transformer{
+			Repository: &repository,
+			Converter:  &converter,
+			Config:     config,
+		}.NewTransformer(nil)
+
+		headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()}
+	})
+
+	It("sets the db", func() {
+		Expect(repository.SetDbCalled).To(BeTrue())
+	})
+
+	It("marks header checked if no logs returned", func() {
+		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
+
+		Expect(err).NotTo(HaveOccurred())
+		repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id)
+	})
+
+	It("doesn't attempt to convert or persist an empty collection when there are no logs", func() {
+		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(converter.ToEntitiesCalledCounter).To(Equal(0))
+		Expect(converter.ToModelsCalledCounter).To(Equal(0))
+		Expect(repository.CreateCalledCounter).To(Equal(0))
+	})
+
+	It("does not call repository.MarkCheckedHeader when there are logs", func() {
+		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+
+		Expect(err).NotTo(HaveOccurred())
+		repository.AssertMarkHeaderCheckedNotCalled()
+	})
+
+	It("returns error if marking header checked returns err", func() {
+		repository.SetMarkHeaderCheckedError(fakes.FakeError)
+
+		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("converts an eth log to an entity", func() {
+		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(converter.ContractAbi).To(Equal(config.ContractAbi))
+		Expect(converter.LogsToConvert).To(Equal(logs))
+	})
+
+	It("returns an error if converter fails", func() {
+		converter.ToEntitiesError = fakes.FakeError
+
+		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("converts an entity to a model", func() {
+		converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
+
+		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(converter.EntitiesToConvert[0]).To(Equal(test_data.GenericEntity{}))
+	})
+
+	It("returns an error if converting to models fails", func() {
+		converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
+		converter.ToModelsError = fakes.FakeError
+
+		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("persists the record", func() {
+		converter.ModelsToReturn = []interface{}{test_data.GenericModel{}}
+
+		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.PassedHeaderID).To(Equal(headerOne.Id))
+		Expect(repository.PassedModels[0]).To(Equal(test_data.GenericModel{}))
+	})
+
+	It("returns error if persisting the record fails", func() {
+		repository.SetCreateError(fakes.FakeError)
+		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+})

libraries/shared/fetcher/fetcher_suite_test.go

@@ -0,0 +1,35 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package fetcher_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	log "github.com/sirupsen/logrus"
+	"io/ioutil"
+)
+
+func TestFactories(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Shared Fetcher Suite")
+}
+
+var _ = BeforeSuite(func() {
+	log.SetOutput(ioutil.Discard)
+})

libraries/shared/mocks/converter.go

@@ -0,0 +1,58 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+type MockConverter struct {
+	ToEntitiesError         error
+	PassedContractAddresses []string
+	ToModelsError           error
+	entityConverterError    error
+	modelConverterError     error
+	ContractAbi             string
+	LogsToConvert           []types.Log
+	EntitiesToConvert       []interface{}
+	EntitiesToReturn        []interface{}
+	ModelsToReturn          []interface{}
+	ToEntitiesCalledCounter int
+	ToModelsCalledCounter   int
+}
+
+func (converter *MockConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]interface{}, error) {
+	for _, log := range ethLogs {
+		converter.PassedContractAddresses = append(converter.PassedContractAddresses, log.Address.Hex())
+	}
+	converter.ContractAbi = contractAbi
+	converter.LogsToConvert = ethLogs
+	return converter.EntitiesToReturn, converter.ToEntitiesError
+}
+
+func (converter *MockConverter) ToModels(entities []interface{}) ([]interface{}, error) {
+	converter.EntitiesToConvert = entities
+	return converter.ModelsToReturn, converter.ToModelsError
+}
+
+func (converter *MockConverter) SetToEntityConverterError(err error) {
+	converter.entityConverterError = err
+}
+
+func (c *MockConverter) SetToModelConverterError(err error) {
+	c.modelConverterError = err
+}

libraries/shared/mocks/log_note_converter.go

@@ -0,0 +1,42 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+type MockLogNoteConverter struct {
+	err                   error
+	returnModels          []interface{}
+	PassedLogs            []types.Log
+	ToModelsCalledCounter int
+}
+
+func (converter *MockLogNoteConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
+	converter.PassedLogs = ethLogs
+	converter.ToModelsCalledCounter++
+	return converter.returnModels, converter.err
+}
+
+func (converter *MockLogNoteConverter) SetConverterError(e error) {
+	converter.err = e
+}
+
+func (converter *MockLogNoteConverter) SetReturnModels(models []interface{}) {
+	converter.returnModels = models
+}

libraries/shared/mocks/mappings.go

@@ -0,0 +1,39 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type MockMappings struct {
+	Metadata     utils.StorageValueMetadata
+	LookupCalled bool
+	LookupErr    error
+}
+
+func (mappings *MockMappings) Lookup(key common.Hash) (utils.StorageValueMetadata, error) {
+	mappings.LookupCalled = true
+	return mappings.Metadata, mappings.LookupErr
+}
+
+func (*MockMappings) SetDB(db *postgres.DB) {
+	panic("implement me")
+}

libraries/shared/mocks/repository.go

@@ -0,0 +1,98 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type MockRepository struct {
+	createError                      error
+	markHeaderCheckedError           error
+	MarkHeaderCheckedPassedHeaderIDs []int64
+	CreatedHeaderIds                 []int64
+	missingHeaders                   []core.Header
+	allHeaders                       []core.Header
+	missingHeadersError              error
+	PassedStartingBlockNumber        int64
+	PassedEndingBlockNumber          int64
+	PassedHeaderID                   int64
+	PassedModels                     []interface{}
+	SetDbCalled                      bool
+	CreateCalledCounter              int
+}
+
+func (repository *MockRepository) Create(headerID int64, models []interface{}) error {
+	repository.PassedHeaderID = headerID
+	repository.PassedModels = models
+	repository.CreatedHeaderIds = append(repository.CreatedHeaderIds, headerID)
+	repository.CreateCalledCounter++
+
+	return repository.createError
+}
+
+func (repository *MockRepository) MarkHeaderChecked(headerID int64) error {
+	repository.MarkHeaderCheckedPassedHeaderIDs = append(repository.MarkHeaderCheckedPassedHeaderIDs, headerID)
+	return repository.markHeaderCheckedError
+}
+
+func (repository *MockRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
+	repository.PassedStartingBlockNumber = startingBlockNumber
+	repository.PassedEndingBlockNumber = endingBlockNumber
+	return repository.missingHeaders, repository.missingHeadersError
+}
+
+func (repository *MockRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
+	repository.PassedStartingBlockNumber = startingBlockNumber
+	repository.PassedEndingBlockNumber = endingBlockNumber
+	return repository.allHeaders, nil
+}
+
+func (repository *MockRepository) SetDB(db *postgres.DB) {
+	repository.SetDbCalled = true
+}
+
+func (repository *MockRepository) SetMissingHeadersError(e error) {
+	repository.missingHeadersError = e
+}
+
+func (repository *MockRepository) SetAllHeaders(headers []core.Header) {
+	repository.allHeaders = headers
+}
+
+func (repository *MockRepository) SetMissingHeaders(headers []core.Header) {
+	repository.missingHeaders = headers
+}
+
+func (repository *MockRepository) SetMarkHeaderCheckedError(e error) {
+	repository.markHeaderCheckedError = e
+}
+
+func (repository *MockRepository) SetCreateError(e error) {
+	repository.createError = e
+}
+
+func (repository *MockRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
+	Expect(repository.MarkHeaderCheckedPassedHeaderIDs).To(ContainElement(i))
+}
+
+func (repository *MockRepository) AssertMarkHeaderCheckedNotCalled() {
+	Expect(len(repository.MarkHeaderCheckedPassedHeaderIDs)).To(Equal(0))
+}

libraries/shared/mocks/storage_queue.go

@@ -0,0 +1,31 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+)
+
+type MockStorageQueue struct {
+	AddCalled bool
+	AddError  error
+}
+
+func (queue *MockStorageQueue) Add(row utils.StorageDiffRow) error {
+	queue.AddCalled = true
+	return queue.AddError
+}

libraries/shared/mocks/storage_repository.go

@@ -0,0 +1,42 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type MockStorageRepository struct {
+	CreateErr         error
+	PassedBlockNumber int
+	PassedBlockHash   string
+	PassedMetadata    utils.StorageValueMetadata
+	PassedValue       interface{}
+}
+
+func (repository *MockStorageRepository) Create(blockNumber int, blockHash string, metadata utils.StorageValueMetadata, value interface{}) error {
+	repository.PassedBlockNumber = blockNumber
+	repository.PassedBlockHash = blockHash
+	repository.PassedMetadata = metadata
+	repository.PassedValue = value
+	return repository.CreateErr
+}
+
+func (*MockStorageRepository) SetDB(db *postgres.DB) {
+	panic("implement me")
+}

libraries/shared/repository/repository.go

@@ -27,30 +27,6 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
-func GetOrCreateIlk(ilk string, db *postgres.DB) (int, error) {
-	var ilkID int
-	err := db.Get(&ilkID, `SELECT id FROM maker.ilks WHERE ilk = $1`, ilk)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			insertErr := db.QueryRow(`INSERT INTO maker.ilks (ilk) VALUES ($1) RETURNING id`, ilk).Scan(&ilkID)
-			return ilkID, insertErr
-		}
-	}
-	return ilkID, err
-}
-
-func GetOrCreateIlkInTransaction(ilk string, tx *sql.Tx) (int, error) {
-	var ilkID int
-	err := tx.QueryRow(`SELECT id FROM maker.ilks WHERE ilk = $1`, ilk).Scan(&ilkID)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			insertErr := tx.QueryRow(`INSERT INTO maker.ilks (ilk) VALUES ($1) RETURNING id`, ilk).Scan(&ilkID)
-			return ilkID, insertErr
-		}
-	}
-	return ilkID, err
-}
-
 func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn string) error {
 	_, err := db.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
 		VALUES ($1, $2)

libraries/shared/repository/repository_suite_test.go

@@ -0,0 +1,35 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package repository_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	log "github.com/sirupsen/logrus"
+	"io/ioutil"
+)
+
+func TestFactories(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Shared Repository Suite")
+}
+
+var _ = BeforeSuite(func() {
+	log.SetOutput(ioutil.Discard)
+})

libraries/shared/repository/repository_utility_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	r2 "github.com/vulcanize/vulcanizedb/pkg/omni/light/repository"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
@@ -45,12 +46,15 @@ var _ = Describe("Repository utilities", func() {
 			headerIDs                []int64
 			notCheckedSQL            string
 			err                      error
+			hr                       r2.HeaderRepository
 		)
 
 		BeforeEach(func() {
 			db = test_config.NewTestDB(test_config.NewTestNode())
 			test_config.CleanTestDB(db)
 			headerRepository = repositories.NewHeaderRepository(db)
+			hr = r2.NewHeaderRepository(db)
+			hr.AddCheckColumns(getExpectedColumnNames())
 
 			columnNames, err := shared.GetCheckedColumnNames(db)
 			Expect(err).NotTo(HaveOccurred())
@@ -71,6 +75,10 @@ var _ = Describe("Repository utilities", func() {
 			}
 		})
 
+		AfterEach(func() {
+			test_config.CleanCheckedHeadersTable(db, getExpectedColumnNames())
+		})
+
 		It("only treats headers as checked if the event specific logs have been checked", func() {
 			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
@@ -111,11 +119,14 @@ var _ = Describe("Repository utilities", func() {
 	Describe("GetCheckedColumnNames", func() {
 		It("gets the column names from checked_headers", func() {
 			db := test_config.NewTestDB(test_config.NewTestNode())
+			hr := r2.NewHeaderRepository(db)
+			hr.AddCheckColumns(getExpectedColumnNames())
 			test_config.CleanTestDB(db)
 			expectedColumnNames := getExpectedColumnNames()
 			actualColumnNames, err := shared.GetCheckedColumnNames(db)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(actualColumnNames).To(Equal(expectedColumnNames))
+			test_config.CleanCheckedHeadersTable(db, getExpectedColumnNames())
 		})
 	})
 

libraries/shared/storage/storage_queue.go

@@ -0,0 +1,42 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage
+
+import (
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type IStorageQueue interface {
+	Add(row utils.StorageDiffRow) error
+}
+
+type StorageQueue struct {
+	db *postgres.DB
+}
+
+func NewStorageQueue(db *postgres.DB) StorageQueue {
+	return StorageQueue{db: db}
+}
+
+func (queue StorageQueue) Add(row utils.StorageDiffRow) error {
+	_, err := queue.db.Exec(`INSERT INTO public.queued_storage (contract,
+		block_hash, block_height, storage_key, storage_value) VALUES
+		($1, $2, $3, $4, $5)`, row.Contract.Bytes(), row.BlockHash.Bytes(),
+		row.BlockHeight, row.StorageKey.Bytes(), row.StorageValue.Bytes())
+	return err
+}

libraries/shared/storage/storage_queue_test.go

@@ -1,17 +1,18 @@
-package shared_test
+package storage_test
 
 import (
 	"github.com/ethereum/go-ethereum/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	shared2 "github.com/vulcanize/vulcanizedb/libraries/shared"
-	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
 var _ = Describe("Storage queue", func() {
 	It("adds a storage row to the db", func() {
-		row := shared.StorageDiffRow{
+		row := utils.StorageDiffRow{
 			Contract:     common.HexToAddress("0x123456"),
 			BlockHash:    common.HexToHash("0x678901"),
 			BlockHeight:  987,
@@ -19,12 +20,12 @@ var _ = Describe("Storage queue", func() {
 			StorageValue: common.HexToHash("0x198765"),
 		}
 		db := test_config.NewTestDB(test_config.NewTestNode())
-		queue := shared2.NewStorageQueue(db)
+		queue := storage.NewStorageQueue(db)
 
 		addErr := queue.Add(row)
 
 		Expect(addErr).NotTo(HaveOccurred())
-		var result shared.StorageDiffRow
+		var result utils.StorageDiffRow
 		getErr := db.Get(&result, `SELECT contract, block_hash, block_height, storage_key, storage_value FROM public.queued_storage`)
 		Expect(getErr).NotTo(HaveOccurred())
 		Expect(result).To(Equal(row))

libraries/shared/storage/storage_suite_test.go

@@ -0,0 +1,35 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	log "github.com/sirupsen/logrus"
+	"io/ioutil"
+)
+
+func TestFactories(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Shared Storage Suite")
+}
+
+var _ = BeforeSuite(func() {
+	log.SetOutput(ioutil.Discard)
+})

libraries/shared/storage/utils/utils_suite_test.go

@@ -27,7 +27,7 @@ import (
 
 func TestShared(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Storage Utils Suite")
+	RunSpecs(t, "Shared Storage Utils Suite")
 }
 
 var _ = BeforeSuite(func() {

libraries/shared/storage/utils/value.go

@@ -26,12 +26,6 @@ const (
 
 type Key string
 
-const (
-	Ilk  Key = "ilk"
-	Guy  Key = "guy"
-	Flip Key = "flip"
-)
-
 type StorageValueMetadata struct {
 	Name string
 	Keys map[Key]string

libraries/shared/storage_queue.go

@@ -1,26 +0,0 @@
-package shared
-
-import (
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
-)
-
-type IStorageQueue interface {
-	Add(row shared.StorageDiffRow) error
-}
-
-type StorageQueue struct {
-	db *postgres.DB
-}
-
-func NewStorageQueue(db *postgres.DB) StorageQueue {
-	return StorageQueue{db: db}
-}
-
-func (queue StorageQueue) Add(row shared.StorageDiffRow) error {
-	_, err := queue.db.Exec(`INSERT INTO public.queued_storage (contract,
-		block_hash, block_height, storage_key, storage_value) VALUES
-		($1, $2, $3, $4, $5)`, row.Contract.Bytes(), row.BlockHash.Bytes(),
-		row.BlockHeight, row.StorageKey.Bytes(), row.StorageValue.Bytes())
-	return err
-}

libraries/shared/test_data/generic.go

@@ -0,0 +1,61 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package test_data
+
+import (
+	"math/rand"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+)
+
+type GenericModel struct{}
+type GenericEntity struct{}
+
+var startingBlockNumber = rand.Int63()
+var topic = "0x" + randomString(64)
+var address = "0x" + randomString(38)
+
+var GenericTestLogs = []types.Log{{
+	Address:     common.HexToAddress(address),
+	Topics:      []common.Hash{common.HexToHash(topic)},
+	BlockNumber: uint64(startingBlockNumber),
+}}
+
+var GenericTestConfig = transformer.TransformerConfig{
+	TransformerName:     "generic-test-transformer",
+	ContractAddresses:   []string{address},
+	ContractAbi:         randomString(100),
+	Topic:               topic,
+	StartingBlockNumber: startingBlockNumber,
+	EndingBlockNumber:   startingBlockNumber + 1,
+}
+
+func randomString(length int) string {
+	var seededRand *rand.Rand = rand.New(
+		rand.NewSource(time.Now().UnixNano()))
+	charset := "abcdefghijklmnopqrstuvwxyz1234567890"
+	b := make([]byte, length)
+	for i := range b {
+		b[i] = charset[seededRand.Intn(len(charset))]
+	}
+
+	return string(b)
+}

libraries/shared/watcher/storage_watcher.go

@@ -17,12 +17,13 @@
 package watcher
 
 import (
-	"strings"
 	"reflect"
+	"strings"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/sirupsen/logrus"
 
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -32,13 +33,13 @@ import (
 type StorageWatcher struct {
 	db           *postgres.DB
 	tailer       fs.Tailer
-	Queue        IStorageQueue
+	Queue        storage.IStorageQueue
 	Transformers map[common.Address]transformer.StorageTransformer
 }
 
 func NewStorageWatcher(tailer fs.Tailer, db *postgres.DB) StorageWatcher {
 	transformers := make(map[common.Address]transformer.StorageTransformer)
-	queue := NewStorageQueue(db)
+	queue := storage.NewStorageQueue(db)
 	return StorageWatcher{
 		db:           db,
 		tailer:       tailer,