Backfill Frob log events

README.md

@@ -112,9 +112,5 @@ false
 If you have full rinkeby chaindata you can move it to `rinkeby_vulcanizedb_geth_data` docker volume to skip long wait of sync.
 
 ## Running the Tests
-
-### Unit Tests
-- `go test ./pkg/...`
-
-### Integration Tests
- - `go test ./...` to run all tests.
+- `make test`
+- Note: requires Ganache chain setup and seeded with `flip-kick.js` and `frob.js` (in that order)

cmd/backfillMakerLogs.go

@@ -30,16 +30,18 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/transformers"
 )
 
-// backfillAuctionLogsCmd represents the backfillAuctionLogs command
-var backfillAuctionLogsCmd = &cobra.Command{
-	Use:   "backfillAuctionLogs",
-	Short: "Backfill auction event logs",
-	Long: `Backfills auction event logs based on previously populated block Header records.
-vulcanize backfillAuctionLogs --config environments/local.toml
+// backfillMakerLogsCmd represents the backfillMakerLogs command
+var backfillMakerLogsCmd = &cobra.Command{
+	Use:   "backfillMakerLogs",
+	Short: "Backfill Maker event logs",
+	Long: `Backfills Maker event logs based on previously populated block Header records.
+This currently includes logs related to Multi-collateral Dai (frob) and Auctions (flip-kick).
+
+vulcanize backfillMakerLogs --config environments/local.toml
 
 This command expects a light sync to have been run, and the presence of header records in the Vulcanize database.`,
 	Run: func(cmd *cobra.Command, args []string) {
-		backfillAuctionLogs()
+		backfillMakerLogs()
 	},
 }
 
@@ -57,7 +59,7 @@ func blockChain() *geth.BlockChain {
 	return geth.NewBlockChain(vdbEthClient, vdbNode, transactionConverter)
 }
 
-func backfillAuctionLogs() {
+func backfillMakerLogs() {
 	blockChain := blockChain()
 	db, err := postgres.NewDB(databaseConfig, blockChain.Node())
 	if err != nil {
@@ -74,5 +76,5 @@ func backfillAuctionLogs() {
 }
 
 func init() {
-	rootCmd.AddCommand(backfillAuctionLogsCmd)
+	rootCmd.AddCommand(backfillMakerLogsCmd)
 }

db/migrations/1533844125_create_frob_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE maker.frob;

db/migrations/1533844125_create_frob_table.up.sql

@@ -0,0 +1,12 @@
+CREATE TABLE maker.frob (
+  id        SERIAL PRIMARY KEY,
+  header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
+  tx_idx    INTEGER,
+  ilk       bytea,
+  lad       bytea,
+  gem       NUMERIC,
+  ink       NUMERIC,
+  art       NUMERIC,
+  era       NUMERIC,
+  UNIQUE (header_id, tx_idx)
+);

db/schema.sql

@@ -82,6 +82,43 @@ CREATE SEQUENCE maker.flip_kick_db_id_seq
 ALTER SEQUENCE maker.flip_kick_db_id_seq OWNED BY maker.flip_kick.db_id;
 
 
+--
+-- Name: frob; Type: TABLE; Schema: maker; Owner: -
+--
+
+CREATE TABLE maker.frob (
+    id integer NOT NULL,
+    header_id integer NOT NULL,
+    tx_idx integer,
+    ilk bytea,
+    lad bytea,
+    gem numeric,
+    ink numeric,
+    art numeric,
+    era numeric
+);
+
+
+--
+-- Name: frob_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
+--
+
+CREATE SEQUENCE maker.frob_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: frob_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
+--
+
+ALTER SEQUENCE maker.frob_id_seq OWNED BY maker.frob.id;
+
+
 --
 -- Name: logs; Type: TABLE; Schema: public; Owner: -
 --
@@ -461,6 +498,13 @@ CREATE VIEW public.watched_event_logs AS
 ALTER TABLE ONLY maker.flip_kick ALTER COLUMN db_id SET DEFAULT nextval('maker.flip_kick_db_id_seq'::regclass);
 
 
+--
+-- Name: frob id; Type: DEFAULT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.frob ALTER COLUMN id SET DEFAULT nextval('maker.frob_id_seq'::regclass);
+
+
 --
 -- Name: blocks id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -540,6 +584,22 @@ ALTER TABLE ONLY maker.flip_kick
     ADD CONSTRAINT flip_kick_pkey PRIMARY KEY (db_id);
 
 
+--
+-- Name: frob frob_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.frob
+    ADD CONSTRAINT frob_header_id_tx_idx_key UNIQUE (header_id, tx_idx);
+
+
+--
+-- Name: frob frob_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.frob
+    ADD CONSTRAINT frob_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: blocks blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -679,6 +739,14 @@ ALTER TABLE ONLY maker.flip_kick
     ADD CONSTRAINT flip_kick_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
 
 
+--
+-- Name: frob frob_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.frob
+    ADD CONSTRAINT frob_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
+
+
 --
 -- Name: transactions blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
 --

examples/erc20_watcher/every_block/transformer.go

@@ -16,12 +16,13 @@ package every_block
 
 import (
 	"fmt"
-	"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
-	"github.com/vulcanize/vulcanizedb/libraries/shared"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"log"
 	"math/big"
+
+	"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
 type Transformer struct {

examples/erc20_watcher/every_block/transformers.go

@@ -16,7 +16,7 @@ package every_block
 
 import (
 	"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
-	"github.com/vulcanize/vulcanizedb/libraries/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
 func TransformerInitializers() []shared.TransformerInitializer {

libraries/shared/transformer_interface.go

@@ -1,23 +0,0 @@
-package shared
-
-import (
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-)
-
-type Transformer interface {
-	Execute() error
-}
-
-type TransformerInitializer func(db *postgres.DB, blockchain core.BlockChain) Transformer
-
-func HexToInt64(byteString string) int64 {
-	value := common.HexToHash(byteString)
-	return value.Big().Int64()
-}
-
-func HexToString(byteString string) string {
-	value := common.HexToHash(byteString)
-	return value.Big().String()
-}

libraries/shared/watcher.go

@@ -3,15 +3,16 @@ package shared
 import (
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
 type Watcher struct {
-	Transformers []Transformer
+	Transformers []shared.Transformer
 	DB           postgres.DB
 	Blockchain   core.BlockChain
 }
 
-func (watcher *Watcher) AddTransformers(us []TransformerInitializer) {
+func (watcher *Watcher) AddTransformers(us []shared.TransformerInitializer) {
 	for _, transformerInitializer := range us {
 		transformer := transformerInitializer(&watcher.DB, watcher.Blockchain)
 		watcher.Transformers = append(watcher.Transformers, transformer)

libraries/shared/watcher_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/libraries/shared"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
 type MockTransformer struct {
@@ -23,7 +24,7 @@ func (mh *MockTransformer) Execute() error {
 	return nil
 }
 
-func fakeTransformerInitializer(db *postgres.DB, blockchain core.BlockChain) shared.Transformer {
+func fakeTransformerInitializer(db *postgres.DB, blockchain core.BlockChain) shared2.Transformer {
 	return &MockTransformer{}
 }
 
@@ -31,7 +32,7 @@ var _ = Describe("Watcher", func() {
 	It("Adds transformers", func() {
 		watcher := shared.Watcher{}
 
-		watcher.AddTransformers([]shared.TransformerInitializer{fakeTransformerInitializer})
+		watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer})
 
 		Expect(len(watcher.Transformers)).To(Equal(1))
 		Expect(watcher.Transformers).To(ConsistOf(&MockTransformer{}))
@@ -40,8 +41,8 @@ var _ = Describe("Watcher", func() {
 	It("Adds transformers from multiple sources", func() {
 		watcher := shared.Watcher{}
 
-		watcher.AddTransformers([]shared.TransformerInitializer{fakeTransformerInitializer})
-		watcher.AddTransformers([]shared.TransformerInitializer{fakeTransformerInitializer})
+		watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer})
+		watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer})
 
 		Expect(len(watcher.Transformers)).To(Equal(2))
 	})
@@ -49,7 +50,7 @@ var _ = Describe("Watcher", func() {
 	It("Executes each transformer", func() {
 		watcher := shared.Watcher{}
 		fakeTransformer := &MockTransformer{}
-		watcher.Transformers = []shared.Transformer{fakeTransformer}
+		watcher.Transformers = []shared2.Transformer{fakeTransformer}
 
 		watcher.Execute()
 
@@ -59,7 +60,7 @@ var _ = Describe("Watcher", func() {
 	It("Returns an error if transformer returns an error", func() {
 		watcher := shared.Watcher{}
 		fakeTransformer := &MockTransformer{executeError: errors.New("Something bad happened")}
-		watcher.Transformers = []shared.Transformer{fakeTransformer}
+		watcher.Transformers = []shared2.Transformer{fakeTransformer}
 
 		err := watcher.Execute()
 

pkg/transformers/flip_kick/converter_test.go

@@ -28,7 +28,7 @@ import (
 )
 
 var _ = Describe("FlipKickEntity Converter", func() {
-	It("converts an Eth Log to and Entity", func() {
+	It("converts an Eth Log to an Entity", func() {
 		converter := flip_kick.FlipKickConverter{}
 		entity, err := converter.ToEntity(test_data.TemporaryFlipAddress, flip_kick.FlipperABI, test_data.EthFlipKickLog)
 

pkg/transformers/flip_kick/flip_kick_suite_test.go

@@ -23,9 +23,9 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-func TestEveryBlock(t *testing.T) {
+func TestFlipKick(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "EveryBlock Suite")
+	RunSpecs(t, "FlipKick Suite")
 }
 
 var _ = BeforeSuite(func() {

pkg/transformers/flip_kick/integration_test.go

@@ -26,6 +26,7 @@ import (
 	rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
 	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/flip_kick"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
@@ -43,7 +44,7 @@ var _ = Describe("Integration tests", func() {
 		realNode := node.MakeNode(rpcClient)
 		transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
 		realBlockChain := geth.NewBlockChain(blockChainClient, realNode, transactionConverter)
-		realFetcher := flip_kick.NewFetcher(realBlockChain)
+		realFetcher := shared.NewFetcher(realBlockChain)
 		topic0 := common.HexToHash(flip_kick.FlipKickSignature)
 		topics := [][]common.Hash{{topic0}}
 

pkg/transformers/flip_kick/transformer.go

@@ -21,13 +21,13 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
 type FlipKickTransformer struct {
-	Fetcher    LogFetcher
+	Fetcher    shared.LogFetcher
 	Converter  Converter
 	Repository Repository
 	Config     TransformerConfig
@@ -38,7 +38,7 @@ type FlipKickTransformerInitializer struct {
 }
 
 func (i FlipKickTransformerInitializer) NewFlipKickTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
-	fetcher := NewFetcher(blockChain)
+	fetcher := shared.NewFetcher(blockChain)
 	repository := NewFlipKickRepository(db)
 	transformer := FlipKickTransformer{
 		Fetcher:    fetcher,

pkg/transformers/flip_kick/transformer_test.go

@@ -26,13 +26,15 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/flip_kick"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
+	flip_kick_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/flip_kick"
 )
 
 var _ = Describe("FlipKick Transformer", func() {
 	var transformer flip_kick.FlipKickTransformer
-	var fetcher test_data.MockLogFetcher
-	var converter test_data.MockFlipKickConverter
-	var repository test_data.MockFlipKickRepository
+	var fetcher mocks.MockLogFetcher
+	var converter flip_kick_mocks.MockFlipKickConverter
+	var repository flip_kick_mocks.MockFlipKickRepository
 	var testConfig flip_kick.TransformerConfig
 	var blockNumber int64
 	var headerId int64
@@ -40,9 +42,9 @@ var _ = Describe("FlipKick Transformer", func() {
 	var logs []types.Log
 
 	BeforeEach(func() {
-		fetcher = test_data.MockLogFetcher{}
-		converter = test_data.MockFlipKickConverter{}
-		repository = test_data.MockFlipKickRepository{}
+		fetcher = mocks.MockLogFetcher{}
+		converter = flip_kick_mocks.MockFlipKickConverter{}
+		repository = flip_kick_mocks.MockFlipKickRepository{}
 		transformer = flip_kick.FlipKickTransformer{
 			Fetcher:    &fetcher,
 			Converter:  &converter,

pkg/transformers/frob/config.go

@@ -0,0 +1,31 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package frob
+
+type TransformerConfig struct {
+	ContractAddress     string
+	ContractAbi         string
+	Topics              []string
+	StartingBlockNumber int64
+	EndingBlockNumber   int64
+}
+
+var FrobConfig = TransformerConfig{
+	ContractAddress:     "0xff3f2400f1600f3f493a9a92704a29b96795af1a", //this is a temporary address deployed locally
+	ContractAbi:         FrobABI,
+	Topics:              []string{FrobEventSignature},
+	StartingBlockNumber: 0,
+	EndingBlockNumber:   100,
+}

pkg/transformers/frob/converter.go

@@ -0,0 +1,54 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package frob
+
+import (
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+
+	"github.com/vulcanize/vulcanizedb/pkg/geth"
+)
+
+type Converter interface {
+	ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (FrobEntity, error)
+	ToModel(flipKick FrobEntity) FrobModel
+}
+
+type FrobConverter struct {
+}
+
+func (FrobConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (FrobEntity, error) {
+	entity := FrobEntity{}
+	address := common.HexToAddress(contractAddress)
+	abi, err := geth.ParseAbi(contractAbi)
+	if err != nil {
+		return entity, err
+	}
+	contract := bind.NewBoundContract(address, abi, nil, nil, nil)
+	err = contract.UnpackLog(&entity, "Frob", ethLog)
+	return entity, err
+}
+
+func (FrobConverter) ToModel(frob FrobEntity) FrobModel {
+	return FrobModel{
+		Ilk: frob.Ilk[:],
+		Lad: frob.Lad[:],
+		Gem: frob.Gem.String(),
+		Ink: frob.Ink.String(),
+		Art: frob.Art.String(),
+		Era: frob.Era.String(),
+	}
+}

pkg/transformers/frob/converter_test.go

@@ -0,0 +1,42 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package frob_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+)
+
+var _ = Describe("Frob converter", func() {
+	It("converts a log to an entity", func() {
+		converter := frob.FrobConverter{}
+
+		entity, err := converter.ToEntity(test_data.TemporaryFrobAddress, frob.FrobABI, test_data.EthFrobLog)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(entity).To(Equal(test_data.FrobEntity))
+	})
+
+	It("converts an entity to a model", func() {
+		converter := frob.FrobConverter{}
+
+		model := converter.ToModel(test_data.FrobEntity)
+
+		Expect(model).To(Equal(test_data.FrobModel))
+	})
+})

pkg/transformers/frob/entity.go

@@ -0,0 +1,30 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package frob
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+type FrobEntity struct {
+	Ilk [32]byte
+	Lad common.Address
+	Gem *big.Int
+	Ink *big.Int
+	Art *big.Int
+	Era *big.Int
+}

pkg/transformers/frob/frob_suite_test.go

@@ -0,0 +1,27 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package frob_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestFrob(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Frob Suite")
+}

pkg/transformers/frob/integration_test.go

@@ -0,0 +1,84 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package frob_test
+
+import (
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
+	vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Integration tests", func() {
+	It("Fetches frob event logs from a local test chain", func() {
+		ipcPath := test_config.TestClient.IPCPath
+
+		rawRpcClient, err := rpc.Dial(ipcPath)
+		Expect(err).NotTo(HaveOccurred())
+
+		rpcClient := client.NewRpcClient(rawRpcClient, ipcPath)
+		ethClient := ethclient.NewClient(rawRpcClient)
+		blockChainClient := client.NewEthClient(ethClient)
+		realNode := node.MakeNode(rpcClient)
+		transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
+		realBlockChain := geth.NewBlockChain(blockChainClient, realNode, transactionConverter)
+		realFetcher := shared.NewFetcher(realBlockChain)
+		topic0 := common.HexToHash(frob.FrobEventSignature)
+		topics := [][]common.Hash{{topic0}}
+
+		result, err := realFetcher.FetchLogs(test_data.TemporaryFrobAddress, topics, int64(12))
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(len(result) > 0).To(BeTrue())
+		Expect(result[0].Address).To(Equal(common.HexToAddress(test_data.TemporaryFrobAddress)))
+		Expect(result[0].TxHash).To(Equal(test_data.EthFrobLog.TxHash))
+		Expect(result[0].BlockNumber).To(Equal(test_data.EthFrobLog.BlockNumber))
+		Expect(result[0].Topics).To(Equal(test_data.EthFrobLog.Topics))
+		Expect(result[0].Index).To(Equal(test_data.EthFrobLog.Index))
+	})
+
+	It("unpacks an event log", func() {
+		address := common.HexToAddress(test_data.TemporaryFrobAddress)
+		abi, err := geth.ParseAbi(frob.FrobABI)
+		Expect(err).NotTo(HaveOccurred())
+
+		contract := bind.NewBoundContract(address, abi, nil, nil, nil)
+		entity := &frob.FrobEntity{}
+
+		var eventLog = test_data.EthFrobLog
+
+		err = contract.UnpackLog(entity, "Frob", eventLog)
+		Expect(err).NotTo(HaveOccurred())
+
+		expectedEntity := test_data.FrobEntity
+		Expect(entity.Art).To(Equal(expectedEntity.Art))
+		Expect(entity.Era).To(Equal(expectedEntity.Era))
+		Expect(entity.Gem).To(Equal(expectedEntity.Gem))
+		Expect(entity.Ilk).To(Equal(expectedEntity.Ilk))
+		Expect(entity.Ink).To(Equal(expectedEntity.Ink))
+		Expect(entity.Lad).To(Equal(expectedEntity.Lad))
+	})
+})

pkg/transformers/frob/model.go

@@ -0,0 +1,24 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package frob
+
+type FrobModel struct {
+	Ilk []byte
+	Lad []byte
+	Gem string
+	Ink string
+	Art string
+	Era string
+}

pkg/transformers/frob/repository.go

@@ -0,0 +1,57 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package frob
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type Repository interface {
+	Create(headerID int64, transactionIndex uint, model FrobModel) error
+	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
+}
+
+type FrobRepository struct {
+	db *postgres.DB
+}
+
+func NewFrobRepository(db *postgres.DB) FrobRepository {
+	return FrobRepository{db: db}
+}
+
+func (repository FrobRepository) Create(headerID int64, transactionIndex uint, model FrobModel) error {
+	_, err := repository.db.Exec(`INSERT INTO maker.frob (header_id, tx_idx, art, era, gem, ilk, ink, lad)
+		VALUES($1, $2, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6, $7::NUMERIC, $8)`,
+		headerID, transactionIndex, model.Art, model.Era, model.Gem, model.Ilk, model.Ink, model.Lad)
+	return err
+}
+
+func (repository FrobRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
+	var result []core.Header
+	err := repository.db.Select(
+		&result,
+		`SELECT headers.id, headers.block_number FROM headers
+               LEFT JOIN maker.frob on headers.id = header_id
+               WHERE header_id ISNULL
+               AND headers.block_number >= $1
+               AND headers.block_number <= $2
+               AND headers.eth_node_fingerprint = $3`,
+		startingBlockNumber,
+		endingBlockNumber,
+		repository.db.Node.ID,
+	)
+	return result, err
+}

pkg/transformers/frob/repository_test.go

@@ -0,0 +1,147 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package frob_test
+
+import (
+	"database/sql"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Frob repository", func() {
+	Describe("Create", func() {
+		It("adds a frob", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			frobRepository := frob.NewFrobRepository(db)
+
+			err = frobRepository.Create(headerID, 123, test_data.FrobModel)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbFrob frob.FrobModel
+			err = db.Get(&dbFrob, `SELECT art, era, gem, ilk, ink, lad FROM maker.frob WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(dbFrob).To(Equal(test_data.FrobModel))
+		})
+
+		It("does not duplicate frob events", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			frobRepository := frob.NewFrobRepository(db)
+			err = frobRepository.Create(headerID, 123, test_data.FrobModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = frobRepository.Create(headerID, 123, test_data.FrobModel)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
+		})
+
+		It("removes frob if corresponding header is deleted", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			frobRepository := frob.NewFrobRepository(db)
+			err = frobRepository.Create(headerID, 123, test_data.FrobModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbFrob frob.FrobModel
+			err = db.Get(&dbFrob, `SELECT art, era, gem, ilk, ink, lad FROM maker.frob WHERE header_id = $1`, headerID)
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(sql.ErrNoRows))
+		})
+	})
+
+	Describe("MissingHeaders", func() {
+		It("returns headers with no associated frob event", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			startingBlockNumber := int64(1)
+			frobBlockNumber := int64(2)
+			endingBlockNumber := int64(3)
+			blockNumbers := []int64{startingBlockNumber, frobBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			frobRepository := frob.NewFrobRepository(db)
+			err := frobRepository.Create(headerIDs[1], 123, test_data.FrobModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := frobRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(2))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+		})
+
+		It("only returns headers associated with the current node", func() {
+			nodeOne := core.Node{}
+			db := test_config.NewTestDB(nodeOne)
+			test_config.CleanTestDB(db)
+			blockNumbers := []int64{1, 2, 3}
+			headerRepository := repositories.NewHeaderRepository(db)
+			nodeTwo := core.Node{ID: "second"}
+			dbTwo := test_config.NewTestDB(nodeTwo)
+			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				Expect(err).NotTo(HaveOccurred())
+			}
+			frobRepository := frob.NewFrobRepository(db)
+			frobRepositoryTwo := frob.NewFrobRepository(dbTwo)
+			err := frobRepository.Create(headerIDs[0], 0, test_data.FrobModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			nodeOneMissingHeaders, err := frobRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1))
+
+			nodeTwoMissingHeaders, err := frobRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers)))
+		})
+	})
+})

pkg/transformers/frob/transformer.go

@@ -0,0 +1,70 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package frob
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+)
+
+type FrobTransformer struct {
+	Converter  Converter
+	Fetcher    shared.LogFetcher
+	Repository Repository
+}
+
+type FrobTransformerInitializer struct {
+	Config TransformerConfig
+}
+
+func (initializer FrobTransformerInitializer) NewFrobTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
+	converter := FrobConverter{}
+	fetcher := shared.NewFetcher(blockChain)
+	repository := NewFrobRepository(db)
+	return FrobTransformer{
+		Converter:  converter,
+		Fetcher:    fetcher,
+		Repository: repository,
+	}
+}
+
+func (transformer FrobTransformer) Execute() error {
+	missingHeaders, err := transformer.Repository.MissingHeaders(FrobConfig.StartingBlockNumber, FrobConfig.EndingBlockNumber)
+	if err != nil {
+		return err
+	}
+	for _, header := range missingHeaders {
+		topics := [][]common.Hash{{common.HexToHash(FrobEventSignature)}}
+		matchingLogs, err := transformer.Fetcher.FetchLogs(FrobConfig.ContractAddress, topics, header.BlockNumber)
+		if err != nil {
+			return err
+		}
+		for _, log := range matchingLogs {
+			entity, err := transformer.Converter.ToEntity(FrobConfig.ContractAddress, FrobConfig.ContractAbi, log)
+			if err != nil {
+				return err
+			}
+			model := transformer.Converter.ToModel(entity)
+			err = transformer.Repository.Create(header.Id, log.TxIndex, model)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

pkg/transformers/frob/transformer_test.go

@@ -0,0 +1,176 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package frob_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
+	frob_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/frob"
+)
+
+var _ = Describe("Frob transformer", func() {
+	It("gets missing headers for block numbers specified in config", func() {
+		repository := &frob_mocks.MockFrobRepository{}
+		transformer := frob.FrobTransformer{
+			Fetcher:    &mocks.MockLogFetcher{},
+			Converter:  &frob_mocks.MockFrobConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.PassedStartingBlockNumber).To(Equal(frob.FrobConfig.StartingBlockNumber))
+		Expect(repository.PassedEndingBlockNumber).To(Equal(frob.FrobConfig.EndingBlockNumber))
+	})
+
+	It("returns error if repository returns error for missing headers", func() {
+		repository := &frob_mocks.MockFrobRepository{}
+		repository.SetMissingHeadersErr(fakes.FakeError)
+		transformer := frob.FrobTransformer{
+			Fetcher:    &mocks.MockLogFetcher{},
+			Converter:  &frob_mocks.MockFrobConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("fetches logs for missing headers", func() {
+		fetcher := &mocks.MockLogFetcher{}
+		repository := &frob_mocks.MockFrobRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}})
+		transformer := frob.FrobTransformer{
+			Fetcher:    fetcher,
+			Converter:  &frob_mocks.MockFrobConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2}))
+		Expect(fetcher.FetchedContractAddress).To(Equal(frob.FrobConfig.ContractAddress))
+		Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(frob.FrobEventSignature)}}))
+	})
+
+	It("returns error if fetcher returns error", func() {
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetcherError(fakes.FakeError)
+		repository := &frob_mocks.MockFrobRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		transformer := frob.FrobTransformer{
+			Fetcher:    fetcher,
+			Converter:  &frob_mocks.MockFrobConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("converts matching logs", func() {
+		converter := &frob_mocks.MockFrobConverter{}
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
+		repository := &frob_mocks.MockFrobRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		transformer := frob.FrobTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(converter.PassedContractAddress).To(Equal(frob.FrobConfig.ContractAddress))
+		Expect(converter.PassedContractABI).To(Equal(frob.FrobConfig.ContractAbi))
+		Expect(converter.PassedLog).To(Equal(test_data.EthFrobLog))
+		Expect(converter.PassedEntity).To(Equal(test_data.FrobEntity))
+	})
+
+	It("returns error if converter returns error", func() {
+		converter := &frob_mocks.MockFrobConverter{}
+		converter.SetConverterError(fakes.FakeError)
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
+		repository := &frob_mocks.MockFrobRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		transformer := frob.FrobTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("persists frob model", func() {
+		converter := &frob_mocks.MockFrobConverter{}
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
+		repository := &frob_mocks.MockFrobRepository{}
+		fakeHeader := core.Header{BlockNumber: 1, Id: 2}
+		repository.SetMissingHeaders([]core.Header{fakeHeader})
+		transformer := frob.FrobTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
+		Expect(repository.PassedTransactionIndex).To(Equal(test_data.EthFrobLog.TxIndex))
+		Expect(repository.PassedFrobModel).To(Equal(test_data.FrobModel))
+	})
+
+	It("returns error if repository returns error for create", func() {
+		converter := &frob_mocks.MockFrobConverter{}
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
+		repository := &frob_mocks.MockFrobRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}})
+		repository.SetCreateError(fakes.FakeError)
+		transformer := frob.FrobTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+})

pkg/transformers/shared/fetcher.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package flip_kick
+package shared
 
 import (
 	"math/big"
@@ -29,24 +29,23 @@ type LogFetcher interface {
 }
 
 type Fetcher struct {
-	Blockchain core.BlockChain
+	blockChain core.BlockChain
 }
 
 func NewFetcher(blockchain core.BlockChain) Fetcher {
 	return Fetcher{
-		Blockchain: blockchain,
+		blockChain: blockchain,
 	}
 }
 
-func (f Fetcher) FetchLogs(contractAddress string, topicZeros [][]common.Hash, blockNumber int64) ([]types.Log, error) {
+func (fetcher Fetcher) FetchLogs(contractAddress string, topics [][]common.Hash, blockNumber int64) ([]types.Log, error) {
 	block := big.NewInt(blockNumber)
 	address := common.HexToAddress(contractAddress)
 	query := ethereum.FilterQuery{
 		FromBlock: block,
 		ToBlock:   block,
 		Addresses: []common.Address{address},
-		Topics:    topicZeros,
+		Topics:    topics,
 	}
-
-	return f.Blockchain.GetEthLogsWithCustomQuery(query)
+	return fetcher.blockChain.GetEthLogsWithCustomQuery(query)
 }

pkg/transformers/shared/fetcher_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package flip_kick_test
+package shared_test
 
 import (
 	"math/big"
@@ -23,38 +23,39 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
-	"github.com/vulcanize/vulcanizedb/pkg/transformers/flip_kick"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
 var _ = Describe("Fetcher", func() {
 	Describe("FetchLogs", func() {
-		var blockChain *fakes.MockBlockChain
-		var fetcher flip_kick.Fetcher
-
-		BeforeEach(func() {
-			blockChain = fakes.NewMockBlockChain()
-			fetcher = flip_kick.Fetcher{Blockchain: blockChain}
-		})
 
 		It("fetches logs based on the given query", func() {
-			blockNumber := int64(3)
-			address := "0x4D2"
-			topicZeros := [][]common.Hash{{common.HexToHash("0x")}}
+			blockChain := fakes.NewMockBlockChain()
+			fetcher := shared.NewFetcher(blockChain)
+			blockNumber := int64(123)
+			address := "0xfakeAddress"
+			topicZeros := [][]common.Hash{{common.BytesToHash([]byte{1, 2, 3, 4, 5})}}
 
-			query := ethereum.FilterQuery{
+			_, err := fetcher.FetchLogs(address, topicZeros, blockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			expectedQuery := ethereum.FilterQuery{
 				FromBlock: big.NewInt(blockNumber),
 				ToBlock:   big.NewInt(blockNumber),
 				Addresses: []common.Address{common.HexToAddress(address)},
 				Topics:    topicZeros,
 			}
-			fetcher.FetchLogs(address, topicZeros, blockNumber)
-			blockChain.AssertGetEthLogsWithCustomQueryCalledWith(query)
+			blockChain.AssertGetEthLogsWithCustomQueryCalledWith(expectedQuery)
 
 		})
 
 		It("returns an error if fetching the logs fails", func() {
+			blockChain := fakes.NewMockBlockChain()
 			blockChain.SetGetLogsErr(fakes.FakeError)
+			fetcher := shared.NewFetcher(blockChain)
+
 			_, err := fetcher.FetchLogs("", [][]common.Hash{}, int64(1))
+
 			Expect(err).To(HaveOccurred())
 			Expect(err).To(MatchError(fakes.FakeError))
 		})

pkg/transformers/shared/shared_suite_test.go

@@ -0,0 +1,27 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shared_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestShared(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Shared Suite")
+}

pkg/transformers/shared/transformer.go

@@ -0,0 +1,38 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shared
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type Transformer interface {
+	Execute() error
+}
+
+type TransformerInitializer func(db *postgres.DB, blockChain core.BlockChain) Transformer
+
+func HexToInt64(byteString string) int64 {
+	value := common.HexToHash(byteString)
+	return value.Big().Int64()
+}
+
+func HexToString(byteString string) string {
+	value := common.HexToHash(byteString)
+	return value.Big().String()
+}

pkg/transformers/test_data/flip_kick.go

@@ -24,13 +24,13 @@ import (
 	"time"
 )
 
-var TemporaryFlipAddress = "0x08cb6176addcca2e1d1ffe21bee464b72ee4cd8d"
-var TemporaryFlipKickTransaction = "0x6b155a55fd77b751195deeebf7abfd8691ca01ee588817a920f19d5b27f65191"
-var Topic0Hex = "0xc2d733c352a325d6ecf23d1cf0a189dbcc236b0636388b8c5e6953c061d7f0b3"
-var TemporaryFlipKickData = "0x000000000000000000000000000000000000000000000000000000000000000100000000000000000000000008cb6176addcca2e1d1ffe21bee464b72ee4cd8d00000000000000000000000038219779a699d67d7e7740b8c8f43d3e2dae218266616b6520696c6b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064d922894153be9eef7b7218dc565d1d0ce2a09200000000000000000000000007fa9ef6609ca7921112231f8f195138ebba2977000000000000000000000000000000000000000000000000000000005b69b8e7000000000000000000000000000000000000000000000000000000005b607e670000000000000000000000007340e006f4135ba6970d43bf43d88dcad4e7a8ca0000000000000000000000000000000000000000000000000000000000000032"
-var TempBlockNumber = int64(10)
-var TemporaryFlipKickBlockNumber = int64(TempBlockNumber)
-var TemporaryFlipKickBlockHash = "0x32f8b12023b3a1b4c73f9a46da976931b0355714ada8b8044ebcb2cd295751a9"
+var (
+	TemporaryFlipBlockNumber     = int64(10)
+	TemporaryFlipAddress         = "0x08cb6176addcca2e1d1ffe21bee464b72ee4cd8d"
+	TemporaryFlipKickBlockHash   = "0x32f8b12023b3a1b4c73f9a46da976931b0355714ada8b8044ebcb2cd295751a9"
+	TemporaryFlipKickData        = "0x000000000000000000000000000000000000000000000000000000000000000100000000000000000000000008cb6176addcca2e1d1ffe21bee464b72ee4cd8d00000000000000000000000038219779a699d67d7e7740b8c8f43d3e2dae218266616b6520696c6b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064d922894153be9eef7b7218dc565d1d0ce2a09200000000000000000000000007fa9ef6609ca7921112231f8f195138ebba2977000000000000000000000000000000000000000000000000000000005b69b8e7000000000000000000000000000000000000000000000000000000005b607e670000000000000000000000007340e006f4135ba6970d43bf43d88dcad4e7a8ca0000000000000000000000000000000000000000000000000000000000000032"
+	TemporaryFlipKickTransaction = "0x6b155a55fd77b751195deeebf7abfd8691ca01ee588817a920f19d5b27f65191"
+)
 
 var idString = "1"
 var id, _ = new(big.Int).SetString(idString, 10)
@@ -53,7 +53,7 @@ var EthFlipKickLog = types.Log{
 	Address:     common.HexToAddress(TemporaryFlipAddress),
 	Topics:      []common.Hash{common.HexToHash(flip_kick.FlipKickSignature)},
 	Data:        hexutil.MustDecode(TemporaryFlipKickData),
-	BlockNumber: uint64(TempBlockNumber),
+	BlockNumber: uint64(TemporaryFlipBlockNumber),
 	TxHash:      common.HexToHash(TemporaryFlipKickTransaction),
 	TxIndex:     0,
 	BlockHash:   common.HexToHash(TemporaryFlipKickBlockHash),

pkg/transformers/test_data/frob.go

@@ -0,0 +1,70 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test_data
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
+	"math/big"
+)
+
+var (
+	TemporaryFrobAddress     = "0xff3f2400f1600f3f493a9a92704a29b96795af1a"
+	TemporaryFrobBlockHash   = common.HexToHash("0xe1c4264e245ac31d4aed678df007199cffcfcaea7a75aeecc45122957abf4298")
+	TemporaryFrobBlockNumber = int64(12)
+	TemporaryFrobData        = "0x66616b6520696c6b00000000000000000000000000000000000000000000000000000000000000000000000064d922894153be9eef7b7218dc565d1d0ce2a092fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff6000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005b6b39e4"
+	TemporaryFrobTransaction = "0xbcff98316acb5732891d1a7e02f23ec12fbf8c231ca4b5530fa7a21c1e9b6aa9"
+)
+
+var (
+	// need to set bytes as 0 or else the big Int 0 evaluates differently from the one unpacked by the abi
+	art     = big.NewInt(0).SetBytes([]byte{0})
+	frobEra = big.NewInt(1533753828)
+	frobLad = common.HexToAddress("0x64d922894153BE9EEf7b7218dc565d1D0Ce2a092")
+	gem, _  = big.NewInt(0).SetString("115792089237316195423570985008687907853269984665640564039457584007913129639926", 10)
+	ink     = big.NewInt(10)
+)
+
+var EthFrobLog = types.Log{
+	Address:     common.HexToAddress(TemporaryFrobAddress),
+	Topics:      []common.Hash{common.HexToHash(frob.FrobEventSignature)},
+	Data:        hexutil.MustDecode(TemporaryFrobData),
+	BlockNumber: uint64(TemporaryFrobBlockNumber),
+	TxHash:      common.HexToHash(TemporaryFrobTransaction),
+	TxIndex:     123,
+	BlockHash:   TemporaryFrobBlockHash,
+	Index:       1,
+	Removed:     false,
+}
+
+var FrobEntity = frob.FrobEntity{
+	Ilk: ilk,
+	Lad: frobLad,
+	Gem: gem,
+	Ink: ink,
+	Art: art,
+	Era: frobEra,
+}
+
+var FrobModel = frob.FrobModel{
+	Ilk: ilk[:],
+	Lad: frobLad[:],
+	Gem: gem.String(),
+	Ink: ink.String(),
+	Art: art.String(),
+	Era: frobEra.String(),
+}

pkg/transformers/test_data/mocks/flip_kick/converter.go

@@ -0,0 +1,44 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package flip_kick
+
+import (
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/flip_kick"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+)
+
+type MockFlipKickConverter struct {
+	ConverterContract string
+	ConverterAbi      string
+	LogsToConvert     []types.Log
+	EntitiesToConvert []flip_kick.FlipKickEntity
+	ConverterError    error
+}
+
+func (mfkc *MockFlipKickConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (*flip_kick.FlipKickEntity, error) {
+	mfkc.ConverterContract = contractAddress
+	mfkc.ConverterAbi = contractAbi
+	mfkc.LogsToConvert = append(mfkc.LogsToConvert, ethLog)
+	return &test_data.FlipKickEntity, mfkc.ConverterError
+}
+
+func (mfkc *MockFlipKickConverter) ToModel(flipKickEntity flip_kick.FlipKickEntity) (flip_kick.FlipKickModel, error) {
+	mfkc.EntitiesToConvert = append(mfkc.EntitiesToConvert, flipKickEntity)
+	return test_data.FlipKickModel, nil
+}
+func (mfkc *MockFlipKickConverter) SetConverterError(err error) {
+	mfkc.ConverterError = err
+}

pkg/transformers/test_data/mocks/flip_kick/repository.go

@@ -12,63 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package test_data
+package flip_kick
 
 import (
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/flip_kick"
 )
 
-type MockLogFetcher struct {
-	FetchedContractAddress string
-	FetchedTopics          [][]common.Hash
-	FetchedBlocks          []int64
-	FetcherError           error
-	FetchedLogs            []types.Log
-}
-
-func (mlf *MockLogFetcher) FetchLogs(contractAddress string, topics [][]common.Hash, blockNumber int64) ([]types.Log, error) {
-	mlf.FetchedContractAddress = contractAddress
-	mlf.FetchedTopics = topics
-	mlf.FetchedBlocks = append(mlf.FetchedBlocks, blockNumber)
-
-	return mlf.FetchedLogs, mlf.FetcherError
-}
-
-func (mlf *MockLogFetcher) SetFetcherError(err error) {
-	mlf.FetcherError = err
-}
-
-func (mlf *MockLogFetcher) SetFetchedLogs(logs []types.Log) {
-	mlf.FetchedLogs = logs
-}
-
-type MockFlipKickConverter struct {
-	ConverterContract string
-	ConverterAbi      string
-	LogsToConvert     []types.Log
-	EntitiesToConvert []flip_kick.FlipKickEntity
-	ConverterError    error
-}
-
-func (mfkc *MockFlipKickConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (*flip_kick.FlipKickEntity, error) {
-	mfkc.ConverterContract = contractAddress
-	mfkc.ConverterAbi = contractAbi
-	mfkc.LogsToConvert = append(mfkc.LogsToConvert, ethLog)
-	return &FlipKickEntity, mfkc.ConverterError
-}
-
-func (mfkc *MockFlipKickConverter) ToModel(flipKick flip_kick.FlipKickEntity) (flip_kick.FlipKickModel, error) {
-	mfkc.EntitiesToConvert = append(mfkc.EntitiesToConvert, flipKick)
-	return FlipKickModel, nil
-}
-func (mfkc *MockFlipKickConverter) SetConverterError(err error) {
-	mfkc.ConverterError = err
-}
-
 type MockFlipKickRepository struct {
 	HeaderIds           []int64
 	HeadersToReturn     []core.Header

pkg/transformers/test_data/mocks/frob/converter.go

@@ -0,0 +1,45 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package frob
+
+import (
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+)
+
+type MockFrobConverter struct {
+	PassedContractAddress string
+	PassedContractABI     string
+	PassedLog             types.Log
+	PassedEntity          frob.FrobEntity
+	converterError        error
+}
+
+func (converter *MockFrobConverter) SetConverterError(err error) {
+	converter.converterError = err
+}
+
+func (converter *MockFrobConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (frob.FrobEntity, error) {
+	converter.PassedContractAddress = contractAddress
+	converter.PassedContractABI = contractAbi
+	converter.PassedLog = ethLog
+	return test_data.FrobEntity, converter.converterError
+}
+
+func (converter *MockFrobConverter) ToModel(frobEntity frob.FrobEntity) frob.FrobModel {
+	converter.PassedEntity = frobEntity
+	return test_data.FrobModel
+}

pkg/transformers/test_data/mocks/frob/repository.go

@@ -0,0 +1,56 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package frob
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
+)
+
+type MockFrobRepository struct {
+	createError               error
+	PassedEndingBlockNumber   int64
+	PassedFrobModel           frob.FrobModel
+	PassedHeaderID            int64
+	PassedStartingBlockNumber int64
+	PassedTransactionIndex    uint
+	missingHeaders            []core.Header
+	missingHeadersErr         error
+}
+
+func (repository *MockFrobRepository) SetCreateError(err error) {
+	repository.createError = err
+}
+
+func (repository *MockFrobRepository) SetMissingHeadersErr(err error) {
+	repository.missingHeadersErr = err
+}
+
+func (repository *MockFrobRepository) SetMissingHeaders(headers []core.Header) {
+	repository.missingHeaders = headers
+}
+
+func (repository *MockFrobRepository) Create(headerID int64, transactionIndex uint, model frob.FrobModel) error {
+	repository.PassedHeaderID = headerID
+	repository.PassedTransactionIndex = transactionIndex
+	repository.PassedFrobModel = model
+	return repository.createError
+}
+
+func (repository *MockFrobRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
+	repository.PassedStartingBlockNumber = startingBlockNumber
+	repository.PassedEndingBlockNumber = endingBlockNumber
+	return repository.missingHeaders, repository.missingHeadersErr
+}

pkg/transformers/test_data/mocks/log_fetcher.go

@@ -0,0 +1,44 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mocks
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+type MockLogFetcher struct {
+	FetchedContractAddress string
+	FetchedTopics          [][]common.Hash
+	FetchedBlocks          []int64
+	fetcherError           error
+	FetchedLogs            []types.Log
+}
+
+func (mlf *MockLogFetcher) FetchLogs(contractAddress string, topics [][]common.Hash, blockNumber int64) ([]types.Log, error) {
+	mlf.FetchedContractAddress = contractAddress
+	mlf.FetchedTopics = topics
+	mlf.FetchedBlocks = append(mlf.FetchedBlocks, blockNumber)
+
+	return mlf.FetchedLogs, mlf.fetcherError
+}
+
+func (mlf *MockLogFetcher) SetFetcherError(err error) {
+	mlf.fetcherError = err
+}
+
+func (mlf *MockLogFetcher) SetFetchedLogs(logs []types.Log) {
+	mlf.FetchedLogs = logs
+}

pkg/transformers/transformers.go

@@ -15,14 +15,18 @@
 package transformers
 
 import (
-	"github.com/vulcanize/vulcanizedb/libraries/shared"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/flip_kick"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
 func TransformerInitializers() []shared.TransformerInitializer {
-	config := flip_kick.FlipKickConfig
-	initializer := flip_kick.FlipKickTransformerInitializer{Config: config}
+	flipKickConfig := flip_kick.FlipKickConfig
+	flipKickTransformerInitializer := flip_kick.FlipKickTransformerInitializer{Config: flipKickConfig}
+	frobConfig := frob.FrobConfig
+	frobTransformerInitializer := frob.FrobTransformerInitializer{Config: frobConfig}
 	return []shared.TransformerInitializer{
-		initializer.NewFlipKickTransformer,
+		flipKickTransformerInitializer.NewFlipKickTransformer,
+		frobTransformerInitializer.NewFrobTransformer,
 	}
 }